def main(args):
env = load_env(args)
model = Model(env.observation_space.shape[0], env.action_space.n, args)
dq = deque(maxlen=100)
dq.append(0)
for episode in range(1, args.episodes + 1):
h_out = (torch.zeros([1, 1, 32], dtype=torch.float), torch.zeros([1, 1, 32], dtype=torch.float))
state = env.reset()
done = False
score = 0.
while not done:
for _ in range(args.n_rollout):
h_in = h_out
prob, hidden_out = model._pi(torch.from_numpy(state).float(), h_in)
prob = prob.squeeze()
m = Categorical(prob)
action = m.sample().item()
next_state, reward, done, _ = env.step(action)
model._put_data(state, next_state, reward, done, action, prob[action], hidden_in=h_in, hidden_out=hidden_out)
state = next_state
score += reward
if done:
break
model._train()
dq.append(score)
if episode % args.print_intervals == 0:
print('Episode: {}, Score mean: {}'.format(episode, np.mean(dq)))
def main(args):
env = load_env(args)
model = Model(env.observation_space.shape[0], env.action_space.n, args)
dq = deque(maxlen=100)
dq.append(0)
for episode in range(1, args.episodes + 1):
state = env.reset()
done = False
score = 0.
while not done:
state = torch.from_numpy(state).float()
prob = model(state)
m = Categorical(prob)
action = m.sample().item()
next_state, reward, done, _ = env.step(action)
score += 1.
model._put_data(reward, prob[action])
state = next_state
model._train()
dq.append(score)
if episode % args.print_intervals == 0 and episode != 0:
print("# of episode :{}, avg score : {}".format(episode, np.mean(dq), axis=-1))
def main(args):
env = load_env(args)
model = Model(env.observation_space.shape[0], env.action_space.n, args)
dq = deque(maxlen=100)
dq.append(0)
for episode in range(1, args.episodes + 1):
state = env.reset()
done = False
score = 0.
while not done:
for _ in range(args.n_rollout):
state = torch.from_numpy(state).float()
prob = model._pi(state)
m = Categorical(prob)
action = m.sample().item()
next_state, reward, done, _ = env.step(action)
model._put_data(state.numpy(), next_state, reward, done,
action, prob[action])
state = next_state
score += 1.
if done:
break
model._train()
dq.append(score)
if episode % args.print_intervals == 0:
print('Episdoe: {}, Score Mean: {}'.format(episode, np.mean(dq)))
def _train(global_model, rank, args):
env = load_env(args)
local_model = Model(env.observation_space.shape[0], env.action_space.n, args)
local_model.load_state_dict(global_model.state_dict())
optimizer = optim.Adam(global_model.parameters(), lr=args.lr)
dq = deque(maxlen=100)
dq.append(0)
for episode in range(args.episodes):
done = False
state = env.reset()
score = 0.
while not done:
for _ in range(args.n_rollout):
state = torch.from_numpy(state).float()
prob = local_model._pi(state)
m = Categorical(prob)
action = m.sample().item()
next_state, reward, done, _ = env.step(action)
local_model._put_data(state.numpy(), next_state, reward, done, action, prob[action])
score += 1
if done:
break
state = next_state
local_model._train(optimizer)
for global_param, local_param in zip(global_model.parameters(), local_model.parameters()):
global_param._grad = local_param.grad
dq.append(score)
local_model.load_state_dict(global_model.state_dict())
if episode % args.print_intervals == 0:
print('[Episode: {}, Rank: {}] score: {}'.format(episode, rank, np.mean(dq)))
def main(args):
env = load_env(args)
model = Model(env.observation_space.shape[0], env.action_space.n, args)
memory = ReplayBuffer(args)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
for episode in range(1, args.episodes + 1):
state = env.reset()
done = False
while not done:
seq_data = []
for _ in range(args.n_rollout):
prob = model._pi(torch.from_numpy(state).float())
m = Categorical(prob)
action = m.sample().item()
next_state, reward, done, _ = env.step(action)
seq_data.append((state, next_state, reward / 100., done, action, prob))
state = next_state
if done:
break
memory._put_data(seq_data)
if memory._size() > 500:
_train(model, optimizer, memory, on_policy=True)
_train(model, optimizer, memory)
def main(args):
env = load_env(args)
in_channels, out_channels = env.observation_space.shape[
0], env.action_space.n
print(in_channels, out_channels)
dq = deque(maxlen=100)
dq.append(0)
model = ActorCritic(in_channels, out_channels, args)
if args.is_cuda:
model = model.cuda()
rewards_list = []
episodes_list = []
losses_list = []
for episode in range(args.episodes):
score = 0.
done = False
state = env.reset()
loss = 0.
while not done:
for t in range(args.n_rollout):
state = torch.from_numpy(state).float()
if args.is_cuda:
state = state.cuda()
prob = model._pi(state)
m = Categorical(prob)
action = m.sample().item()
next_state, reward, done, _ = env.step(action)
model._history(state, next_state, reward, done, action,
prob[action])
state = next_state
score += reward
if done:
break
loss += model._train()
rewards_list.append(score)
dq.append(score)
losses_list.append(loss)
episodes_list.append(episode)
if episode % args.print_intervals == 0 and episode != 0:
print("# of episode :{0}, avg score : {1:.2f}".format(episode,
np.mean(dq),
axis=-1))
if not os.path.isdir('./actor_critic'):
os.mkdir('actor_critic')
draw_plot(episodes_list,
losses_list,
rewards_list,
path='./actor_critic/loss_reward_plot.jpg')
def main(args):
lr_mu = 0.0005
lr_q = 0.001
gamma = 0.99
batch_size = 32
buffer_limit = 50000
tau = 0.005 # for target network soft update
args.env = 'Pendulum-v0'
env = load_env(args)
in_channels = env.observation_space.shape[0]
memory = ReplayBuffer(args)
q = QNet(in_channels, args)
mu = MuNet(in_channels, args)
q_target = QNet(in_channels, args)
mu_target = MuNet(in_channels, args)
q_target.load_state_dict(q.state_dict())
mu_target.load_state_dict(mu.state_dict())
q_optimizer = optim.Adam(q.parameters(), lr=lr_q)
mu_optimizer = optim.Adam(mu.parameters(), lr=lr_mu)
ou_noise = OrnsteinUhlenbeckNoise(np.zeros(1))
score = 0.
for episode in range(1, args.episodes):
state = env.reset()
for t in range(300):
state = torch.from_numpy(state).float()
action = mu(state)
action = action.item() + ou_noise()[0]
next_state, reward, done, info = env.step([action])
memory._put_data(
(state.numpy(), next_state, reward / 100., done, action))
score += reward
state = next_state
if done:
break
if memory.size() > 2000:
for i in range(10):
_train(mu, mu_target, q, q_target, memory, q_optimizer,
mu_optimizer, args)
soft_update(mu, mu_target)
soft_update(q, q_target)
if episode % 20 == 0 and episode != 0:
print("# of episode :{}, avg score : {:.1f}".format(
episode, score / 20))
score = 0.0
def main(args):
env = load_env(args)
model = Model(env.observation_space.shape[0], env.action_space.n, args)
dq = deque(maxlen=100)
dq.append(0)
episodes_list = []
losses_list = []
rewards_list = []
for episode in range(1, args.episodes + 1):
state = env.reset()
done = False
score = 0.
losses = 0.
while not done:
for _ in range(args.n_rollout):
state = torch.from_numpy(state).float()
prob = model._pi(state)
m = Categorical(prob)
action = m.sample().item()
next_state, reward, done, _ = env.step(action)
score += 1.
model._put_data(state.numpy(), next_state, reward, done,
action, prob[action])
state = next_state
if done:
break
losses += model._train()
dq.append(score)
if episode % args.print_intervals == 0 and episode != 0:
print("# of episode :{}, avg score : {}".format(episode,
np.mean(dq),
end=' '))
episodes_list.append(episode)
losses_list.append(losses)
rewards_list.append(score)
if not os.path.isdir('./ppo'):
os.mkdir('ppo')
draw_plot(episodes_list,
losses_list,
rewards_list,
path='./ppo/loss_reward_plot.jpg')
def main(args):
env = load_env(args)
global_model = Model(env.observation_space.shape[0], env.action_space.n, args)
global_model.share_memory()
processes = []
for rank in range(args.world_size):
p = mp.Process(target=_train, args=(global_model, rank, args))
p.start()
processes.append(p)
for p in processes:
p.join()
def main(args):
env = load_env(args)
dq = deque(maxlen=100)
dq.append(0)
model = Model(env.observation_space.shape[0], env.action_space.n, args)
if args.is_cuda:
model = model.cuda()
episodes_list = []
losses_list = []
rewards_list = []
for episode in range(args.episodes):
state = env.reset()
done = False
score = 0.
while not done:
# env.render()
state = torch.from_numpy(state).float()
if args.is_cuda:
state = state.cuda()
prob = model(state)
m = Categorical(prob)
action = m.sample()
next_state, reward, done, info = env.step(action.item())
model._put_data(state.numpy(), reward, prob[action], done)
score += reward
state = next_state
losses = model._train()
dq.append(score)
if episode % args.print_intervals == 0 and episode != 0:
print("# of episode :{}, avg score : {}".format(episode,
np.mean(dq),
axis=-1))
episodes_list.append(episode)
losses_list.append(losses)
rewards_list.append(score)
if not os.path.isdir('./reinforce'):
os.mkdir('reinforce')
draw_plot(episodes_list,
losses_list,
rewards_list,
path='./reinforce/loss_reward_plot.jpg')
def main(args):
env = load_env(args)
in_channels, out_channels = env.observation_space.shape[
0], env.action_space.n
q = Model(in_channels, out_channels, args)
q_target = Model(in_channels, out_channels, args)
q_target.load_state_dict(q.state_dict())
memory = ReplayBuffer(args)
score = 0.
optimizer = optim.Adam(q.parameters(), lr=args.lr)
for episode in range(1, args.episodes + 1):
epsilon = max(0.01, 0.08 - 0.01 * (episode / 200))
state = env.reset()
done = False
while not done:
state = torch.from_numpy(state).float()
action = q._sample_action(state, epsilon)
next_state, reward, done, _ = env.step(action)
done_mask = 0. if done else 1.
memory._put_data(
(state.numpy(), next_state, reward / 100., action, done_mask))
state = next_state
score += 1.
if done:
break
if memory.size() > 2000:
_train(q, q_target, memory, optimizer, args)
if episode % args.print_intervals == 0:
q_target.load_state_dict(q.state_dict())
print(
"n_episode :{}, score : {:.1f}, n_buffer : {}, eps : {:.1f}%".
format(episode, score / args.print_intervals, memory.size(),
epsilon * 100))
score = 0.
def main(args):
dq = deque(maxlen=100)
dq.append(0)
env = load_env(args)
in_channels, out_channels = env.observation_space.shape[0], env.action_space.n
model = Model(in_channels, out_channels, args)
if args.is_cuda:
model = model.cuda()
rewards_list = []
episodes_list = []
losses_list = []
for episode in range(args.episodes):
score = 0.
state = env.reset()
done = False
while not done:
for _ in range(args.n_rollout):
state = torch.from_numpy(state).float()
if args.is_cuda:
state = state.cuda()
prob = model._pi(state)
m = Categorical(prob)
action = m.sample().item()
next_state, reward, done, _ = env.step(action)
model._put_data(state, next_state, reward, done, action, prob[action])
score += reward
state = next_state
if done:
break
loss = model._train()
losses_list.append(loss)
dq.append(score)
episodes_list.append(episode)
if episode % args.print_intervals == 0 and episode != 0:
print("# of episode :{}, avg score : {}".format(episode, np.mean(dq), end=' '))
print(np.mean(np.array(losses_list)))
def main(args):
env = load_env(args)
env.seed(500)
torch.manual_seed(500)
in_channels, out_channels = env.observation_space.shape[
0], env.action_space.shape[0]
print(in_channels, out_channels)
actor = Actor(in_channels, out_channels, args)
critic = Critic(in_channels, out_channels, args)
target_critic = Critic(in_channels, out_channels, args)
actor_optimizer = optim.Adam(actor.parameters(), lr=args.actor_lr)
critic_optimizer = optim.Adam(critic.parameters(), lr=args.critic_lr)
hard_target_update(critic, target_critic)
target_entropy = -torch.prod(torch.tensor(out_channels)).item()
log_alpha = torch.zeros(1, requires_grad=True)
alpha = torch.exp(log_alpha)
alpha_optimizer = optim.Adam([log_alpha], lr=args.alpha_lr)
replay_buffer = deque(maxlen=100000)
recent_rewards = deque(maxlen=100)
steps = 0
for episode in range(args.max_iter_num):
done = False
score = 0
state = env.reset()
state = np.reshape(state, [1, in_channels])
while not done:
steps += 1
mu, std = actor(torch.tensor(state).float())
action = get_action(mu, std)
next_state, reward, done, _ = env.step(action)
next_state = np.reshape(next_state, [1, in_channels])
mask = 0. if done else 1.
replay_buffer.append((state, action, reward, next_state, mask))
state = next_state
score += reward
if steps > args.batch_size:
mini_batch = random.sample(replay_buffer, args.batch_size)
actor.train(), critic.train(), target_critic.train()
alpha = _train_model(actor, critic, target_critic, mini_batch,
actor_optimizer, critic_optimizer,
alpha_optimizer, target_entropy,
log_alpha, alpha, args)
soft_target_update(critic, target_critic, args.tau)
if done:
recent_rewards.append(score)
if episode % args.log_interval == 0:
print('{} episode | score_avg: {:.2f}'.format(
episode, np.mean(recent_rewards)))
if np.mean(recent_rewards) > args.goal_score:
# if not os.path.isdir(args.save_path):
# os.makedirs(args.save_path)
# ckpt_path = args.save_path + 'model.pth.tar'
# torch.save(actor.state_dict(), ckpt_path)
print('Recent rewards exceed -300. So end')
break