def sample(eval_env,
agent,
render_eval=False,
start_ckpt=None,
demo_files=None,
nb_epochs=1000,
**kwargs):
with tf_util.single_threaded_session() as sess:
# Run demo network
agent.initialize(sess, start_ckpt=start_ckpt)
sess.graph.finalize()
agent.reset()
dbg_tf_init(sess, agent.dbg_vars)
total_nb_train = 0
pretrain_demo(agent,
eval_env,
demo_files,
total_nb_train,
train_params=[(100, 1.0)],
start_ckpt=start_ckpt)
# run sampling
# agent reward can be disabled.
# eval_env.set_intf_fp(None)
sample_fd1norm2(eval_env,
render_eval=render_eval,
target_nb_resets=nb_epochs)
def test_multikwargs():
with tf.Graph().as_default():
x = tf.placeholder(tf.int32, (), name="x")
with tf.variable_scope("other"):
x2 = tf.placeholder(tf.int32, (), name="x")
z = 3 * x + 2 * x2
lin = function([x, x2], z, givens={x2: 0})
with single_threaded_session():
initialize()
assert lin(2) == 6
assert lin(2, 2) == 10
def test_function():
with tf.Graph().as_default():
x = tf.placeholder(tf.int32, (), name="x")
y = tf.placeholder(tf.int32, (), name="y")
z = 3 * x + 2 * y
lin = function([x, y], z, givens={y: 0})
with single_threaded_session():
initialize()
assert lin(2) == 6
assert lin(x=3) == 9
assert lin(2, 2) == 10
assert lin(x=2, y=3) == 12
def test(eval_env,
agent,
render_eval=True,
nb_epochs=1,
start_ckpt=None,
**kwargs):
logger.info('Start testing:', start_ckpt, '\n')
with tf_util.single_threaded_session() as sess:
agent.initialize(sess, start_ckpt=start_ckpt)
sess.graph.finalize()
for _ in range(nb_epochs):
combined_stats = {}
eval_episode(eval_env, render_eval, agent, combined_stats)
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
logger.dump_tabular()
logger.info('')
def train(env,
nb_epochs,
nb_epoch_cycles,
render_eval,
reward_scale,
render,
param_noise,
actor,
critic,
normalize_returns,
normalize_observations,
critic_l2_reg,
actor_lr,
critic_lr,
action_noise,
popart,
gamma,
clip_norm,
nb_train_steps,
nb_rollout_steps,
nb_eval_steps,
batch_size,
memory,
tau=0.01,
eval_env=None,
param_noise_adaption_interval=50):
rank = MPI.COMM_WORLD.Get_rank()
assert (np.abs(env.action_space.low) == env.action_space.high
).all() # we assume symmetric actions.
max_action = env.action_space.high
logger.info(
'scaling actions by {} before executing in env'.format(max_action))
agent = DDPG(actor,
critic,
memory,
env.observation_space.shape,
env.action_space.shape,
gamma=gamma,
tau=tau,
normalize_returns=normalize_returns,
normalize_observations=normalize_observations,
batch_size=batch_size,
action_noise=action_noise,
param_noise=param_noise,
critic_l2_reg=critic_l2_reg,
actor_lr=actor_lr,
critic_lr=critic_lr,
enable_popart=popart,
clip_norm=clip_norm,
reward_scale=reward_scale)
logger.info('Using agent with the following configuration:')
logger.info(str(agent.__dict__.items()))
# Set up logging stuff only for a single worker.
if rank == 0:
saver = tf.train.Saver()
else:
saver = None
step = 0
episode = 0
eval_episode_rewards_history = deque(maxlen=100)
episode_rewards_history = deque(maxlen=100)
with U.single_threaded_session() as sess:
# Prepare everything.
agent.initialize(sess)
sess.graph.finalize()
agent.reset()
obs = env.reset()
if eval_env is not None:
eval_obs = eval_env.reset()
done = False
episode_reward = 0.
episode_step = 0
episodes = 0
t = 0
epoch = 0
start_time = time.time()
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_episode_eval_rewards = []
epoch_episode_eval_steps = []
epoch_start_time = time.time()
epoch_actions = []
epoch_qs = []
epoch_episodes = 0
for epoch in range(nb_epochs):
for cycle in range(nb_epoch_cycles):
# Perform rollouts.
for t_rollout in range(nb_rollout_steps):
# Predict next action.
action, q = agent.pi(obs, apply_noise=True, compute_Q=True)
assert action.shape == env.action_space.shape
# Execute next action.
if rank == 0 and render:
env.render()
assert max_action.shape == action.shape
new_obs, r, done, info = env.step(
max_action * action
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
t += 1
if rank == 0 and render:
env.render()
episode_reward += r
episode_step += 1
# Book-keeping.
epoch_actions.append(action)
epoch_qs.append(q)
agent.store_transition(obs, action, r, new_obs, done)
obs = new_obs
if done:
# Episode done.
epoch_episode_rewards.append(episode_reward)
episode_rewards_history.append(episode_reward)
epoch_episode_steps.append(episode_step)
episode_reward = 0.
episode_step = 0
epoch_episodes += 1
episodes += 1
agent.reset()
obs = env.reset()
# Train.
epoch_actor_losses = []
epoch_critic_losses = []
epoch_adaptive_distances = []
for t_train in range(nb_train_steps):
# Adapt param noise, if necessary.
if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0:
distance = agent.adapt_param_noise()
epoch_adaptive_distances.append(distance)
cl, al = agent.train()
epoch_critic_losses.append(cl)
epoch_actor_losses.append(al)
agent.update_target_net()
# Evaluate.
eval_episode_rewards = []
eval_qs = []
if eval_env is not None:
eval_episode_reward = 0.
for t_rollout in range(nb_eval_steps):
eval_action, eval_q = agent.pi(eval_obs,
apply_noise=False,
compute_Q=True)
eval_obs, eval_r, eval_done, eval_info = eval_env.step(
max_action * eval_action
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
if render_eval:
eval_env.render()
eval_episode_reward += eval_r
eval_qs.append(eval_q)
if eval_done:
eval_obs = eval_env.reset()
eval_episode_rewards.append(eval_episode_reward)
eval_episode_rewards_history.append(
eval_episode_reward)
eval_episode_reward = 0.
mpi_size = MPI.COMM_WORLD.Get_size()
# Log stats.
# XXX shouldn't call np.mean on variable length lists
duration = time.time() - start_time
stats = agent.get_stats()
combined_stats = stats.copy()
combined_stats['rollout/return'] = np.mean(epoch_episode_rewards)
combined_stats['rollout/return_history'] = np.mean(
episode_rewards_history)
combined_stats['rollout/episode_steps'] = np.mean(
epoch_episode_steps)
combined_stats['rollout/actions_mean'] = np.mean(epoch_actions)
combined_stats['rollout/Q_mean'] = np.mean(epoch_qs)
combined_stats['train/loss_actor'] = np.mean(epoch_actor_losses)
combined_stats['train/loss_critic'] = np.mean(epoch_critic_losses)
combined_stats['train/param_noise_distance'] = np.mean(
epoch_adaptive_distances)
combined_stats['total/duration'] = duration
combined_stats['total/steps_per_second'] = float(t) / float(
duration)
combined_stats['total/episodes'] = episodes
combined_stats['rollout/episodes'] = epoch_episodes
combined_stats['rollout/actions_std'] = np.std(epoch_actions)
# Evaluation statistics.
if eval_env is not None:
combined_stats['eval/return'] = eval_episode_rewards
combined_stats['eval/return_history'] = np.mean(
eval_episode_rewards_history)
combined_stats['eval/Q'] = eval_qs
combined_stats['eval/episodes'] = len(eval_episode_rewards)
def as_scalar(x):
if isinstance(x, np.ndarray):
assert x.size == 1
return x[0]
elif np.isscalar(x):
return x
else:
raise ValueError('expected scalar, got %s' % x)
combined_stats_sums = MPI.COMM_WORLD.allreduce(
np.array([as_scalar(x) for x in combined_stats.values()]))
combined_stats = {
k: v / mpi_size
for (k, v) in zip(combined_stats.keys(), combined_stats_sums)
}
# Total statistics.
combined_stats['total/epochs'] = epoch + 1
combined_stats['total/steps'] = t
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
logger.dump_tabular()
logger.info('')
logdir = logger.get_dir()
if rank == 0 and logdir:
if hasattr(env, 'get_state'):
with open(os.path.join(logdir, 'env_state.pkl'),
'wb') as f:
pickle.dump(env.get_state(), f)
if eval_env and hasattr(eval_env, 'get_state'):
with open(os.path.join(logdir, 'eval_env_state.pkl'),
'wb') as f:
pickle.dump(eval_env.get_state(), f)
def train(env, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic,
normalize_returns, normalize_observations, critic_l2_reg, actor_lr, critic_lr, action_noise,
popart, gamma, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_steps, batch_size, memory,
tau=0.01, eval_env=None, param_noise_adaption_interval=50):
assert (np.abs(env.action_space.low) == env.action_space.high).all() # we assume symmetric actions.
max_action = env.action_space.high
logger.info('scaling actions by {} before executing in env'.format(max_action))
agent = DDPG(actor, critic, memory, env.observation_space.shape, env.action_space.shape,
gamma=gamma, tau=tau, normalize_returns=normalize_returns,
normalize_observations=normalize_observations,
batch_size=batch_size, action_noise=action_noise, param_noise=param_noise, critic_l2_reg=critic_l2_reg,
actor_lr=actor_lr, critic_lr=critic_lr, enable_popart=popart, clip_norm=clip_norm,
reward_scale=reward_scale)
logger.info('Using agent with the following configuration:')
logger.info(str(agent.__dict__.items()))
# Set up logging stuff only for a single worker.
saver = tf.train.Saver()
step = 0
episode = 0
eval_episode_rewards_history = deque(maxlen=100)
episode_rewards_history = deque(maxlen=100)
with U.single_threaded_session() as sess:
# Prepare everything.
agent.initialize(sess)
sess.graph.finalize()
agent.reset()
obs = env.reset()
if eval_env is not None:
eval_obs = eval_env.reset()
done = False
episode_reward = 0.
episode_step = 0
episodes = 0
t = 0
epoch = 0
start_time = time.time()
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_episode_eval_rewards = []
epoch_episode_eval_steps = []
epoch_start_time = time.time()
epoch_actions = []
epoch_qs = []
epoch_episodes = 0
for epoch in range(nb_epochs):
for cycle in range(nb_epoch_cycles):
# Perform rollouts.
for t_rollout in range(nb_rollout_steps):
# Predict next action.
action, q = agent.pi(obs, apply_noise=True, compute_Q=True)
assert action.shape == env.action_space.shape
# Execute next action.
if render:
env.render()
assert max_action.shape == action.shape
new_obs, r, done, info = env.step(
max_action * action) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
t += 1
if render:
env.render()
episode_reward += r
episode_step += 1
# Book-keeping.
epoch_actions.append(action)
epoch_qs.append(q)
agent.store_transition(obs, action, r, new_obs, done)
obs = new_obs
if done:
# Episode done.
epoch_episode_rewards.append(episode_reward)
episode_rewards_history.append(episode_reward)
epoch_episode_steps.append(episode_step)
episode_reward = 0.
episode_step = 0
epoch_episodes += 1
episodes += 1
agent.reset()
obs = env.reset()
# Train.
epoch_actor_losses = []
epoch_critic_losses = []
epoch_adaptive_distances = []
for t_train in range(nb_train_steps):
# Adapt param noise, if necessary.
if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0:
distance = agent.adapt_param_noise()
epoch_adaptive_distances.append(distance)
cl, al = agent.train()
epoch_critic_losses.append(cl)
epoch_actor_losses.append(al)
agent.update_target_net()
# Evaluate.
eval_episode_rewards = []
eval_qs = []
if eval_env is not None:
eval_episode_reward = 0.
for t_rollout in range(nb_eval_steps):
eval_action, eval_q = agent.pi(eval_obs, apply_noise=False, compute_Q=True)
eval_obs, eval_r, eval_done, eval_info = eval_env.step(
max_action * eval_action) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
if render_eval:
eval_env.render()
eval_episode_reward += eval_r
eval_qs.append(eval_q)
if eval_done:
eval_obs = eval_env.reset()
eval_episode_rewards.append(eval_episode_reward)
eval_episode_rewards_history.append(eval_episode_reward)
eval_episode_reward = 0.
# Log stats.
# XXX shouldn't call np.mean on variable length lists
duration = time.time() - start_time
stats = agent.get_stats()
combined_stats = stats.copy()
combined_stats['rollout/return'] = np.mean(epoch_episode_rewards)
combined_stats['rollout/return_history'] = np.mean(episode_rewards_history)
combined_stats['rollout/episode_steps'] = np.mean(epoch_episode_steps)
combined_stats['rollout/actions_mean'] = np.mean(epoch_actions)
combined_stats['rollout/Q_mean'] = np.mean(epoch_qs)
combined_stats['train/loss_actor'] = np.mean(epoch_actor_losses)
combined_stats['train/loss_critic'] = np.mean(epoch_critic_losses)
combined_stats['train/param_noise_distance'] = np.mean(epoch_adaptive_distances)
combined_stats['total/duration'] = duration
combined_stats['total/steps_per_second'] = float(t) / float(duration)
combined_stats['total/episodes'] = episodes
combined_stats['rollout/episodes'] = epoch_episodes
combined_stats['rollout/actions_std'] = np.std(epoch_actions)
# Evaluation statistics.
if eval_env is not None:
combined_stats['eval/return'] = eval_episode_rewards
combined_stats['eval/return_history'] = np.mean(eval_episode_rewards_history)
combined_stats['eval/Q'] = eval_qs
combined_stats['eval/episodes'] = len(eval_episode_rewards)
def as_scalar(x):
if isinstance(x, np.ndarray):
assert x.size == 1
return x[0]
elif np.isscalar(x):
return x
else:
raise ValueError('expected scalar, got %s' % x)
# Total statistics.
combined_stats['total/epochs'] = epoch + 1
combined_stats['total/steps'] = t
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
logger.dump_tabular()
logger.info('')
logdir = logger.get_dir()
if logdir:
if hasattr(env, 'get_state'):
with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as f:
pickle.dump(env.get_state(), f)
if eval_env and hasattr(eval_env, 'get_state'):
with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as f:
pickle.dump(eval_env.get_state(), f)
parser.add_argument("--max_episode_len", type = int, default = 50, help = "max episode length")
parser.add_argument("--warm_up_steps", type = int, default = 1000, help = "set the warm up steps")
parser.add_argument("--lr", type = float, default = 0.001, help = "learning rate")
parser.add_argument("--gamma", type = float, default = 0.99, help = "discount rate")
return parser.parse_args()
if __name__ == '__main__':
arglist = parse_args()
model_path = './tmp/model'
random.seed(arglist.seed )
log_path = './logs/{}_{}_{}_{}.csv'.format(arglist.game, arglist.agent, arglist.seed, time.time())
log = open(log_path, '+w', 1)
with U.single_threaded_session():
env = PongGame()
obs_space = env.observation_space
act_space = env.action_space
agent1 = DDPGLearner(obs_space, act_space, "agent1",arglist)
if arglist.adv_agent == "agent":
agent2 = DDPGLearner(obs_space, act_space, "agent2", arglist)
agent2 = agent1
U.initialize()
saver = tf.train.Saver()
for epo in range(arglist.epoch):
agent1.reset_replay_buffer()
agent2.reset_replay_buffer()
agent1_q_loss = []
agent1_p_loss = []
def train_maddpg(arglist):
with U.single_threaded_session():
# Create environment
env = make_env(arglist.scenario, arglist, arglist.benchmark)
# Create agent trainers
obs_shape_n = [env.observation_space[i].shape for i in range(env.n)]
num_adversaries = min(env.n, arglist.num_adversaries)
trainers = get_trainers(env,
num_adversaries,
obs_shape_n,
arglist,
good_agent_mode=arglist.good_policy,
adv_agent_mode=arglist.adv_policy)
print('Using good policy {} and adv policy {}'.format(
arglist.good_policy, arglist.adv_policy))
# Initialize
U.initialize()
# Load previous results, if necessary
if arglist.load_dir == "":
arglist.load_dir = arglist.save_dir
if arglist.display or arglist.restore or arglist.benchmark:
print('Loading previous state...')
U.load_state(arglist.load_dir)
episode_rewards = [0.0] # sum of rewards for all agents
agent_rewards = [[0.0]
for _ in range(env.n)] # individual agent reward
final_ep_rewards = [] # sum of rewards for training curve
final_ep_ag_rewards = [] # agent rewards for training curve
agent_info = [[[]]] # placeholder for benchmarking info
saver = tf.train.Saver(max_to_keep=None)
obs_n = env.reset()
episode_step = 0
train_step = 0
t_start = time.time()
if arglist.real_q_log:
world_state_buffer, action_n_buffer, start_episode_step_buffer, obs_n_buffer = [], [], [], []
q_means, real_means = [], []
print('Starting iterations...')
while True:
# get action
action_n = [
agent.action(obs) for agent, obs in zip(trainers, obs_n)
]
# environment step
new_obs_n, rew_n, done_n, info_n = env.step(action_n)
episode_step += 1
done = all(done_n) # note: unused, never happens
terminal = (episode_step >= arglist.max_episode_len)
done = done or terminal
if arglist.real_q_log:
world_state_buffer.append(deepcopy(env.world))
obs_n_buffer.append(obs_n)
action_n_buffer.append(action_n)
start_episode_step_buffer.append(episode_step)
# collect experience
for i, agent in enumerate(trainers):
agent.experience(obs_n[i], action_n[i], rew_n[i], new_obs_n[i],
done, terminal)
obs_n = new_obs_n
for i, rew in enumerate(rew_n):
episode_rewards[-1] += rew
agent_rewards[i][-1] += rew
if done or terminal:
obs_n = env.reset()
episode_step = 0
episode_rewards.append(0) # add element for next episode
for a in agent_rewards:
a.append(0)
agent_info.append([[]])
# increment global step counter
train_step += 1
# for benchmarking learned policies
if arglist.benchmark:
for i, info in enumerate(info_n):
agent_info[-1][i].append(info_n['n'])
if train_step > arglist.benchmark_iters and (done or terminal):
file_name = arglist.benchmark_dir + arglist.exp_name + '.pkl'
print('Finished benchmarking, now saving...')
with open(file_name, 'wb') as fp:
pickle.dump(agent_info[:-1], fp)
break
continue
# for displaying learned policies
if arglist.display:
time.sleep(0.1)
env.render()
continue
for agent in trainers:
loss = agent.update(trainers, train_step)
# save model, display training output
if terminal and (len(episode_rewards) % arglist.save_rate == 0):
if arglist.save_dir != '/tmp/policy/':
U.save_state(arglist.save_dir + arglist.exp_name,
saver=saver,
global_step=len(episode_rewards))
else:
U.save_state(
arglist.save_dir, saver=saver
) # print statement depends on whether or not there are adversaries
if num_adversaries == 0:
print(
"steps: {}, episodes: {}, mean episode reward: {}, time: {}"
.format(
train_step, len(episode_rewards),
np.mean(episode_rewards[-arglist.save_rate:-1]),
round(time.time() - t_start, 3)))
else:
print(
"steps: {}, episodes: {}, mean episode reward: {}, agent episode reward: {}, time: {}"
.format(
train_step, len(episode_rewards),
np.mean(episode_rewards[-arglist.save_rate:-1]), [
np.mean(rew[-arglist.save_rate:])
for rew in agent_rewards
], round(time.time() - t_start, 3)))
t_start = time.time()
# Keep track of final episode reward
final_ep_rewards.append(
np.mean(episode_rewards[-arglist.save_rate:-1]))
for rew in agent_rewards:
final_ep_ag_rewards.append(
np.mean(rew[-arglist.save_rate:-1]))
if arglist.real_q_log and (len(episode_rewards) %
(5 * arglist.save_rate) == 0):
q_mean, real_mean = calculate_real_q_value(
deepcopy(env),
trainers,
world_state_buffer=world_state_buffer,
action_n_buffer=action_n_buffer,
obs_n_buffer=obs_n_buffer,
start_episode_step_buffer=start_episode_step_buffer,
num_start_states=200,
args=arglist)
world_state_buffer, action_n_buffer, start_episode_step_buffer, obs_n_buffer = [], [], [], []
q_means.append(q_mean)
real_means.append(real_mean)
print('Q-mean: ' + str(q_mean) + ' Real mean: ' +
str(real_mean))
# saves final episode reward for plotting training curve later
if len(episode_rewards) > arglist.num_episodes:
rew_file_name = arglist.plots_dir + arglist.exp_name + '_rewards.pkl'
with open(rew_file_name, 'wb') as fp:
pickle.dump(final_ep_rewards, fp)
agrew_file_name = arglist.plots_dir + arglist.exp_name + '_agrewards.pkl'
with open(agrew_file_name, 'wb') as fp:
pickle.dump(final_ep_ag_rewards, fp)
args_file_name = arglist.plots_dir + arglist.exp_name + '_args.pkl'
with open(args_file_name, 'wb') as fp:
pickle.dump(arglist, fp)
if arglist.real_q_log:
real_q_path = arglist.plots_dir + arglist.exp_name + '_q_values.pkl'
with open(real_q_path, 'wb') as fp:
pickle.dump(
{
'q_means': q_means,
'real_means': real_means
}, fp)
print('...Finished total of {} episodes.'.format(
len(episode_rewards)))
break
def train(env,
eval_env,
agent,
render=False,
render_eval=False,
sanity_run=False,
nb_epochs=500,
nb_epoch_cycles=20,
nb_rollout_steps=100,
nb_train_steps=50,
param_noise_adaption_interval=50,
hist_files=None,
start_ckpt=None,
demo_files=None):
rank = MPI.COMM_WORLD.Get_rank()
mpi_size = MPI.COMM_WORLD.Get_size()
if rank == 0:
logdir = logger.get_dir()
else:
logdir = None
memory = agent.memory
batch_size = agent.batch_size
with tf_util.single_threaded_session() as sess:
# Prepare everything.
agent.initialize(sess, start_ckpt=start_ckpt)
sess.graph.finalize()
agent.reset()
dbg_tf_init(sess, agent.dbg_vars)
total_nb_train = 0
total_nb_rollout = 0
total_nb_eval = 0
# pre-train demo and critic_step
# train_params: (nb_steps, lr_scale)
total_nb_train = pretrain_demo(agent,
env,
demo_files,
total_nb_train,
train_params=[(100, 1.0)],
start_ckpt=start_ckpt)
load_history(agent, env, hist_files)
# main training
obs = env.reset()
reset = False
episode_step = 0
last_episode_step = 0
for i_epoch in range(nb_epochs):
t_epoch_start = time.time()
logger.info('\n%s epoch %d starts:' %
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
i_epoch))
for i_cycle in range(nb_epoch_cycles):
logger.info(
'\n%s cycles_%d of epoch_%d' %
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
i_cycle, i_epoch))
# rollout
rcd_obs, rcd_action, rcd_r, rcd_new_obs, rcd_done = [], [], [], [], []
if not sanity_run and mpi_size == 1 and last_episode_step != 0:
# todo: use mpi_max(last_episode_step)
# dynamically set nb_rollout_steps
nb_rollout_steps = max(last_episode_step * 4, batch_size)
logger.info(
'[%d, %d] rollout for %d steps.' %
(total_nb_rollout, memory.nb_entries, nb_rollout_steps))
t_rollout_start = time.time()
for i_rollout in range(nb_rollout_steps):
rollout_log = i_cycle == 0
# 50% param_noise, 40% action_noise
action, q = agent.pi(obs,
total_nb_rollout,
compute_Q=True,
rollout_log=rollout_log,
apply_param_noise=i_rollout % 10 < 5,
apply_action_noise=i_rollout % 10 > 5)
assert action.shape == env.action_space.shape
new_obs, r, done, reset, info = env.step(action)
if rank == 0 and render:
env.render()
episode_step += 1
total_nb_rollout += 1
if rollout_log:
summary_list = [('rollout/%s' % tp, info[tp])
for tp in ['rwd_walk', 'rwd_total']]
tp = 'rwd_agent'
summary_list += [
('rollout/%s_x%d' % (tp, info['rf_agent']),
info[tp] * info['rf_agent'])
]
summary_list += [('rollout/q', q)]
if r != 0:
summary_list += [('rollout/q_div_r', q / r)]
agent.add_list_summary(summary_list, total_nb_rollout)
# store at the end of cycle to speed up MPI rollout
# agent.store_transition(obs, action, r, new_obs, done)
rcd_obs.append(obs)
rcd_action.append(action)
rcd_r.append(r)
rcd_new_obs.append(new_obs)
rcd_done.append(done)
obs = new_obs
if reset:
# Episode done.
last_episode_step = episode_step
episode_step = 0
agent.reset()
obs = env.reset()
agent.store_multrans(memory, rcd_obs, rcd_action, rcd_r,
rcd_new_obs, rcd_done)
t_train_start = time.time()
steps_per_second = float(nb_rollout_steps) / (t_train_start -
t_rollout_start)
agent.add_list_summary(
[('rollout/steps_per_second', steps_per_second)],
total_nb_rollout)
# Train.
if not sanity_run:
# dynamically set nb_train_steps
if memory.nb_entries > batch_size * 20:
# using 1% of data for training every step?
nb_train_steps = max(
int(memory.nb_entries * 0.01 / batch_size), 1)
else:
nb_train_steps = 0
logger.info('[%d] training for %d steps.' %
(total_nb_train, nb_train_steps))
for _ in range(nb_train_steps):
# Adapt param noise, if necessary.
if memory.nb_entries >= batch_size and total_nb_train % param_noise_adaption_interval == 0:
agent.adapt_param_noise(total_nb_train)
agent.train_main(total_nb_train)
agent.update_target_net()
total_nb_train += 1
if i_epoch == 0 and i_cycle < 5:
rollout_duration = t_train_start - t_rollout_start
train_duration = time.time() - t_train_start
logger.info(
'rollout_time(%d) = %.3fs, train_time(%d) = %.3fs' %
(nb_rollout_steps, rollout_duration, nb_train_steps,
train_duration))
logger.info(
'rollout_speed=%.3fs/step, train_speed = %.3fs/step' %
(np.divide(rollout_duration, nb_rollout_steps),
np.divide(train_duration, nb_train_steps)))
logger.info('')
mpi_size = MPI.COMM_WORLD.Get_size()
# Log stats.
stats = agent.get_stats(memory)
combined_stats = stats.copy()
def as_scalar(x):
if isinstance(x, np.ndarray):
assert x.size == 1
return x[0]
elif np.isscalar(x):
return x
else:
raise ValueError('expected scalar, got %s' % x)
combined_stats_sums = MPI.COMM_WORLD.allreduce(
np.array([as_scalar(x) for x in combined_stats.values()]))
combined_stats = {
k: v / mpi_size
for (k, v) in zip(combined_stats.keys(), combined_stats_sums)
}
# exclude logging zobs_dbg_%d, zobs_dbg_%d_normalized
summary_list = [(key, combined_stats[key])
for key, v in combined_stats.items()
if 'dbg' not in key]
agent.add_list_summary(summary_list, i_epoch)
# only print out train stats for epoch_0 for sanity check
if i_epoch > 0:
combined_stats = {}
# Evaluation and statistics.
if eval_env is not None:
logger.info('[%d, %d] run evaluation' %
(i_epoch, total_nb_eval))
total_nb_eval = eval_episode(eval_env, render_eval, agent,
combined_stats, total_nb_eval)
logger.info('epoch %d duration: %.2f mins' %
(i_epoch, (time.time() - t_epoch_start) / 60))
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
logger.dump_tabular()
logger.info('')
if rank == 0:
agent.store_ckpt(os.path.join(logdir, '%s.ckpt' % 'ddpg'),
i_epoch)