def train(env,
nb_epochs,
nb_epoch_cycles,
render_eval,
reward_scale,
render,
param_noise,
actor,
critic,
additional_critic,
normalize_returns,
normalize_observations,
critic_l2_reg,
actor_lr,
critic_lr,
action_noise,
popart,
gamma,
clip_norm,
nb_train_steps,
nb_rollout_steps,
nb_eval_steps,
batch_size,
memory,
tau=0.05,
eval_env=None,
param_noise_adaption_interval=50,
nb_eval_episodes=20,
**kwargs):
rank = MPI.COMM_WORLD.Get_rank()
assert (np.abs(env.action_space.low) == env.action_space.high
).all() # we assume symmetric actions.
max_action = env.action_space.high
if "dologging" in kwargs:
dologging = kwargs["dologging"]
else:
dologging = True
if "tf_sum_logging" in kwargs:
tf_sum_logging = kwargs["tf_sum_logging"]
else:
tf_sum_logging = False
if "invert_grad" in kwargs:
invert_grad = kwargs["invert_grad"]
else:
invert_grad = False
if "actor_reg" in kwargs:
actor_reg = kwargs["actor_reg"]
else:
actor_reg = False
if dologging:
logger.info(
'scaling actions by {} before executing in env'.format(max_action))
agent = CDQ(actor,
critic,
additional_critic,
memory,
env.observation_space.shape,
env.action_space.shape,
gamma=gamma,
tau=tau,
normalize_returns=normalize_returns,
normalize_observations=normalize_observations,
batch_size=batch_size,
action_noise=action_noise,
param_noise=param_noise,
critic_l2_reg=critic_l2_reg,
actor_lr=actor_lr,
critic_lr=critic_lr,
enable_popart=popart,
clip_norm=clip_norm,
reward_scale=reward_scale,
inverting_grad=invert_grad,
actor_reg=actor_reg)
if dologging: logger.debug('Using agent with the following configuration:')
if dologging: logger.debug(str(agent.__dict__.items()))
# Set up logging stuff only for a single worker.
if rank != -1:
saver = tf.train.Saver(keep_checkpoint_every_n_hours=2,
max_to_keep=5,
save_relative_paths=True)
save_freq = kwargs["save_freq"]
else:
saver = None
# step = 0
global_t = 0
episode = 0
eval_episode_rewards_history = deque(maxlen=100)
episode_rewards_history = deque(maxlen=100)
with U.single_threaded_session() as sess:
# Set summary saver
if dologging and tf_sum_logging and rank == 0:
tf.summary.histogram("actor_grads", agent.actor_grads)
tf.summary.histogram("critic_grads", agent.critic_grads)
actor_trainable_vars = actor.trainable_vars
for var in actor_trainable_vars:
tf.summary.histogram(var.name, var)
critic_trainable_vars = critic.trainable_vars
for var in critic_trainable_vars:
tf.summary.histogram(var.name, var)
tf.summary.histogram("actions_out", agent.actor_tf)
tf.summary.histogram("critic_out", agent.critic_tf)
tf.summary.histogram("target_Q", agent.target_Q)
summary_var = tf.summary.merge_all()
writer_t = tf.summary.FileWriter(
osp.join(logger.get_dir(), 'train'), sess.graph)
else:
summary_var = tf.no_op()
# Prepare everything.
agent.initialize(sess)
sess.graph.finalize()
#set_trace()
## restore
if kwargs["restore_dir"] is not None:
restore_dir = osp.join(kwargs["restore_dir"], "model")
if (restore_dir is not None):
print('Restore path : ', restore_dir)
# checkpoint = tf.train.get_checkpoint_state(restore_dir)
# if checkpoint and checkpoint.model_checkpoint_path:
model_checkpoint_path = read_checkpoint_local(restore_dir)
if model_checkpoint_path:
saver.restore(U.get_session(), model_checkpoint_path)
print("checkpoint loaded:", model_checkpoint_path)
logger.info("checkpoint loaded:" +
str(model_checkpoint_path))
tokens = model_checkpoint_path.split("-")[-1]
# set global step
global_t = int(tokens)
print(">>> global step set:", global_t)
agent.reset()
obs = env.reset()
done = False
episode_reward = 0.
episode_step = 0
episodes = 0
t = 0
epoch = 0
start_time = time.time()
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_episode_eval_rewards = []
epoch_episode_eval_steps = []
epoch_start_time = time.time()
epoch_actions = []
epoch_qs = []
epoch_episodes = 0
## containers for hindsight
if kwargs["her"]:
# logger.info("-"*50 +'\nWill create HER\n' + "-"*50)
states, actions = [], []
print("Ready to go!")
for epoch in range(global_t, nb_epochs):
# stat containers
epoch_actor_losses = []
epoch_critic_losses = []
epoch_adaptive_distances = []
eval_episode_rewards = []
eval_qs = []
eval_episode_success = []
for cycle in range(nb_epoch_cycles):
# print("cycle:%d"%cycle)
# Perform rollouts.
for t_rollout in range(
int(nb_rollout_steps / MPI.COMM_WORLD.Get_size())):
# print(rank, t_rollout)
# Predict next action.
action, q = agent.pi(obs, apply_noise=True, compute_Q=True)
assert action.shape == env.action_space.shape
# Execute next action.
if rank == 0 and render:
env.render()
assert max_action.shape == action.shape
new_obs, r, done, info = env.step(
max_action * action
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
#if((t+1)%100) == 0:
# print(max_action*action, new_obs, r)
t += 1
if rank == 0 and render:
env.render()
sleep(0.1)
episode_reward += r
episode_step += 1
# Book-keeping.
epoch_actions.append(action)
epoch_qs.append(q)
agent.store_transition(obs, action, r, new_obs, done)
## storing info for hindsight
states.append(obs.copy())
actions.append(action.copy())
obs = new_obs
if done:
# Episode done.
epoch_episode_rewards.append(episode_reward)
episode_rewards_history.append(episode_reward)
epoch_episode_steps.append(episode_step)
episode_reward = 0.
episode_step = 0
epoch_episodes += 1
episodes += 1
if kwargs["her"]:
# logger.info("-"*50 +'\nCreating HER\n' + "-"*50)
## create hindsight experience replay
her_states, her_rewards = env.env.apply_hindsight(
states, actions, new_obs.copy())
## store her transitions: her_states: n+1, her_rewards: n
for her_i in range(len(her_states) - 2):
agent.store_transition(her_states[her_i],
actions[her_i],
her_rewards[her_i],
her_states[her_i + 1],
False)
#store last transition
agent.store_transition(her_states[-2], actions[-1],
her_rewards[-1],
her_states[-1], True)
## refresh the storage containers
del states, actions
states, actions = [], []
agent.reset()
obs = env.reset()
#print(obs)
# print(rank, "Training!")
# Train.
for t_train in range(nb_train_steps):
# print(rank, t_train)
# Adapt param noise, if necessary.
if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0:
distance = agent.adapt_param_noise()
epoch_adaptive_distances.append(distance)
cl, al, current_summary = agent.train(summary_var)
epoch_critic_losses.append(cl)
epoch_actor_losses.append(al)
agent.update_target_net()
if dologging and tf_sum_logging and rank == 0:
writer_t.add_summary(
current_summary,
epoch * nb_epoch_cycles * nb_train_steps +
cycle * nb_train_steps + t_train)
# print("Evaluating!")
# Evaluate.
if (eval_env is not None) and rank == 0:
for _ in range(nb_eval_episodes):
eval_episode_reward = 0.
eval_obs = eval_env.reset()
eval_obs_start = eval_obs.copy()
eval_done = False
while (not eval_done):
eval_action, eval_q = agent.pi(eval_obs,
apply_noise=False,
compute_Q=True)
eval_obs, eval_r, eval_done, eval_info = eval_env.step(
max_action * eval_action
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
if render_eval:
sleep(0.1)
print("Render!")
eval_env.render()
print("rendered!")
eval_episode_reward += eval_r
eval_qs.append(eval_q)
eval_episode_rewards.append(eval_episode_reward)
eval_episode_rewards_history.append(eval_episode_reward)
eval_episode_success.append(
eval_info["done"] == "goal reached")
if (eval_info["done"] == "goal reached"):
logger.info(
"success, training epoch:%d,starting config:" %
epoch, eval_obs_start, 'final state', eval_obs)
if dologging and rank == 0:
print("Logging!")
# Log stats.
epoch_train_duration = time.time() - epoch_start_time
duration = time.time() - start_time
stats = agent.get_stats()
combined_stats = {}
for key in sorted(stats.keys()):
combined_stats[key] = normal_mean(stats[key])
# Rollout statistics.
combined_stats['rollout/return'] = normal_mean(
epoch_episode_rewards)
if len(episode_rewards_history) > 0:
combined_stats['rollout/return_history'] = normal_mean(
np.mean(episode_rewards_history))
else:
combined_stats['rollout/return_history'] = 0.
combined_stats['rollout/episode_steps'] = normal_mean(
epoch_episode_steps)
combined_stats['rollout/episodes'] = np.sum(epoch_episodes)
combined_stats['rollout/actions_mean'] = normal_mean(
epoch_actions)
combined_stats['rollout/actions_std'] = normal_std(
epoch_actions)
combined_stats['rollout/Q_mean'] = normal_mean(epoch_qs)
# Train statistics.
combined_stats['train/loss_actor'] = normal_mean(
epoch_actor_losses)
combined_stats['train/loss_critic'] = normal_mean(
epoch_critic_losses)
combined_stats['train/param_noise_distance'] = normal_mean(
epoch_adaptive_distances)
# Evaluation statistics.
if eval_env is not None:
combined_stats['eval/return'] = normal_mean(
eval_episode_rewards)
combined_stats['eval/success'] = normal_mean(
eval_episode_success)
if len(eval_episode_rewards_history) > 0:
combined_stats['eval/return_history'] = normal_mean(
np.mean(eval_episode_rewards_history))
else:
combined_stats['eval/return_history'] = 0.
combined_stats['eval/Q'] = normal_mean(eval_qs)
combined_stats['eval/episodes'] = normal_mean(
len(eval_episode_rewards))
# Total statistics.
combined_stats['total/duration'] = normal_mean(duration)
combined_stats['total/steps_per_second'] = normal_mean(
float(t) / float(duration))
combined_stats['total/episodes'] = normal_mean(episodes)
combined_stats['total/epochs'] = epoch + 1
combined_stats['total/steps'] = t
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
logger.dump_tabular()
logger.info('')
logdir = logger.get_dir()
if rank == 0 and logdir:
print("Dumping progress!")
if hasattr(env, 'get_state'):
with open(os.path.join(logdir, 'env_state.pkl'),
'wb') as f:
pickle.dump(env.get_state(), f)
if eval_env and hasattr(eval_env, 'get_state'):
with open(os.path.join(logdir, 'eval_env_state.pkl'),
'wb') as f:
pickle.dump(eval_env.get_state(), f)
## save tf model
if rank == 0 and (epoch + 1) % save_freq == 0:
print("Saving the model!")
os.makedirs(osp.join(logdir, "model"), exist_ok=True)
saver.save(U.get_session(),
logdir + "/model/cdq",
global_step=epoch)
def train(env,
nb_epochs,
nb_epoch_cycles,
render_eval,
reward_scale,
render,
param_noise,
actor,
critic,
normalize_returns,
normalize_observations,
critic_l2_reg,
actor_lr,
critic_lr,
action_noise,
popart,
gamma,
clip_norm,
nb_train_steps,
nb_rollout_steps,
nb_eval_episodes,
batch_size,
memory,
tau=0.05,
eval_env=None,
param_noise_adaption_interval=50,
**kwargs):
rank = MPI.COMM_WORLD.Get_rank()
assert (np.abs(env.action_space.low) == env.action_space.high
).all() # we assume symmetric actions.
max_action = env.action_space.high
if "dologging" in kwargs:
dologging = kwargs["dologging"]
else:
dologging = True
if "tf_sum_logging" in kwargs:
tf_sum_logging = kwargs["tf_sum_logging"]
else:
tf_sum_logging = False
if "invert_grad" in kwargs:
invert_grad = kwargs["invert_grad"]
else:
invert_grad = False
if "actor_reg" in kwargs:
actor_reg = kwargs["actor_reg"]
else:
actor_reg = False
if dologging:
logger.debug(
'scaling actions by {} before executing in env'.format(max_action))
if kwargs['look_ahead']:
look_ahead = True
look_ahead_planner = Planning_with_memories(
skillset=kwargs['my_skill_set'],
env=env,
num_samples=kwargs['num_samples'])
exploration = LinearSchedule(schedule_timesteps=int(nb_epochs *
nb_epoch_cycles),
initial_p=1.0,
final_p=kwargs['exploration_final_eps'])
else:
look_ahead = False
if kwargs['skillset']:
action_shape = (kwargs['my_skill_set'].len +
kwargs['my_skill_set'].num_params, )
else:
action_shape = env.action_space.shape
agent = DDPG(actor,
critic,
memory,
env.observation_space.shape,
action_shape,
gamma=gamma,
tau=tau,
normalize_returns=normalize_returns,
normalize_observations=normalize_observations,
batch_size=batch_size,
action_noise=action_noise,
param_noise=param_noise,
critic_l2_reg=critic_l2_reg,
actor_lr=actor_lr,
critic_lr=critic_lr,
enable_popart=popart,
clip_norm=clip_norm,
reward_scale=reward_scale,
inverting_grad=invert_grad,
actor_reg=actor_reg)
if dologging and MPI.COMM_WORLD.Get_rank() == 0:
logger.debug('Using agent with the following configuration:')
logger.debug(str(agent.__dict__.items()))
# should have saver for all thread to restore. But dump only using 1 saver
saver = tf.train.Saver(keep_checkpoint_every_n_hours=2,
max_to_keep=20,
save_relative_paths=True)
save_freq = kwargs["save_freq"]
# step = 0
global_t = 0
eval_episode_rewards_history = deque(maxlen=100)
episode_rewards_history = deque(maxlen=100)
## get the session with the current graph => identical graph is used for each session
with U.single_threaded_session() as sess:
# Set summary saver
if dologging and tf_sum_logging and rank == 0:
tf.summary.histogram("actor_grads", agent.actor_grads)
tf.summary.histogram("critic_grads", agent.critic_grads)
actor_trainable_vars = actor.trainable_vars
for var in actor_trainable_vars:
tf.summary.histogram(var.name, var)
critic_trainable_vars = critic.trainable_vars
for var in critic_trainable_vars:
tf.summary.histogram(var.name, var)
tf.summary.histogram("actions_out", agent.actor_tf)
tf.summary.histogram("critic_out", agent.critic_tf)
tf.summary.histogram("target_Q", agent.target_Q)
summary_var = tf.summary.merge_all()
writer_t = tf.summary.FileWriter(
osp.join(logger.get_dir(), 'train'), sess.graph)
else:
summary_var = tf.no_op()
# Prepare everything.
agent.initialize(sess)
sess.graph.finalize()
## restore
if kwargs['skillset']:
## restore skills
my_skill_set = kwargs['my_skill_set']
my_skill_set.restore_skillset(sess=sess)
## restore current controller
if kwargs["restore_dir"] is not None:
restore_dir = osp.join(kwargs["restore_dir"], "model")
if (restore_dir is not None) and rank == 0:
print('Restore path : ', restore_dir)
model_checkpoint_path = read_checkpoint_local(restore_dir)
if model_checkpoint_path:
saver.restore(U.get_session(), model_checkpoint_path)
logger.info("checkpoint loaded:" +
str(model_checkpoint_path))
tokens = model_checkpoint_path.split("-")[-1]
# set global step
global_t = int(tokens)
print(">>> global step set:", global_t)
agent.reset()
obs = env.reset()
# maintained across epochs
episodes = 0
t = 0
start_time = time.time()
# creating vars. this is done to keep the syntax for deleting the list simple a[:] = []
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_actions = []
epoch_actor_losses = []
epoch_critic_losses = []
if param_noise is not None:
epoch_adaptive_distances = []
eval_episode_rewards = []
eval_episode_success = []
# for each episode
done = False
episode_reward = 0.
episode_step = 0
## containers for hierarchical hindsight
if kwargs["her"]:
logger.debug("-" * 50 + '\nWill create HER\n' + "-" * 50)
# per episode
states, pactions, sub_states = [], [], []
print("Ready to go!")
for epoch in range(global_t, nb_epochs):
# stat containers
epoch_episodes = 0.
epoch_start_time = time.time()
epoch_episode_rewards[:] = []
epoch_episode_steps[:] = []
epoch_actions[:] = [
] # action mean: don't know if this indicates anything
epoch_actor_losses[:] = []
epoch_critic_losses[:] = []
if param_noise is not None:
epoch_adaptive_distances[:] = []
eval_episode_rewards[:] = []
eval_episode_success[:] = []
for cycle in range(nb_epoch_cycles):
# Perform rollouts.
for t_rollout in range(
int(nb_rollout_steps / MPI.COMM_WORLD.Get_size())):
# print(rank, t_rollout)
# Predict next action.
# exploration check
if kwargs['look_ahead'] and (np.random.rand(
) < exploration.value(epoch * nb_epoch_cycles + cycle)):
paction, planner_info = look_ahead_planner.create_plan(
obs)
else:
paction, _ = agent.pi(obs,
apply_noise=True,
compute_Q=True)
if (my_skill_set):
## break actions into primitives and their params
primitives_prob = paction[:kwargs['my_skill_set'].len]
primitive_id = np.argmax(primitives_prob)
# print("skill chosen", primitive_id)
r = 0.
skill_obs = obs.copy()
if kwargs['her']:
curr_sub_states = [skill_obs.copy()]
for _ in range(kwargs['commit_for']):
action = my_skill_set.pi(
primitive_id=primitive_id,
obs=skill_obs.copy(),
primitive_params=paction[my_skill_set.len:])
# Execute next action.
if rank == 0 and render:
sleep(0.1)
env.render()
assert max_action.shape == action.shape
new_obs, skill_r, done, info = env.step(
max_action * action
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
r += skill_r
if kwargs['her']:
curr_sub_states.append(new_obs.copy())
skill_obs = new_obs
if done or my_skill_set.termination(
new_obs,
primitive_id,
primitive_params=paction[my_skill_set.
len:]):
break
# assuming the skill is trained from different reward signal
r = skill_r
else:
action = paction
# Execute next action.
if rank == 0 and render:
env.render()
assert max_action.shape == action.shape
new_obs, r, done, info = env.step(
max_action * action
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
assert action.shape == env.action_space.shape
t += 1
episode_reward += r
episode_step += 1
# Book-keeping.
epoch_actions.append(paction)
agent.store_transition(obs, paction, r, new_obs, done)
# storing info for hindsight
if kwargs['her']:
states.append(obs.copy())
pactions.append(paction.copy())
sub_states.append(curr_sub_states)
# print(planner_info['next_state'][:6], new_obs[:6])
obs = new_obs
if done:
# Episode done.
# update stats
epoch_episode_rewards.append(episode_reward)
episode_rewards_history.append(episode_reward)
epoch_episode_steps.append(episode_step)
epoch_episodes += 1
episodes += 1
# reinit
episode_reward = 0.
episode_step = 0
agent.reset()
obs = env.reset()
if kwargs["her"]:
# logger.info("-"*50 +'\nCreating HER\n' + "-"*50)
# create hindsight experience replay
if kwargs['skillset']:
her_states, her_rewards = env.apply_hierarchical_hindsight(
states, pactions, new_obs.copy(),
sub_states)
else:
her_states, her_rewards = env.apply_hindsight(
states, pactions, new_obs.copy())
## store her transitions: her_states: n+1, her_rewards: n
for her_i in range(len(her_states) - 2):
agent.store_transition(her_states[her_i],
pactions[her_i],
her_rewards[her_i],
her_states[her_i + 1],
False)
#store last transition
agent.store_transition(her_states[-2],
pactions[-1],
her_rewards[-1],
her_states[-1], True)
## refresh the storage containers
states[:], pactions[:] = [], []
if kwargs['skillset']:
sub_states[:] = []
# print(rank, "Training!")
# Train.
for t_train in range(nb_train_steps):
# print(rank, t_train)
# Adapt param noise, if necessary.
if (memory.nb_entries >= batch_size) and (
t % param_noise_adaption_interval
== 0) and (param_noise is not None):
distance = agent.adapt_param_noise()
epoch_adaptive_distances.append(distance)
cl, al, current_summary = agent.train(summary_var)
epoch_critic_losses.append(cl)
epoch_actor_losses.append(al)
agent.update_target_net()
if dologging and tf_sum_logging and rank == 0:
writer_t.add_summary(
current_summary,
epoch * nb_epoch_cycles * nb_train_steps +
cycle * nb_train_steps + t_train)
# print("Evaluating!")
# Evaluate after training is done.
if (eval_env is not None) and rank == 0:
for _ in range(nb_eval_episodes):
eval_episode_reward = 0.
eval_obs = eval_env.reset()
eval_obs_start = eval_obs.copy()
eval_done = False
while (not eval_done):
eval_paction, _ = agent.pi(eval_obs,
apply_noise=False,
compute_Q=False)
if (kwargs['skillset']):
## break actions into primitives and their params
eval_primitives_prob = eval_paction[:kwargs[
'my_skill_set'].len]
eval_primitive_id = np.argmax(eval_primitives_prob)
eval_r = 0.
eval_skill_obs = eval_obs.copy()
for _ in range(kwargs['commit_for']):
eval_action = my_skill_set.pi(
primitive_id=eval_primitive_id,
obs=eval_skill_obs.copy(),
primitive_params=eval_paction[my_skill_set.
len:])
eval_new_obs, eval_skill_r, eval_done, eval_info = eval_env.step(
max_action * eval_action
) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
if render_eval:
eval_env.render()
eval_r += eval_skill_r
# check for skill termination or episode termination
eval_terminate_skill = my_skill_set.termination(
eval_new_obs,
eval_primitive_id,
primitive_params=eval_paction[my_skill_set.
len:])
if eval_done or eval_terminate_skill:
break
eval_skill_obs = eval_new_obs
# hack assuming the skills are trained from diff reward signal
eval_r = eval_skill_r
else:
eval_action, _ = eval_paction, eval_pq
eval_new_obs, eval_r, eval_done, eval_info = eval_env.step(
max_action * eval_action)
eval_episode_reward += eval_r
eval_obs = eval_new_obs
eval_episode_rewards.append(eval_episode_reward)
eval_episode_rewards_history.append(eval_episode_reward)
eval_episode_success.append(
eval_info["done"] == "goal reached")
if (eval_info["done"] == "goal reached"):
logger.info(
"success, training epoch:%d,starting config:" %
epoch, eval_obs_start, 'final state', eval_obs)
if dologging and rank == 0:
print("Logging!")
# Log stats.
epoch_train_duration = time.time() - epoch_start_time
duration = time.time() - start_time
stats = agent.get_stats()
combined_stats = {}
for key in sorted(stats.keys()):
combined_stats[key] = normal_mean(stats[key])
# Rollout statistics.
combined_stats['rollout/return'] = normal_mean(
epoch_episode_rewards)
if len(episode_rewards_history) > 0:
combined_stats['rollout/return_history'] = normal_mean(
np.mean(episode_rewards_history))
else:
combined_stats['rollout/return_history'] = 0.
combined_stats['rollout/episode_steps'] = normal_mean(
epoch_episode_steps)
combined_stats['rollout/episodes'] = np.sum(epoch_episodes)
combined_stats['rollout/actions_mean'] = normal_mean(
epoch_actions)
combined_stats['rollout/actions_std'] = normal_std(
epoch_actions)
# Train statistics.
combined_stats['train/loss_actor'] = normal_mean(
epoch_actor_losses)
combined_stats['train/loss_critic'] = normal_mean(
epoch_critic_losses)
if param_noise is not None:
combined_stats['train/param_noise_distance'] = normal_mean(
epoch_adaptive_distances)
if kwargs['look_ahead']:
combined_stats['train/exploration'] = exploration.value(
epoch * nb_epoch_cycles + cycle)
# Evaluation statistics.
if eval_env is not None:
combined_stats['eval/return'] = normal_mean(
eval_episode_rewards)
combined_stats['eval/success'] = normal_mean(
eval_episode_success)
if len(eval_episode_rewards_history) > 0:
combined_stats['eval/return_history'] = normal_mean(
np.mean(eval_episode_rewards_history))
else:
combined_stats['eval/return_history'] = 0.
combined_stats['eval/episodes'] = normal_mean(
len(eval_episode_rewards))
# Total statistics.
combined_stats['total/duration'] = normal_mean(duration)
combined_stats['total/rollout_per_second'] = normal_mean(
float(t) / float(duration))
combined_stats['total/episodes'] = normal_mean(episodes)
combined_stats['total/epochs'] = epoch + 1
combined_stats['total/steps'] = t
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
logger.dump_tabular()
logger.info('')
logdir = logger.get_dir()
# if rank == 0 and logdir:
# print("Dumping progress!")
# if hasattr(env, 'get_state'):
# with open(osp.join(logdir, 'env_state.pkl'), 'wb') as f:
# pickle.dump(env.get_state(), f)
# if eval_env and hasattr(eval_env, 'get_state'):
# with open(osp.join(logdir, 'eval_env_state.pkl'), 'wb') as f:
# pickle.dump(eval_env.get_state(), f)
## save tf model
if rank == 0 and (epoch + 1) % save_freq == 0:
print("Saving the model!")
os.makedirs(osp.join(logdir, "model"), exist_ok=True)
saver.save(U.get_session(),
logdir + "/model/ddpg",
global_step=epoch)
def train(env,
eval_env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
my_skill_set=None,
log_dir = None,
num_eval_episodes=10,
render=False,
render_eval = False,
commit_for = 1
):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
if my_skill_set: assert commit_for>=1, "commit_for >= 1"
save_idx = 0
with U.single_threaded_session() as sess:
## restore
if my_skill_set:
action_shape = my_skill_set.len
else:
action_shape = env.action_space.n
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space_shape = env.observation_space.shape
def make_obs_ph(name):
return U.BatchInput(observation_space_shape, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=action_shape,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': action_shape,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
# sess.run(tf.variables_initializer(new_variables))
# sess.run(tf.global_variables_initializer())
update_target()
if my_skill_set:
## restore skills
my_skill_set.restore_skillset(sess=sess)
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
model_saved = False
model_file = os.path.join(log_dir, "model", "deepq")
# save the initial act model
print("Saving the starting model")
os.makedirs(os.path.dirname(model_file), exist_ok=True)
act.save(model_file + '.pkl')
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
paction = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
if(my_skill_set):
skill_obs = obs.copy()
primitive_id = paction
rew = 0.
for _ in range(commit_for):
## break actions into primitives and their params
action = my_skill_set.pi(primitive_id=primitive_id, obs = skill_obs.copy(), primitive_params=None)
new_obs, skill_rew, done, _ = env.step(action)
if render:
# print(action)
env.render()
sleep(0.1)
rew += skill_rew
skill_obs = new_obs
terminate_skill = my_skill_set.termination(new_obs)
if done or terminate_skill:
break
else:
action= paction
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
if render:
env.render()
sleep(0.1)
# Store transition in the replay buffer for the outer env
replay_buffer.add(obs, paction, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
print("Time:%d, episodes:%d"%(t,len(episode_rewards)))
# add hindsight experience
if t > learning_starts and t % train_freq == 0:
# print('Training!')
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
# print(len(episode_rewards), episode_rewards[-11:-1])
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 50 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
U.save_state(model_file)
act.save(model_file + '%d.pkl'%save_idx)
save_idx += 1
model_saved = True
saved_mean_reward = mean_100ep_reward
# else:
# print(saved_mean_reward, mean_100ep_reward)
if (eval_env is not None) and t > learning_starts and t % target_network_update_freq == 0:
# dumping other stats
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("%d time spent exploring", int(100 * exploration.value(t)))
print("Testing!")
eval_episode_rewards = []
eval_episode_successes = []
for i in range(num_eval_episodes):
eval_episode_reward = 0.
eval_obs = eval_env.reset()
eval_obs_start = eval_obs.copy()
eval_done = False
while(not eval_done):
eval_paction = act(np.array(eval_obs)[None])[0]
if(my_skill_set):
eval_skill_obs = eval_obs.copy()
eval_primitive_id = eval_paction
eval_r = 0.
for _ in range(commit_for):
## break actions into primitives and their params
eval_action, _ = my_skill_set.pi(primitive_id=eval_primitive_id, obs = eval_skill_obs.copy(), primitive_params=None)
eval_new_obs, eval_skill_rew, eval_done, eval_info = eval_env.step(eval_action)
# print('env reward:%f'%eval_skill_rew)
if render_eval:
print("Render!")
eval_env.render()
print("rendered!")
eval_r += eval_skill_rew
eval_skill_obs = eval_new_obs
eval_terminate_skill = my_skill_set.termination(eval_new_obs)
if eval_done or eval_terminate_skill:
break
else:
eval_action= eval_paction
env_action = eval_action
reset = False
eval_new_obs, eval_r, eval_done, eval_info = eval_env.step(env_action)
if render_eval:
# print("Render!")
eval_env.render()
# print("rendered!")
eval_episode_reward += eval_r
# print("eval_r:%f, eval_episode_reward:%f"%(eval_r, eval_episode_reward))
eval_obs = eval_new_obs
eval_episode_success = (eval_info["done"]=="goal reached")
if(eval_episode_success):
logger.info("success, training epoch:%d,starting config:"%t)
eval_episode_rewards.append(eval_episode_reward)
eval_episode_successes.append(eval_episode_success)
combined_stats = {}
# print(eval_episode_successes, np.mean(eval_episode_successes))
combined_stats['eval/return'] = normal_mean(eval_episode_rewards)
combined_stats['eval/success'] = normal_mean(eval_episode_successes)
combined_stats['eval/episodes'] = (len(eval_episode_rewards))
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
print("dumping the stats!")
logger.dump_tabular()
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
U.load_state(model_file)
