def train(env, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic, additional_critic, normalize_returns, normalize_observations, critic_l2_reg, actor_lr, critic_lr, action_noise, popart, gamma, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_steps, batch_size, memory, tau=0.05, eval_env=None, param_noise_adaption_interval=50, nb_eval_episodes=20, **kwargs): rank = MPI.COMM_WORLD.Get_rank() assert (np.abs(env.action_space.low) == env.action_space.high ).all() # we assume symmetric actions. max_action = env.action_space.high if "dologging" in kwargs: dologging = kwargs["dologging"] else: dologging = True if "tf_sum_logging" in kwargs: tf_sum_logging = kwargs["tf_sum_logging"] else: tf_sum_logging = False if "invert_grad" in kwargs: invert_grad = kwargs["invert_grad"] else: invert_grad = False if "actor_reg" in kwargs: actor_reg = kwargs["actor_reg"] else: actor_reg = False if dologging: logger.info( 'scaling actions by {} before executing in env'.format(max_action)) agent = CDQ(actor, critic, additional_critic, memory, env.observation_space.shape, env.action_space.shape, gamma=gamma, tau=tau, normalize_returns=normalize_returns, normalize_observations=normalize_observations, batch_size=batch_size, action_noise=action_noise, param_noise=param_noise, critic_l2_reg=critic_l2_reg, actor_lr=actor_lr, critic_lr=critic_lr, enable_popart=popart, clip_norm=clip_norm, reward_scale=reward_scale, inverting_grad=invert_grad, actor_reg=actor_reg) if dologging: logger.debug('Using agent with the following configuration:') if dologging: logger.debug(str(agent.__dict__.items())) # Set up logging stuff only for a single worker. if rank != -1: saver = tf.train.Saver(keep_checkpoint_every_n_hours=2, max_to_keep=5, save_relative_paths=True) save_freq = kwargs["save_freq"] else: saver = None # step = 0 global_t = 0 episode = 0 eval_episode_rewards_history = deque(maxlen=100) episode_rewards_history = deque(maxlen=100) with U.single_threaded_session() as sess: # Set summary saver if dologging and tf_sum_logging and rank == 0: tf.summary.histogram("actor_grads", agent.actor_grads) tf.summary.histogram("critic_grads", agent.critic_grads) actor_trainable_vars = actor.trainable_vars for var in actor_trainable_vars: tf.summary.histogram(var.name, var) critic_trainable_vars = critic.trainable_vars for var in critic_trainable_vars: tf.summary.histogram(var.name, var) tf.summary.histogram("actions_out", agent.actor_tf) tf.summary.histogram("critic_out", agent.critic_tf) tf.summary.histogram("target_Q", agent.target_Q) summary_var = tf.summary.merge_all() writer_t = tf.summary.FileWriter( osp.join(logger.get_dir(), 'train'), sess.graph) else: summary_var = tf.no_op() # Prepare everything. agent.initialize(sess) sess.graph.finalize() #set_trace() ## restore if kwargs["restore_dir"] is not None: restore_dir = osp.join(kwargs["restore_dir"], "model") if (restore_dir is not None): print('Restore path : ', restore_dir) # checkpoint = tf.train.get_checkpoint_state(restore_dir) # if checkpoint and checkpoint.model_checkpoint_path: model_checkpoint_path = read_checkpoint_local(restore_dir) if model_checkpoint_path: saver.restore(U.get_session(), model_checkpoint_path) print("checkpoint loaded:", model_checkpoint_path) logger.info("checkpoint loaded:" + str(model_checkpoint_path)) tokens = model_checkpoint_path.split("-")[-1] # set global step global_t = int(tokens) print(">>> global step set:", global_t) agent.reset() obs = env.reset() done = False episode_reward = 0. episode_step = 0 episodes = 0 t = 0 epoch = 0 start_time = time.time() epoch_episode_rewards = [] epoch_episode_steps = [] epoch_episode_eval_rewards = [] epoch_episode_eval_steps = [] epoch_start_time = time.time() epoch_actions = [] epoch_qs = [] epoch_episodes = 0 ## containers for hindsight if kwargs["her"]: # logger.info("-"*50 +'\nWill create HER\n' + "-"*50) states, actions = [], [] print("Ready to go!") for epoch in range(global_t, nb_epochs): # stat containers epoch_actor_losses = [] epoch_critic_losses = [] epoch_adaptive_distances = [] eval_episode_rewards = [] eval_qs = [] eval_episode_success = [] for cycle in range(nb_epoch_cycles): # print("cycle:%d"%cycle) # Perform rollouts. for t_rollout in range( int(nb_rollout_steps / MPI.COMM_WORLD.Get_size())): # print(rank, t_rollout) # Predict next action. action, q = agent.pi(obs, apply_noise=True, compute_Q=True) assert action.shape == env.action_space.shape # Execute next action. if rank == 0 and render: env.render() assert max_action.shape == action.shape new_obs, r, done, info = env.step( max_action * action ) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1]) #if((t+1)%100) == 0: # print(max_action*action, new_obs, r) t += 1 if rank == 0 and render: env.render() sleep(0.1) episode_reward += r episode_step += 1 # Book-keeping. epoch_actions.append(action) epoch_qs.append(q) agent.store_transition(obs, action, r, new_obs, done) ## storing info for hindsight states.append(obs.copy()) actions.append(action.copy()) obs = new_obs if done: # Episode done. epoch_episode_rewards.append(episode_reward) episode_rewards_history.append(episode_reward) epoch_episode_steps.append(episode_step) episode_reward = 0. episode_step = 0 epoch_episodes += 1 episodes += 1 if kwargs["her"]: # logger.info("-"*50 +'\nCreating HER\n' + "-"*50) ## create hindsight experience replay her_states, her_rewards = env.env.apply_hindsight( states, actions, new_obs.copy()) ## store her transitions: her_states: n+1, her_rewards: n for her_i in range(len(her_states) - 2): agent.store_transition(her_states[her_i], actions[her_i], her_rewards[her_i], her_states[her_i + 1], False) #store last transition agent.store_transition(her_states[-2], actions[-1], her_rewards[-1], her_states[-1], True) ## refresh the storage containers del states, actions states, actions = [], [] agent.reset() obs = env.reset() #print(obs) # print(rank, "Training!") # Train. for t_train in range(nb_train_steps): # print(rank, t_train) # Adapt param noise, if necessary. if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0: distance = agent.adapt_param_noise() epoch_adaptive_distances.append(distance) cl, al, current_summary = agent.train(summary_var) epoch_critic_losses.append(cl) epoch_actor_losses.append(al) agent.update_target_net() if dologging and tf_sum_logging and rank == 0: writer_t.add_summary( current_summary, epoch * nb_epoch_cycles * nb_train_steps + cycle * nb_train_steps + t_train) # print("Evaluating!") # Evaluate. if (eval_env is not None) and rank == 0: for _ in range(nb_eval_episodes): eval_episode_reward = 0. eval_obs = eval_env.reset() eval_obs_start = eval_obs.copy() eval_done = False while (not eval_done): eval_action, eval_q = agent.pi(eval_obs, apply_noise=False, compute_Q=True) eval_obs, eval_r, eval_done, eval_info = eval_env.step( max_action * eval_action ) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1]) if render_eval: sleep(0.1) print("Render!") eval_env.render() print("rendered!") eval_episode_reward += eval_r eval_qs.append(eval_q) eval_episode_rewards.append(eval_episode_reward) eval_episode_rewards_history.append(eval_episode_reward) eval_episode_success.append( eval_info["done"] == "goal reached") if (eval_info["done"] == "goal reached"): logger.info( "success, training epoch:%d,starting config:" % epoch, eval_obs_start, 'final state', eval_obs) if dologging and rank == 0: print("Logging!") # Log stats. epoch_train_duration = time.time() - epoch_start_time duration = time.time() - start_time stats = agent.get_stats() combined_stats = {} for key in sorted(stats.keys()): combined_stats[key] = normal_mean(stats[key]) # Rollout statistics. combined_stats['rollout/return'] = normal_mean( epoch_episode_rewards) if len(episode_rewards_history) > 0: combined_stats['rollout/return_history'] = normal_mean( np.mean(episode_rewards_history)) else: combined_stats['rollout/return_history'] = 0. combined_stats['rollout/episode_steps'] = normal_mean( epoch_episode_steps) combined_stats['rollout/episodes'] = np.sum(epoch_episodes) combined_stats['rollout/actions_mean'] = normal_mean( epoch_actions) combined_stats['rollout/actions_std'] = normal_std( epoch_actions) combined_stats['rollout/Q_mean'] = normal_mean(epoch_qs) # Train statistics. combined_stats['train/loss_actor'] = normal_mean( epoch_actor_losses) combined_stats['train/loss_critic'] = normal_mean( epoch_critic_losses) combined_stats['train/param_noise_distance'] = normal_mean( epoch_adaptive_distances) # Evaluation statistics. if eval_env is not None: combined_stats['eval/return'] = normal_mean( eval_episode_rewards) combined_stats['eval/success'] = normal_mean( eval_episode_success) if len(eval_episode_rewards_history) > 0: combined_stats['eval/return_history'] = normal_mean( np.mean(eval_episode_rewards_history)) else: combined_stats['eval/return_history'] = 0. combined_stats['eval/Q'] = normal_mean(eval_qs) combined_stats['eval/episodes'] = normal_mean( len(eval_episode_rewards)) # Total statistics. combined_stats['total/duration'] = normal_mean(duration) combined_stats['total/steps_per_second'] = normal_mean( float(t) / float(duration)) combined_stats['total/episodes'] = normal_mean(episodes) combined_stats['total/epochs'] = epoch + 1 combined_stats['total/steps'] = t for key in sorted(combined_stats.keys()): logger.record_tabular(key, combined_stats[key]) logger.dump_tabular() logger.info('') logdir = logger.get_dir() if rank == 0 and logdir: print("Dumping progress!") if hasattr(env, 'get_state'): with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as f: pickle.dump(env.get_state(), f) if eval_env and hasattr(eval_env, 'get_state'): with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as f: pickle.dump(eval_env.get_state(), f) ## save tf model if rank == 0 and (epoch + 1) % save_freq == 0: print("Saving the model!") os.makedirs(osp.join(logdir, "model"), exist_ok=True) saver.save(U.get_session(), logdir + "/model/cdq", global_step=epoch)
def train(env, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic, normalize_returns, normalize_observations, critic_l2_reg, actor_lr, critic_lr, action_noise, popart, gamma, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_episodes, batch_size, memory, tau=0.05, eval_env=None, param_noise_adaption_interval=50, **kwargs): rank = MPI.COMM_WORLD.Get_rank() assert (np.abs(env.action_space.low) == env.action_space.high ).all() # we assume symmetric actions. max_action = env.action_space.high if "dologging" in kwargs: dologging = kwargs["dologging"] else: dologging = True if "tf_sum_logging" in kwargs: tf_sum_logging = kwargs["tf_sum_logging"] else: tf_sum_logging = False if "invert_grad" in kwargs: invert_grad = kwargs["invert_grad"] else: invert_grad = False if "actor_reg" in kwargs: actor_reg = kwargs["actor_reg"] else: actor_reg = False if dologging: logger.debug( 'scaling actions by {} before executing in env'.format(max_action)) if kwargs['look_ahead']: look_ahead = True look_ahead_planner = Planning_with_memories( skillset=kwargs['my_skill_set'], env=env, num_samples=kwargs['num_samples']) exploration = LinearSchedule(schedule_timesteps=int(nb_epochs * nb_epoch_cycles), initial_p=1.0, final_p=kwargs['exploration_final_eps']) else: look_ahead = False if kwargs['skillset']: action_shape = (kwargs['my_skill_set'].len + kwargs['my_skill_set'].num_params, ) else: action_shape = env.action_space.shape agent = DDPG(actor, critic, memory, env.observation_space.shape, action_shape, gamma=gamma, tau=tau, normalize_returns=normalize_returns, normalize_observations=normalize_observations, batch_size=batch_size, action_noise=action_noise, param_noise=param_noise, critic_l2_reg=critic_l2_reg, actor_lr=actor_lr, critic_lr=critic_lr, enable_popart=popart, clip_norm=clip_norm, reward_scale=reward_scale, inverting_grad=invert_grad, actor_reg=actor_reg) if dologging and MPI.COMM_WORLD.Get_rank() == 0: logger.debug('Using agent with the following configuration:') logger.debug(str(agent.__dict__.items())) # should have saver for all thread to restore. But dump only using 1 saver saver = tf.train.Saver(keep_checkpoint_every_n_hours=2, max_to_keep=20, save_relative_paths=True) save_freq = kwargs["save_freq"] # step = 0 global_t = 0 eval_episode_rewards_history = deque(maxlen=100) episode_rewards_history = deque(maxlen=100) ## get the session with the current graph => identical graph is used for each session with U.single_threaded_session() as sess: # Set summary saver if dologging and tf_sum_logging and rank == 0: tf.summary.histogram("actor_grads", agent.actor_grads) tf.summary.histogram("critic_grads", agent.critic_grads) actor_trainable_vars = actor.trainable_vars for var in actor_trainable_vars: tf.summary.histogram(var.name, var) critic_trainable_vars = critic.trainable_vars for var in critic_trainable_vars: tf.summary.histogram(var.name, var) tf.summary.histogram("actions_out", agent.actor_tf) tf.summary.histogram("critic_out", agent.critic_tf) tf.summary.histogram("target_Q", agent.target_Q) summary_var = tf.summary.merge_all() writer_t = tf.summary.FileWriter( osp.join(logger.get_dir(), 'train'), sess.graph) else: summary_var = tf.no_op() # Prepare everything. agent.initialize(sess) sess.graph.finalize() ## restore if kwargs['skillset']: ## restore skills my_skill_set = kwargs['my_skill_set'] my_skill_set.restore_skillset(sess=sess) ## restore current controller if kwargs["restore_dir"] is not None: restore_dir = osp.join(kwargs["restore_dir"], "model") if (restore_dir is not None) and rank == 0: print('Restore path : ', restore_dir) model_checkpoint_path = read_checkpoint_local(restore_dir) if model_checkpoint_path: saver.restore(U.get_session(), model_checkpoint_path) logger.info("checkpoint loaded:" + str(model_checkpoint_path)) tokens = model_checkpoint_path.split("-")[-1] # set global step global_t = int(tokens) print(">>> global step set:", global_t) agent.reset() obs = env.reset() # maintained across epochs episodes = 0 t = 0 start_time = time.time() # creating vars. this is done to keep the syntax for deleting the list simple a[:] = [] epoch_episode_rewards = [] epoch_episode_steps = [] epoch_actions = [] epoch_actor_losses = [] epoch_critic_losses = [] if param_noise is not None: epoch_adaptive_distances = [] eval_episode_rewards = [] eval_episode_success = [] # for each episode done = False episode_reward = 0. episode_step = 0 ## containers for hierarchical hindsight if kwargs["her"]: logger.debug("-" * 50 + '\nWill create HER\n' + "-" * 50) # per episode states, pactions, sub_states = [], [], [] print("Ready to go!") for epoch in range(global_t, nb_epochs): # stat containers epoch_episodes = 0. epoch_start_time = time.time() epoch_episode_rewards[:] = [] epoch_episode_steps[:] = [] epoch_actions[:] = [ ] # action mean: don't know if this indicates anything epoch_actor_losses[:] = [] epoch_critic_losses[:] = [] if param_noise is not None: epoch_adaptive_distances[:] = [] eval_episode_rewards[:] = [] eval_episode_success[:] = [] for cycle in range(nb_epoch_cycles): # Perform rollouts. for t_rollout in range( int(nb_rollout_steps / MPI.COMM_WORLD.Get_size())): # print(rank, t_rollout) # Predict next action. # exploration check if kwargs['look_ahead'] and (np.random.rand( ) < exploration.value(epoch * nb_epoch_cycles + cycle)): paction, planner_info = look_ahead_planner.create_plan( obs) else: paction, _ = agent.pi(obs, apply_noise=True, compute_Q=True) if (my_skill_set): ## break actions into primitives and their params primitives_prob = paction[:kwargs['my_skill_set'].len] primitive_id = np.argmax(primitives_prob) # print("skill chosen", primitive_id) r = 0. skill_obs = obs.copy() if kwargs['her']: curr_sub_states = [skill_obs.copy()] for _ in range(kwargs['commit_for']): action = my_skill_set.pi( primitive_id=primitive_id, obs=skill_obs.copy(), primitive_params=paction[my_skill_set.len:]) # Execute next action. if rank == 0 and render: sleep(0.1) env.render() assert max_action.shape == action.shape new_obs, skill_r, done, info = env.step( max_action * action ) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1]) r += skill_r if kwargs['her']: curr_sub_states.append(new_obs.copy()) skill_obs = new_obs if done or my_skill_set.termination( new_obs, primitive_id, primitive_params=paction[my_skill_set. len:]): break # assuming the skill is trained from different reward signal r = skill_r else: action = paction # Execute next action. if rank == 0 and render: env.render() assert max_action.shape == action.shape new_obs, r, done, info = env.step( max_action * action ) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1]) assert action.shape == env.action_space.shape t += 1 episode_reward += r episode_step += 1 # Book-keeping. epoch_actions.append(paction) agent.store_transition(obs, paction, r, new_obs, done) # storing info for hindsight if kwargs['her']: states.append(obs.copy()) pactions.append(paction.copy()) sub_states.append(curr_sub_states) # print(planner_info['next_state'][:6], new_obs[:6]) obs = new_obs if done: # Episode done. # update stats epoch_episode_rewards.append(episode_reward) episode_rewards_history.append(episode_reward) epoch_episode_steps.append(episode_step) epoch_episodes += 1 episodes += 1 # reinit episode_reward = 0. episode_step = 0 agent.reset() obs = env.reset() if kwargs["her"]: # logger.info("-"*50 +'\nCreating HER\n' + "-"*50) # create hindsight experience replay if kwargs['skillset']: her_states, her_rewards = env.apply_hierarchical_hindsight( states, pactions, new_obs.copy(), sub_states) else: her_states, her_rewards = env.apply_hindsight( states, pactions, new_obs.copy()) ## store her transitions: her_states: n+1, her_rewards: n for her_i in range(len(her_states) - 2): agent.store_transition(her_states[her_i], pactions[her_i], her_rewards[her_i], her_states[her_i + 1], False) #store last transition agent.store_transition(her_states[-2], pactions[-1], her_rewards[-1], her_states[-1], True) ## refresh the storage containers states[:], pactions[:] = [], [] if kwargs['skillset']: sub_states[:] = [] # print(rank, "Training!") # Train. for t_train in range(nb_train_steps): # print(rank, t_train) # Adapt param noise, if necessary. if (memory.nb_entries >= batch_size) and ( t % param_noise_adaption_interval == 0) and (param_noise is not None): distance = agent.adapt_param_noise() epoch_adaptive_distances.append(distance) cl, al, current_summary = agent.train(summary_var) epoch_critic_losses.append(cl) epoch_actor_losses.append(al) agent.update_target_net() if dologging and tf_sum_logging and rank == 0: writer_t.add_summary( current_summary, epoch * nb_epoch_cycles * nb_train_steps + cycle * nb_train_steps + t_train) # print("Evaluating!") # Evaluate after training is done. if (eval_env is not None) and rank == 0: for _ in range(nb_eval_episodes): eval_episode_reward = 0. eval_obs = eval_env.reset() eval_obs_start = eval_obs.copy() eval_done = False while (not eval_done): eval_paction, _ = agent.pi(eval_obs, apply_noise=False, compute_Q=False) if (kwargs['skillset']): ## break actions into primitives and their params eval_primitives_prob = eval_paction[:kwargs[ 'my_skill_set'].len] eval_primitive_id = np.argmax(eval_primitives_prob) eval_r = 0. eval_skill_obs = eval_obs.copy() for _ in range(kwargs['commit_for']): eval_action = my_skill_set.pi( primitive_id=eval_primitive_id, obs=eval_skill_obs.copy(), primitive_params=eval_paction[my_skill_set. len:]) eval_new_obs, eval_skill_r, eval_done, eval_info = eval_env.step( max_action * eval_action ) # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1]) if render_eval: eval_env.render() eval_r += eval_skill_r # check for skill termination or episode termination eval_terminate_skill = my_skill_set.termination( eval_new_obs, eval_primitive_id, primitive_params=eval_paction[my_skill_set. len:]) if eval_done or eval_terminate_skill: break eval_skill_obs = eval_new_obs # hack assuming the skills are trained from diff reward signal eval_r = eval_skill_r else: eval_action, _ = eval_paction, eval_pq eval_new_obs, eval_r, eval_done, eval_info = eval_env.step( max_action * eval_action) eval_episode_reward += eval_r eval_obs = eval_new_obs eval_episode_rewards.append(eval_episode_reward) eval_episode_rewards_history.append(eval_episode_reward) eval_episode_success.append( eval_info["done"] == "goal reached") if (eval_info["done"] == "goal reached"): logger.info( "success, training epoch:%d,starting config:" % epoch, eval_obs_start, 'final state', eval_obs) if dologging and rank == 0: print("Logging!") # Log stats. epoch_train_duration = time.time() - epoch_start_time duration = time.time() - start_time stats = agent.get_stats() combined_stats = {} for key in sorted(stats.keys()): combined_stats[key] = normal_mean(stats[key]) # Rollout statistics. combined_stats['rollout/return'] = normal_mean( epoch_episode_rewards) if len(episode_rewards_history) > 0: combined_stats['rollout/return_history'] = normal_mean( np.mean(episode_rewards_history)) else: combined_stats['rollout/return_history'] = 0. combined_stats['rollout/episode_steps'] = normal_mean( epoch_episode_steps) combined_stats['rollout/episodes'] = np.sum(epoch_episodes) combined_stats['rollout/actions_mean'] = normal_mean( epoch_actions) combined_stats['rollout/actions_std'] = normal_std( epoch_actions) # Train statistics. combined_stats['train/loss_actor'] = normal_mean( epoch_actor_losses) combined_stats['train/loss_critic'] = normal_mean( epoch_critic_losses) if param_noise is not None: combined_stats['train/param_noise_distance'] = normal_mean( epoch_adaptive_distances) if kwargs['look_ahead']: combined_stats['train/exploration'] = exploration.value( epoch * nb_epoch_cycles + cycle) # Evaluation statistics. if eval_env is not None: combined_stats['eval/return'] = normal_mean( eval_episode_rewards) combined_stats['eval/success'] = normal_mean( eval_episode_success) if len(eval_episode_rewards_history) > 0: combined_stats['eval/return_history'] = normal_mean( np.mean(eval_episode_rewards_history)) else: combined_stats['eval/return_history'] = 0. combined_stats['eval/episodes'] = normal_mean( len(eval_episode_rewards)) # Total statistics. combined_stats['total/duration'] = normal_mean(duration) combined_stats['total/rollout_per_second'] = normal_mean( float(t) / float(duration)) combined_stats['total/episodes'] = normal_mean(episodes) combined_stats['total/epochs'] = epoch + 1 combined_stats['total/steps'] = t for key in sorted(combined_stats.keys()): logger.record_tabular(key, combined_stats[key]) logger.dump_tabular() logger.info('') logdir = logger.get_dir() # if rank == 0 and logdir: # print("Dumping progress!") # if hasattr(env, 'get_state'): # with open(osp.join(logdir, 'env_state.pkl'), 'wb') as f: # pickle.dump(env.get_state(), f) # if eval_env and hasattr(eval_env, 'get_state'): # with open(osp.join(logdir, 'eval_env_state.pkl'), 'wb') as f: # pickle.dump(eval_env.get_state(), f) ## save tf model if rank == 0 and (epoch + 1) % save_freq == 0: print("Saving the model!") os.makedirs(osp.join(logdir, "model"), exist_ok=True) saver.save(U.get_session(), logdir + "/model/ddpg", global_step=epoch)
def train(env, eval_env, q_func, lr=5e-4, max_timesteps=100000, buffer_size=50000, exploration_fraction=0.1, exploration_final_eps=0.02, train_freq=1, batch_size=32, print_freq=100, checkpoint_freq=10000, learning_starts=1000, gamma=1.0, target_network_update_freq=500, prioritized_replay=False, prioritized_replay_alpha=0.6, prioritized_replay_beta0=0.4, prioritized_replay_beta_iters=None, prioritized_replay_eps=1e-6, param_noise=False, callback=None, my_skill_set=None, log_dir = None, num_eval_episodes=10, render=False, render_eval = False, commit_for = 1 ): """Train a deepq model. Parameters ------- env: gym.Env environment to train on q_func: (tf.Variable, int, str, bool) -> tf.Variable the model that takes the following inputs: observation_in: object the output of observation placeholder num_actions: int number of actions scope: str reuse: bool should be passed to outer variable scope and returns a tensor of shape (batch_size, num_actions) with values of every action. lr: float learning rate for adam optimizer max_timesteps: int number of env steps to optimizer for buffer_size: int size of the replay buffer exploration_fraction: float fraction of entire training period over which the exploration rate is annealed exploration_final_eps: float final value of random action probability train_freq: int update the model every `train_freq` steps. set to None to disable printing batch_size: int size of a batched sampled from replay buffer for training print_freq: int how often to print out training progress set to None to disable printing checkpoint_freq: int how often to save the model. This is so that the best version is restored at the end of the training. If you do not wish to restore the best version at the end of the training set this variable to None. learning_starts: int how many steps of the model to collect transitions for before learning starts gamma: float discount factor target_network_update_freq: int update the target network every `target_network_update_freq` steps. prioritized_replay: True if True prioritized replay buffer will be used. prioritized_replay_alpha: float alpha parameter for prioritized replay buffer prioritized_replay_beta0: float initial value of beta for prioritized replay buffer prioritized_replay_beta_iters: int number of iterations over which beta will be annealed from initial value to 1.0. If set to None equals to max_timesteps. prioritized_replay_eps: float epsilon to add to the TD errors when updating priorities. callback: (locals, globals) -> None function called at every steps with state of the algorithm. If callback returns true training stops. Returns ------- act: ActWrapper Wrapper over act function. Adds ability to save it and load it. See header of baselines/deepq/categorical.py for details on the act function. """ # Create all the functions necessary to train the model if my_skill_set: assert commit_for>=1, "commit_for >= 1" save_idx = 0 with U.single_threaded_session() as sess: ## restore if my_skill_set: action_shape = my_skill_set.len else: action_shape = env.action_space.n # capture the shape outside the closure so that the env object is not serialized # by cloudpickle when serializing make_obs_ph observation_space_shape = env.observation_space.shape def make_obs_ph(name): return U.BatchInput(observation_space_shape, name=name) act, train, update_target, debug = deepq.build_train( make_obs_ph=make_obs_ph, q_func=q_func, num_actions=action_shape, optimizer=tf.train.AdamOptimizer(learning_rate=lr), gamma=gamma, grad_norm_clipping=10, param_noise=param_noise ) act_params = { 'make_obs_ph': make_obs_ph, 'q_func': q_func, 'num_actions': action_shape, } act = ActWrapper(act, act_params) # Create the replay buffer if prioritized_replay: replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha) if prioritized_replay_beta_iters is None: prioritized_replay_beta_iters = max_timesteps beta_schedule = LinearSchedule(prioritized_replay_beta_iters, initial_p=prioritized_replay_beta0, final_p=1.0) else: replay_buffer = ReplayBuffer(buffer_size) beta_schedule = None # Create the schedule for exploration starting from 1. exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps), initial_p=1.0, final_p=exploration_final_eps) # Initialize the parameters and copy them to the target network. U.initialize() # sess.run(tf.variables_initializer(new_variables)) # sess.run(tf.global_variables_initializer()) update_target() if my_skill_set: ## restore skills my_skill_set.restore_skillset(sess=sess) episode_rewards = [0.0] saved_mean_reward = None obs = env.reset() reset = True model_saved = False model_file = os.path.join(log_dir, "model", "deepq") # save the initial act model print("Saving the starting model") os.makedirs(os.path.dirname(model_file), exist_ok=True) act.save(model_file + '.pkl') for t in range(max_timesteps): if callback is not None: if callback(locals(), globals()): break # Take action and update exploration to the newest value kwargs = {} if not param_noise: update_eps = exploration.value(t) update_param_noise_threshold = 0. else: update_eps = 0. # Compute the threshold such that the KL divergence between perturbed and non-perturbed # policy is comparable to eps-greedy exploration with eps = exploration.value(t). # See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017 # for detailed explanation. update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n)) kwargs['reset'] = reset kwargs['update_param_noise_threshold'] = update_param_noise_threshold kwargs['update_param_noise_scale'] = True paction = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0] if(my_skill_set): skill_obs = obs.copy() primitive_id = paction rew = 0. for _ in range(commit_for): ## break actions into primitives and their params action = my_skill_set.pi(primitive_id=primitive_id, obs = skill_obs.copy(), primitive_params=None) new_obs, skill_rew, done, _ = env.step(action) if render: # print(action) env.render() sleep(0.1) rew += skill_rew skill_obs = new_obs terminate_skill = my_skill_set.termination(new_obs) if done or terminate_skill: break else: action= paction env_action = action reset = False new_obs, rew, done, _ = env.step(env_action) if render: env.render() sleep(0.1) # Store transition in the replay buffer for the outer env replay_buffer.add(obs, paction, rew, new_obs, float(done)) obs = new_obs episode_rewards[-1] += rew if done: obs = env.reset() episode_rewards.append(0.0) reset = True print("Time:%d, episodes:%d"%(t,len(episode_rewards))) # add hindsight experience if t > learning_starts and t % train_freq == 0: # print('Training!') # Minimize the error in Bellman's equation on a batch sampled from replay buffer. if prioritized_replay: experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t)) (obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience else: obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size) weights, batch_idxes = np.ones_like(rewards), None td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights) if prioritized_replay: new_priorities = np.abs(td_errors) + prioritized_replay_eps replay_buffer.update_priorities(batch_idxes, new_priorities) if t > learning_starts and t % target_network_update_freq == 0: # Update target network periodically. update_target() # print(len(episode_rewards), episode_rewards[-11:-1]) mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1) num_episodes = len(episode_rewards) if (checkpoint_freq is not None and t > learning_starts and num_episodes > 50 and t % checkpoint_freq == 0): if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward: if print_freq is not None: logger.log("Saving model due to mean reward increase: {} -> {}".format( saved_mean_reward, mean_100ep_reward)) U.save_state(model_file) act.save(model_file + '%d.pkl'%save_idx) save_idx += 1 model_saved = True saved_mean_reward = mean_100ep_reward # else: # print(saved_mean_reward, mean_100ep_reward) if (eval_env is not None) and t > learning_starts and t % target_network_update_freq == 0: # dumping other stats logger.record_tabular("steps", t) logger.record_tabular("episodes", num_episodes) logger.record_tabular("mean 100 episode reward", mean_100ep_reward) logger.record_tabular("%d time spent exploring", int(100 * exploration.value(t))) print("Testing!") eval_episode_rewards = [] eval_episode_successes = [] for i in range(num_eval_episodes): eval_episode_reward = 0. eval_obs = eval_env.reset() eval_obs_start = eval_obs.copy() eval_done = False while(not eval_done): eval_paction = act(np.array(eval_obs)[None])[0] if(my_skill_set): eval_skill_obs = eval_obs.copy() eval_primitive_id = eval_paction eval_r = 0. for _ in range(commit_for): ## break actions into primitives and their params eval_action, _ = my_skill_set.pi(primitive_id=eval_primitive_id, obs = eval_skill_obs.copy(), primitive_params=None) eval_new_obs, eval_skill_rew, eval_done, eval_info = eval_env.step(eval_action) # print('env reward:%f'%eval_skill_rew) if render_eval: print("Render!") eval_env.render() print("rendered!") eval_r += eval_skill_rew eval_skill_obs = eval_new_obs eval_terminate_skill = my_skill_set.termination(eval_new_obs) if eval_done or eval_terminate_skill: break else: eval_action= eval_paction env_action = eval_action reset = False eval_new_obs, eval_r, eval_done, eval_info = eval_env.step(env_action) if render_eval: # print("Render!") eval_env.render() # print("rendered!") eval_episode_reward += eval_r # print("eval_r:%f, eval_episode_reward:%f"%(eval_r, eval_episode_reward)) eval_obs = eval_new_obs eval_episode_success = (eval_info["done"]=="goal reached") if(eval_episode_success): logger.info("success, training epoch:%d,starting config:"%t) eval_episode_rewards.append(eval_episode_reward) eval_episode_successes.append(eval_episode_success) combined_stats = {} # print(eval_episode_successes, np.mean(eval_episode_successes)) combined_stats['eval/return'] = normal_mean(eval_episode_rewards) combined_stats['eval/success'] = normal_mean(eval_episode_successes) combined_stats['eval/episodes'] = (len(eval_episode_rewards)) for key in sorted(combined_stats.keys()): logger.record_tabular(key, combined_stats[key]) print("dumping the stats!") logger.dump_tabular() if model_saved: if print_freq is not None: logger.log("Restored model with mean reward: {}".format(saved_mean_reward)) U.load_state(model_file)