def load_index(self, index, load_path=None): file_list = os.listdir(self.def_path_pre) file_list.sort(key=lambda x: os.path.getmtime(os.path.join(self.def_path_pre, x)), reverse=True) if load_path is None: load_path = os.path.join(self.def_path_pre, file_list[index]) load_variables(load_path=load_path, sess=self.sess) print('load_path: ', load_path)
def load_newest(self, load_path=None): file_list = os.listdir(self.def_path_pre) file_list.sort(key=lambda x: os.path.getmtime(os.path.join(self.def_path_pre, x))) if load_path is None: load_path = os.path.join(self.def_path_pre, file_list[-1]) load_variables(load_path=load_path, sess=self.sess) print('load_path: ', load_path)
def learn(self, total_timesteps=int(1e6), log_interval=200, pretrain_load_path=None): """ Parameters: ----------- log_interval: int, specifies how frequently the logs are printed out (default: 100) pretrain_load_path: pre-train model load path """ set_global_seeds(self.seed) # pretrain_load_path="/home/huangjp/CoppeliaSim_Edu_V4_0_0_Ubuntu18_04/programming/RL_Snake_Robot/algorithm/RL_algorithm/a2c/tmp/Y2020M06D29_h19m43s27" # pretrain_load_path="/home/huangjp/CoppeliaSim_Edu_V4_0_0_Ubuntu18_04/programming/RL_Snake_Robot/algorithm/RL_algorithm/a2c/tmp/Y2020M06D29_h20m18s25" # pretrain_load_path="/home/huangjp/CoppeliaSim_Edu_V4_0_0_Ubuntu18_04/programming/RL_Snake_Robot/algorithm/RL_algorithm/a2c/tmp/Y2020M07D11_h18m12s00" if pretrain_load_path is not None: load_variables(load_path=pretrain_load_path, sess=self.sess) # Instantiate the runner object runner = Runner(self.env, self, nsteps=self.nsteps, gamma=self.gamma) epinfobuf = deque(maxlen=100) # Calculate the batch_size nbatch = self.nenvs * self.nsteps # Start total timer tstart = time.time() with TensorboardWriter(self.graph, self.tb_log_path, 'A2C') as writer: for update in range(1, total_timesteps): # if update % learn_frequency != 0: # runner.run() # continue # Get mini batch of experiences obs, rewards, masks, actions, values, epinfos = runner.run() policy_loss, value_loss, policy_entropy, grads = self.train( obs, rewards, masks, actions, values, update, writer) epinfobuf.extend(epinfos) nseconds = time.time() - tstart # Calculate the fps (frame per second) fps = int((update * nbatch) / nseconds) if writer is not None: total_episode_reward_logger( self.episode_reward, rewards.reshape((self.nenvs, self.nsteps)), masks.reshape((self.nenvs, self.nsteps)), writer, update) if update % log_interval == 0 or update == 1: # Calculates if value function is a good predicator of the returns (ev > 1) # or if it's just worse than predicting nothing (ev =< 0) ev = explained_variance(values, rewards) logger.record_tabular("nupdates", update) logger.record_tabular("total_timesteps", update * nbatch) logger.record_tabular("fps", fps) logger.record_tabular("policy_entropy", float(policy_entropy)) logger.record_tabular("value_loss", float(value_loss)) logger.record_tabular("policy_loss", float(policy_loss)) logger.record_tabular("explained_variance", float(ev)) logger.record_tabular( "eprewmean", safe_mean([epinfo['r'] for epinfo in epinfobuf])) logger.record_tabular( "eplenmean", safe_mean([epinfo['l'] for epinfo in epinfobuf])) logger.dump_tabular() if update % 200 == 0 or update == total_timesteps // nbatch: if self.model_save_path is None: file_name = time.strftime('Y%YM%mD%d_h%Hm%Ms%S', time.localtime(time.time())) model_save_path = self.def_path_pre + file_name self.save(model_save_path) else: self.save(self.model_save_path) # print("grads = ", grads[-10:]) return self
def learn(self, total_timesteps=int(1e6), log_interval=4, reset_num_timesteps=True): pretrain_load_path = None # pretrain_load_path="/home/huangjp/CoppeliaSim_Edu_V4_0_0_Ubunt # pretrain_load_path="/home/huangjp/CoppeliaSim_Edu_V4_0_0_Ubunt # pretrain_load_path="/home/huangjp/CoppeliaSim_Edu_V4_0_0_Ubuntu18_04/programming/RL_Snake_Robot/algorithm/RL_algorithm/sac/tmp/Y2020M07D16_h11m15s56" # pretrain_load_path="/home/huangjp/CoppeliaSim_Edu_V4_0_0_Ubuntu18_04/programming/RL_Snake_Robot/algorithm/RL_algorithm/sac/tmp/Y2020M07D16_h18m36s16" if pretrain_load_path is not None: variables = self.params + self.target_params load_variables(load_path=pretrain_load_path, variables=variables, sess=self.sess) new_tb_log = self._init_num_timesteps(reset_num_timesteps) with TensorboardWriter(self.graph, self.tensorboard_log_path, tb_log_name="SAC", new_tb_log=new_tb_log) as writer: # Transform to callable if needed # self.learning_rate_scheduler = Scheduler(initial_value=self.learning_rate, n_values=total_timesteps, # schedule=self.learning_rate_scheduler) start_time = time.time() episode_rewards = [0.0] episode_successes = [] if self.action_noise is not None: self.action_noise.reset() # Retrieve unnormalized observation for saving into the buffer obs = self.env.reset() n_updates = 0 infos_values = [] for update in range(total_timesteps): # Before training starts, randomly sample actions # from a uniform distribution for better exploration. # Afterwards, use the learned policy # if random_exploration is set to 0 (normal setting) if self.num_timesteps < self.learning_start_threshold: # actions sampled from action space are from range specific to the environment # but algorithm operates on tanh-squashed actions therefore simple scaling is used # action = np.array([self.env.action_space.sample() for _ in range(self.n_envs)]) action = np.array([ np.random.random(self.env.action_space.shape) for _ in range(self.n_envs) ]) # action = scale_action(self.env.action_space, unscaled_action) else: action, _, _, _ = self.policy.step(obs, deterministic=False) mu, std = self.policy.proba_step(obs) if update % 500 == 0: print("mu = ", mu, " , std = ", std) # Add noise to the action (improve exploration, # not needed in general) if self.action_noise is not None: action = action + self.action_noise() # inferred actions need to be transformed to environment action_space before stepping # unscaled_action = unscale_action(self.env.action_space, action) # print("action = ", action) assert action[0].shape == self.env.action_space.shape new_obs, reward, done, infos = self.env.step(action) self.num_timesteps += 1 # Store only the unnormalized version obs_, new_obs_, reward_ = obs, new_obs, reward # Store transition in the replay buffer. self.replay_buffer.extend(obs_, action, reward_, new_obs_, done) obs = new_obs # Retrieve reward and episode length if using Monitor wrapper for info in infos: maybe_ep_info = info.get('episode') if maybe_ep_info: self.ep_info_buf.append(maybe_ep_info) if writer is not None: # Write reward per episode to tensorboard ep_reward = np.array([reward_]).reshape((1, -1)) ep_done = np.array([done]).reshape((1, -1)) tb_utils.total_episode_reward_logger( self.episode_reward, ep_reward, ep_done, writer, self.num_timesteps) if self.num_timesteps % self.train_freq == 0: mb_infos_vals = [] # Update policy, critics and target networks for grad_step in range(self.gradient_steps): # Break if the warmup phase is not over # or if there are not enough samples in the replay buffer if not self.replay_buffer.can_sample(self.batch_size) \ or self.num_timesteps < self.learning_start_threshold: break n_updates += 1 # Compute current learning_rate # current_lr = self.learning_rate_scheduler.value() # Update policy and critics (q functions) mb_infos_vals.append(self._train_step(update, writer)) # Update target network if (update + grad_step) % self.target_update_interval == 0: # Update target network self.sess.run(self.target_update_op) # Log losses and entropy, useful for monitor training if len(mb_infos_vals) > 0: infos_values = np.mean(mb_infos_vals, axis=0) # only record the return reward of the first environment episode_rewards[-1] += reward_[0] if done[0]: # if self.action_noise is not None: # self.action_noise.reset() # obs = self.env.reset() episode_rewards.append(0.0) print("The first env's reward: ", episode_rewards[-2]) if len(episode_rewards[-101:-1]) == 0: mean_reward = -np.inf else: mean_reward = round( float(np.mean(episode_rewards[-101:-1])), 1) num_episodes = len(episode_rewards) # Display training infos if done[0] and log_interval is not None and len( episode_rewards) % log_interval == 0: fps = int(update / (time.time() - start_time)) logger.record_tabular("episodes", num_episodes) logger.record_tabular("mean 100 episode reward", mean_reward) if len(self.ep_info_buf) > 0 and len( self.ep_info_buf[0]) > 0: logger.record_tabular( 'ep_rewmean', safe_mean([ ep_info['r'] for ep_info in self.ep_info_buf ])) logger.record_tabular( 'ep_lenmean', safe_mean([ ep_info['l'] for ep_info in self.ep_info_buf ])) logger.record_tabular("n_updates", n_updates) logger.record_tabular("fps", fps) logger.record_tabular('time_elapsed', int(time.time() - start_time)) # if len(episode_successes) > 0: # logger.record_tabular("success rate", episode_successes / num_episodes) if len(infos_values) > 0: for (name, val) in zip(self.infos_names, infos_values): logger.record_tabular(name, val) logger.record_tabular("total timesteps", self.num_timesteps) logger.dump_tabular() # Reset infos: infos_values = [] if update % 1000 == 0 or update == total_timesteps - 1: if self.model_save_path is None: file_name = time.strftime('Y%YM%mD%d_h%Hm%Ms%S', time.localtime(time.time())) model_save_path = self.def_path_pre + file_name self.save(model_save_path) else: self.save(self.model_save_path) return self