class DQN(Base_Agent): """A deep Q learning agent""" agent_name = "DQN" def __init__(self, config): Base_Agent.__init__(self, config) model_path = self.config.model_path if self.config.model_path else 'Models' self.memory = Replay_Buffer(self.hyperparameters["buffer_size"], self.hyperparameters["batch_size"], config.seed) self.q_network_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size) self.q_network_local_path = os.path.join( model_path, "{}_q_network_local.pt".format(self.agent_name)) if self.config.load_model: self.locally_load_policy() self.q_network_optimizer = optim.Adam( self.q_network_local.parameters(), lr=self.hyperparameters["learning_rate"], eps=1e-4) self.exploration_strategy = Epsilon_Greedy_Exploration(config) def reset_game(self): super(DQN, self).reset_game() self.update_learning_rate(self.hyperparameters["learning_rate"], self.q_network_optimizer) def step(self): """Runs a step within a game including a learning step if required""" while not self.done: self.action = self.pick_action() self.conduct_action(self.action) if self.time_for_q_network_to_learn(): for _ in range(self.hyperparameters["learning_iterations"]): self.learn() self.save_experience() self.state = self.next_state #this is to set the state for the next iteration self.global_step_number += 1 self.episode_number += 1 def pick_action(self, state=None): """Uses the local Q network and an epsilon greedy policy to pick an action""" # PyTorch only accepts mini-batches and not single observations so we have to use unsqueeze to add # a "fake" dimension to make it a mini-batch rather than a single observation if state is None: state = self.state if isinstance(state, np.int64) or isinstance(state, int): state = np.array([state]) state = torch.from_numpy(state).float().unsqueeze(0).to(self.device) if len(state.shape) < 2: state = state.unsqueeze(0) self.q_network_local.eval() #puts network in evaluation mode with torch.no_grad(): action_values = self.q_network_local(state) self.q_network_local.train() #puts network back in training mode action = self.exploration_strategy.perturb_action_for_exploration_purposes( { "action_values": action_values, "turn_off_exploration": self.turn_off_exploration, "episode_number": self.episode_number }) self.logger.info("Q values {} -- Action chosen {}".format( action_values, action)) return action def learn(self, experiences=None): """Runs a learning iteration for the Q network""" if experiences is None: states, actions, rewards, next_states, dones = self.sample_experiences( ) #Sample experiences else: states, actions, rewards, next_states, dones = experiences loss = self.compute_loss(states, next_states, rewards, actions, dones) actions_list = [action_X.item() for action_X in actions] self.logger.info("Action counts {}".format(Counter(actions_list))) self.take_optimisation_step( self.q_network_optimizer, self.q_network_local, loss, self.hyperparameters["gradient_clipping_norm"]) def compute_loss(self, states, next_states, rewards, actions, dones): """Computes the loss required to train the Q network""" with torch.no_grad(): Q_targets = self.compute_q_targets(next_states, rewards, dones) Q_expected = self.compute_expected_q_values(states, actions) loss = F.mse_loss(Q_expected, Q_targets) return loss def compute_q_targets(self, next_states, rewards, dones): """Computes the q_targets we will compare to predicted q values to create the loss to train the Q network""" Q_targets_next = self.compute_q_values_for_next_states(next_states) Q_targets = self.compute_q_values_for_current_states( rewards, Q_targets_next, dones) return Q_targets def compute_q_values_for_next_states(self, next_states): """Computes the q_values for next state we will use to create the loss to train the Q network""" Q_targets_next = self.q_network_local(next_states).detach().max( 1)[0].unsqueeze(1) return Q_targets_next def compute_q_values_for_current_states(self, rewards, Q_targets_next, dones): """Computes the q_values for current state we will use to create the loss to train the Q network""" Q_targets_current = rewards + (self.hyperparameters["discount_rate"] * Q_targets_next * (1 - dones)) return Q_targets_current def compute_expected_q_values(self, states, actions): """Computes the expected q_values we will use to create the loss to train the Q network""" Q_expected = self.q_network_local(states).gather(1, actions.long( )) #must convert actions to long so can be used as index return Q_expected def time_for_q_network_to_learn(self): """Returns boolean indicating whether enough steps have been taken for learning to begin and there are enough experiences in the replay buffer to learn from""" return self.right_amount_of_steps_taken( ) and self.enough_experiences_to_learn_from() def right_amount_of_steps_taken(self): """Returns boolean indicating whether enough steps have been taken for learning to begin""" return self.global_step_number % self.hyperparameters[ "update_every_n_steps"] == 0 def sample_experiences(self): """Draws a random sample of experience from the memory buffer""" experiences = self.memory.sample() states, actions, rewards, next_states, dones = experiences return states, actions, rewards, next_states, dones def locally_save_policy(self): """Saves the policy""" """保存策略,待添加""" torch.save(self.q_network_local.state_dict(), self.q_network_local_path) def locally_load_policy(self): print("locall_load_policy") if os.path.isfile(self.q_network_local_path): try: self.q_network_local.load_state_dict( torch.load(self.q_network_local_path)) print("load critic_local_path") except: pass
class HER_Base(object): """Contains methods needed to turn an algorithm into a hindsight experience replay (HER) algorithm""" def __init__(self, buffer_size, batch_size, HER_sample_proportion): self.HER_memory = Replay_Buffer(buffer_size, batch_size, self.config.seed) self.ordinary_buffer_batch_size = int(batch_size * (1.0 - HER_sample_proportion)) self.HER_buffer_batch_size = batch_size - self.ordinary_buffer_batch_size def reset_game(self): """Resets the game information so we are ready to play a new episode""" self.state_dict = self.environment.reset() self.observation = self.state_dict["observation"] self.desired_goal = self.state_dict["desired_goal"] self.achieved_goal = self.state_dict["achieved_goal"] self.state = self.create_state_from_observation_and_desired_goal( self.observation, self.desired_goal) self.next_state = None self.action = None self.reward = None self.done = False self.episode_states = [] self.episode_rewards = [] self.episode_actions = [] self.episode_next_states = [] self.episode_dones = [] self.episode_desired_goals = [] self.episode_achieved_goals = [] self.episode_observations = [] self.episode_next_desired_goals = [] self.episode_next_achieved_goals = [] self.episode_next_observations = [] self.total_episode_score_so_far = 0 def track_changeable_goal_episodes_data(self): """Saves the data from the recent episodes in a way compatible with changeable goal environments""" self.episode_rewards.append(self.reward) self.episode_actions.append(self.action) self.episode_dones.append(self.done) self.episode_states.append(self.state) self.episode_next_states.append(self.next_state) self.episode_desired_goals.append(self.state_dict["desired_goal"]) self.episode_achieved_goals.append(self.state_dict["achieved_goal"]) self.episode_observations.append(self.state_dict["observation"]) self.episode_next_desired_goals.append( self.next_state_dict["desired_goal"]) self.episode_next_achieved_goals.append( self.next_state_dict["achieved_goal"]) self.episode_next_observations.append( self.next_state_dict["observation"]) def conduct_action_in_changeable_goal_envs(self, action): """Adapts conduct_action from base agent so that can handle changeable goal environments""" self.next_state_dict, self.reward, self.done, _ = self.environment.step( action) self.total_episode_score_so_far += self.reward self.reward = max(min(self.reward, 1.0), -1.0) self.observation = self.next_state_dict["observation"] self.desired_goal = self.next_state_dict["desired_goal"] self.achieved_goal = self.next_state_dict["achieved_goal"] self.next_state = self.create_state_from_observation_and_desired_goal( self.observation, self.desired_goal) def create_state_from_observation_and_desired_goal(self, observation, desired_goal): return np.concatenate((observation, desired_goal)) def save_alternative_experience(self): """Saves the experiences as if the final state visited in the episode was the goal state""" new_goal = self.achieved_goal new_states = [ self.create_state_from_observation_and_desired_goal( observation, new_goal) for observation in self.episode_observations ] new_next_states = [ self.create_state_from_observation_and_desired_goal( observation, new_goal) for observation in self.episode_next_observations ] new_rewards = [ self.environment.compute_reward(next_achieved_goal, new_goal, None) for next_achieved_goal in self.episode_next_achieved_goals ] self.HER_memory.add_experience(new_states, self.episode_actions, new_rewards, new_next_states, self.episode_dones) def sample_from_HER_and_Ordinary_Buffer(self): """Samples from the ordinary replay buffer and HER replay buffer according to a proportion specified in config""" states, actions, rewards, next_states, dones = self.memory.sample( self.ordinary_buffer_batch_size) HER_states, HER_actions, HER_rewards, HER_next_states, HER_dones = self.HER_memory.sample( self.HER_buffer_batch_size) states = torch.cat((states, HER_states)) actions = torch.cat((actions, HER_actions)) rewards = torch.cat((rewards, HER_rewards)) next_states = torch.cat((next_states, HER_next_states)) dones = torch.cat((dones, HER_dones)) return states, actions, rewards, next_states, dones
class SAC(Base_Agent): """Soft Actor-Critic model based on the 2018 paper https://arxiv.org/abs/1812.05905 and on this github implementation https://github.com/pranz24/pytorch-soft-actor-critic. It is an actor-critic algorithm where the agent is also trained to maximise the entropy of their actions as well as their cumulative reward""" agent_name = "SAC" def __init__(self, config): Base_Agent.__init__(self, config) assert self.action_types == "CONTINUOUS", "Action types must be continuous. Use SAC Discrete instead for discrete actions" assert self.config.hyperparameters["Actor"][ "final_layer_activation"] != "Softmax", "Final actor layer must not be softmax" self.hyperparameters = config.hyperparameters self.critic_local = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") self.critic_local_2 = self.create_NN( input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic", override_seed=self.config.seed + 1) self.critic_optimizer = torch.optim.Adam( self.critic_local.parameters(), lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4) self.critic_optimizer_2 = torch.optim.Adam( self.critic_local_2.parameters(), lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4) self.critic_target = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") self.critic_target_2 = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") Base_Agent.copy_model_over(self.critic_local, self.critic_target) Base_Agent.copy_model_over(self.critic_local_2, self.critic_target_2) self.memory = Replay_Buffer( self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"], self.config.seed) self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size * 2, key_to_use="Actor") self.actor_optimizer = torch.optim.Adam( self.actor_local.parameters(), lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4) self.automatic_entropy_tuning = self.hyperparameters[ "automatically_tune_entropy_hyperparameter"] if self.automatic_entropy_tuning: self.target_entropy = -torch.prod( torch.Tensor(self.environment.action_space.shape).to( self.device)).item() # heuristic value from the paper self.log_alpha = torch.zeros(1, requires_grad=True, device=self.device) self.alpha = self.log_alpha.exp() self.alpha_optim = Adam( [self.log_alpha], lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4) else: self.alpha = self.hyperparameters["entropy_term_weight"] self.add_extra_noise = self.hyperparameters["add_extra_noise"] if self.add_extra_noise: self.noise = OU_Noise(self.action_size, self.config.seed, self.hyperparameters["mu"], self.hyperparameters["theta"], self.hyperparameters["sigma"]) self.do_evaluation_iterations = self.hyperparameters[ "do_evaluation_iterations"] def save_result(self): """Saves the result of an episode of the game. Overriding the method in Base Agent that does this because we only want to keep track of the results during the evaluation episodes""" if self.episode_number == 1 or not self.do_evaluation_iterations: self.game_full_episode_scores.extend( [self.total_episode_score_so_far]) self.rolling_results.append( np.mean( self.game_full_episode_scores[-1 * self.rolling_score_window:])) self.save_max_result_seen() elif (self.episode_number - 1) % TRAINING_EPISODES_PER_EVAL_EPISODE == 0: self.game_full_episode_scores.extend([ self.total_episode_score_so_far for _ in range(TRAINING_EPISODES_PER_EVAL_EPISODE) ]) self.rolling_results.extend([ np.mean( self.game_full_episode_scores[-1 * self.rolling_score_window:]) for _ in range(TRAINING_EPISODES_PER_EVAL_EPISODE) ]) self.save_max_result_seen() def reset_game(self): """Resets the game information so we are ready to play a new episode""" Base_Agent.reset_game(self) if self.add_extra_noise: self.noise.reset() def step(self): """Runs an episode on the game, saving the experience and running a learning step if appropriate""" eval_ep = self.episode_number % TRAINING_EPISODES_PER_EVAL_EPISODE == 0 and self.do_evaluation_iterations self.episode_step_number_val = 0 while not self.done: self.episode_step_number_val += 1 self.action = self.pick_action(eval_ep) self.conduct_action(self.action) if self.time_for_critic_and_actor_to_learn(): for _ in range(self.hyperparameters[ "learning_updates_per_learning_session"]): self.learn() mask = False if self.episode_step_number_val >= self.environment._max_episode_steps else self.done if not eval_ep: self.save_experience(experience=(self.state, self.action, self.reward, self.next_state, mask)) self.state = self.next_state self.global_step_number += 1 print(self.total_episode_score_so_far) if eval_ep: self.print_summary_of_latest_evaluation_episode() self.episode_number += 1 def pick_action(self, eval_ep, state=None): """Picks an action using one of three methods: 1) Randomly if we haven't passed a certain number of steps, 2) Using the actor in evaluation mode if eval_ep is True 3) Using the actor in training mode if eval_ep is False. The difference between evaluation and training mode is that training mode does more exploration""" if state is None: state = self.state if eval_ep: action = self.actor_pick_action(state=state, eval=True) elif self.global_step_number < self.hyperparameters[ "min_steps_before_learning"]: action = self.environment.action_space.sample() print("Picking random action ", action) else: action = self.actor_pick_action(state=state) if self.add_extra_noise: action += self.noise.sample() return action def actor_pick_action(self, state=None, eval=False): """Uses actor to pick an action in one of two ways: 1) If eval = False and we aren't in eval mode then it picks an action that has partly been randomly sampled 2) If eval = True then we pick the action that comes directly from the network and so did not involve any random sampling""" if state is None: state = self.state state = torch.FloatTensor([state]).to(self.device) if len(state.shape) == 1: state = state.unsqueeze(0) if eval == False: action, _, _ = self.produce_action_and_action_info(state) else: with torch.no_grad(): _, z, action = self.produce_action_and_action_info(state) action = action.detach().cpu().numpy() return action[0] def pick_best_action(self, state=None): """Picks an action using the actor network without noise""" if state is None: state = self.state state = torch.FloatTensor([state]).to(self.device) if len(state.shape) == 1: state = state.unsqueeze(0) self.actor_local.eval() with torch.no_grad(): actor_output = self.actor_local(state) mean = actor_output[:, :self.action_size] action = torch.tanh(mean).detach().cpu().numpy() self.actor_local.train() return action[0] def produce_action_and_action_info(self, state): """Given the state, produces an action, the log probability of the action, and the tanh of the mean action""" actor_output = self.actor_local(state) mean, log_std = actor_output[:, :self. action_size], actor_output[:, self. action_size:] std = log_std.exp() normal = Normal(mean, std) x_t = normal.rsample( ) #rsample means it is sampled using reparameterisation trick action = torch.tanh(x_t) log_prob = normal.log_prob(x_t) log_prob -= torch.log(1 - action.pow(2) + EPSILON) log_prob = log_prob.sum(1, keepdim=True) return action, log_prob, torch.tanh(mean) def time_for_critic_and_actor_to_learn(self): """Returns boolean indicating whether there are enough experiences to learn from and it is time to learn for the actor and critic""" return self.global_step_number > self.hyperparameters["min_steps_before_learning"] and \ self.enough_experiences_to_learn_from() and self.global_step_number % self.hyperparameters["update_every_n_steps"] == 0 def learn(self): """Runs a learning iteration for the actor, both critics and (if specified) the temperature parameter""" state_batch, action_batch, reward_batch, next_state_batch, mask_batch = self.sample_experiences( ) qf1_loss, qf2_loss = self.calculate_critic_losses( state_batch, action_batch, reward_batch, next_state_batch, mask_batch) self.update_critic_parameters(qf1_loss, qf2_loss) policy_loss, log_pi = self.calculate_actor_loss(state_batch) if self.automatic_entropy_tuning: alpha_loss = self.calculate_entropy_tuning_loss(log_pi) else: alpha_loss = None self.update_actor_parameters(policy_loss, alpha_loss) def sample_experiences(self): return self.memory.sample() def calculate_critic_losses(self, state_batch, action_batch, reward_batch, next_state_batch, mask_batch): """Calculates the losses for the two critics. This is the ordinary Q-learning loss except the additional entropy term is taken into account""" with torch.no_grad(): next_state_action, next_state_log_pi, _ = self.produce_action_and_action_info( next_state_batch) qf1_next_target = self.critic_target( torch.cat((next_state_batch, next_state_action), 1)) qf2_next_target = self.critic_target_2( torch.cat((next_state_batch, next_state_action), 1)) min_qf_next_target = torch.min( qf1_next_target, qf2_next_target) - self.alpha * next_state_log_pi next_q_value = reward_batch + ( 1.0 - mask_batch) * self.hyperparameters["discount_rate"] * ( min_qf_next_target) qf1 = self.critic_local(torch.cat((state_batch, action_batch), 1)) qf2 = self.critic_local_2(torch.cat((state_batch, action_batch), 1)) qf1_loss = F.mse_loss(qf1, next_q_value) qf2_loss = F.mse_loss(qf2, next_q_value) return qf1_loss, qf2_loss def calculate_actor_loss(self, state_batch): """Calculates the loss for the actor. This loss includes the additional entropy term""" action, log_pi, _ = self.produce_action_and_action_info(state_batch) qf1_pi = self.critic_local(torch.cat((state_batch, action), 1)) qf2_pi = self.critic_local_2(torch.cat((state_batch, action), 1)) min_qf_pi = torch.min(qf1_pi, qf2_pi) policy_loss = ((self.alpha * log_pi) - min_qf_pi).mean() return policy_loss, log_pi def calculate_entropy_tuning_loss(self, log_pi): """Calculates the loss for the entropy temperature parameter. This is only relevant if self.automatic_entropy_tuning is True.""" alpha_loss = -(self.log_alpha * (log_pi + self.target_entropy).detach()).mean() return alpha_loss def update_critic_parameters(self, critic_loss_1, critic_loss_2): """Updates the parameters for both critics""" self.take_optimisation_step( self.critic_optimizer, self.critic_local, critic_loss_1, self.hyperparameters["Critic"]["gradient_clipping_norm"]) self.take_optimisation_step( self.critic_optimizer_2, self.critic_local_2, critic_loss_2, self.hyperparameters["Critic"]["gradient_clipping_norm"]) self.soft_update_of_target_network( self.critic_local, self.critic_target, self.hyperparameters["Critic"]["tau"]) self.soft_update_of_target_network( self.critic_local_2, self.critic_target_2, self.hyperparameters["Critic"]["tau"]) def update_actor_parameters(self, actor_loss, alpha_loss): """Updates the parameters for the actor and (if specified) the temperature parameter""" self.take_optimisation_step( self.actor_optimizer, self.actor_local, actor_loss, self.hyperparameters["Actor"]["gradient_clipping_norm"]) if alpha_loss is not None: self.take_optimisation_step(self.alpha_optim, None, alpha_loss, None) self.alpha = self.log_alpha.exp() def print_summary_of_latest_evaluation_episode(self): """Prints a summary of the latest episode""" print(" ") print("----------------------------") print("Episode score {} ".format(self.total_episode_score_so_far)) print("----------------------------") def locally_save_policy(self): """Saves the policy""" save_dict = { 'critic_local': self.critic_local.state_dict(), 'critic_local_2': self.critic_local_2.state_dict(), 'critic_target': self.critic_target.state_dict(), 'critic_target_2': self.critic_target_2.state_dict(), 'actor_local': self.actor_local.state_dict(), } save_path = os.path.join( self.hyperparameters['save_path'], self.agent_name + '_' + str(self.episode_number) + '.pt') torch.save(save_dict, save_path) def save_model(self): self.locally_save_policy() def load_model(self, model_path=None): if model_path is None: model_path = os.path.join(self.hyperparameters['save_path'], self.agent_name + '.pt') if os.path.exists(model_path): if torch.cuda.is_available(): save_dict = torch.load(model_path) else: save_dict = torch.load(model_path, map_location=torch.device('cpu')) self.critic_local.load_state_dict(save_dict['critic_local']) self.critic_local_2.load_state_dict(save_dict['critic_local_2']) self.critic_target.load_state_dict(save_dict['critic_target']) self.critic_target_2.load_state_dict(save_dict['critic_target_2']) self.actor_local.load_state_dict(save_dict['actor_local']) print('load model from', os.path.exists(model_path)) else: print('No File:', model_path)
class DDPG(Base_Agent): """A DDPG Agent""" agent_name = "DDPG" def __init__(self, config): Base_Agent.__init__(self, config) self.hyperparameters = config.hyperparameters self.critic_local = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") self.critic_target = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") Base_Agent.copy_model_over(self.critic_local, self.critic_target) self.critic_optimizer = optim.Adam( self.critic_local.parameters(), lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4) self.memory = Replay_Buffer( self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"], self.config.seed) self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor") self.actor_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor") Base_Agent.copy_model_over(self.actor_local, self.actor_target) self.actor_optimizer = optim.Adam( self.actor_local.parameters(), lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4) self.exploration_strategy = OU_Noise_Exploration(self.config) def step(self): """Runs a step in the game""" self.episode_step_number_val = 0 while not self.done: self.episode_step_number_val += 1 # print("State ", self.state.shape) self.action = self.pick_action() self.conduct_action(self.action) if self.time_for_critic_and_actor_to_learn(): for _ in range(self.hyperparameters[ "learning_updates_per_learning_session"]): states, actions, rewards, next_states, dones = self.sample_experiences( ) self.critic_learn(states, actions, rewards, next_states, dones) self.actor_learn(states) self.save_experience() self.state = self.next_state #this is to set the state for the next iteration self.global_step_number += 1 self.episode_number += 1 def sample_experiences(self): return self.memory.sample() def pick_action(self, state=None): """Picks an action using the actor network and then adds some noise to it to ensure exploration""" if state is None: state = torch.from_numpy(self.state).float().unsqueeze(0).to( self.device) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() action = self.exploration_strategy.perturb_action_for_exploration_purposes( { "action": action, "device": self.device }) return action.squeeze(0) def pick_best_action(self, state=None): """Picks an action using the actor network without noise""" if state is None: state = torch.from_numpy(self.state).float().unsqueeze(0).to( self.device) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() return action.squeeze(0) def critic_learn(self, states, actions, rewards, next_states, dones): """Runs a learning iteration for the critic""" loss = self.compute_loss(states, next_states, rewards, actions, dones) self.take_optimisation_step( self.critic_optimizer, self.critic_local, loss, self.hyperparameters["Critic"]["gradient_clipping_norm"]) self.soft_update_of_target_network( self.critic_local, self.critic_target, self.hyperparameters["Critic"]["tau"]) def compute_loss(self, states, next_states, rewards, actions, dones): """Computes the loss for the critic""" with torch.no_grad(): critic_targets = self.compute_critic_targets( next_states, rewards, dones) critic_expected = self.compute_expected_critic_values(states, actions) loss = functional.mse_loss(critic_expected, critic_targets) return loss def compute_critic_targets(self, next_states, rewards, dones): """Computes the critic target values to be used in the loss for the critic""" critic_targets_next = self.compute_critic_values_for_next_states( next_states) critic_targets = self.compute_critic_values_for_current_states( rewards, critic_targets_next, dones) return critic_targets def compute_critic_values_for_next_states(self, next_states): """Computes the critic values for next states to be used in the loss for the critic""" with torch.no_grad(): actions_next = self.actor_target(next_states) critic_targets_next = self.critic_target( torch.cat((next_states, actions_next), 1)) return critic_targets_next def compute_critic_values_for_current_states(self, rewards, critic_targets_next, dones): """Computes the critic values for current states to be used in the loss for the critic""" critic_targets_current = rewards + ( self.hyperparameters["discount_rate"] * critic_targets_next * (1.0 - dones)) return critic_targets_current def compute_expected_critic_values(self, states, actions): """Computes the expected critic values to be used in the loss for the critic""" critic_expected = self.critic_local(torch.cat((states, actions), 1)) return critic_expected def time_for_critic_and_actor_to_learn(self): """Returns boolean indicating whether there are enough experiences to learn from and it is time to learn for the actor and critic""" return self.enough_experiences_to_learn_from( ) and self.global_step_number % self.hyperparameters[ "update_every_n_steps"] == 0 def actor_learn(self, states): """Runs a learning iteration for the actor""" if self.done: #we only update the learning rate at end of each episode self.update_learning_rate( self.hyperparameters["Actor"]["learning_rate"], self.actor_optimizer) actor_loss = self.calculate_actor_loss(states) self.take_optimisation_step( self.actor_optimizer, self.actor_local, actor_loss, self.hyperparameters["Actor"]["gradient_clipping_norm"]) self.soft_update_of_target_network( self.actor_local, self.actor_target, self.hyperparameters["Actor"]["tau"]) def calculate_actor_loss(self, states): """Calculates the loss for the actor""" actions_pred = self.actor_local(states) actor_loss = -self.critic_local(torch.cat( (states, actions_pred), 1)).mean() return actor_loss def locally_save_policy(self): """Saves the policy""" save_dict = { 'critic_local': self.critic_local.state_dict(), 'critic_target': self.critic_target.state_dict(), 'actor_local': self.actor_local.state_dict(), 'actor_target': self.actor_target.state_dict(), } save_path = os.path.join( self.hyperparameters['save_path'], self.agent_name + '_' + str(self.episode_number) + '.pt') torch.save(save_dict, save_path) def save_model(self): self.locally_save_policy() def load_model(self, model_path=None): if model_path is None: model_path = os.path.join(self.hyperparameters['save_path'], self.agent_name + '.pt') if os.path.exists(model_path): save_dict = torch.load(model_path) self.critic_local.load_state_dict(save_dict['critic_local']) self.critic_target.load_state_dict(save_dict['critic_target']) self.actor_local.load_state_dict(save_dict['actor_local']) self.actor_target.load_state_dict(save_dict['actor_target']) print('load model from', os.path.exists(model_path)) else: print('No File:', model_path)
class DDPG(Base_Agent): """A DDPG Agent""" agent_name = "DDPG" def __init__(self, config): Base_Agent.__init__(self, config) self.hyperparameters = config.hyperparameters self.critic_local = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") self.critic_target = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") Base_Agent.copy_model_over(self.critic_local, self.critic_target) self.critic_optimizer = optim.Adam( self.critic_local.parameters(), lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4) self.memory = Replay_Buffer( self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"], self.config.seed) self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor") self.actor_target = self.create_NN(input_dim=self.state_size, output_dim=self.action_size, key_to_use="Actor") Base_Agent.copy_model_over(self.actor_local, self.actor_target) self.actor_optimizer = optim.Adam( self.actor_local.parameters(), lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4) self.exploration_strategy = OU_Noise_Exploration(self.config) if self.video_mode: self.file_name = self.environment_title + "_" + self.agent_name + "_videos" for i in range(config.num_episodes_to_run): pathset = os.path.join(self.file_name) if not (os.path.exists(pathset)): os.mkdir(pathset) # f = tables.open_file(self.file_name, mode = 'w') # f.close() # datainfo = "DDPG_"+ self.environment_title + "_info.txt" # f = open(self.file_name, 'w') # f.close() # f = open(datainfo, 'w') # f.write(str(self.height)) # f.write(str(self.width)) # f.write(str(self.channel)) # f.write(str(config.max_step)) # f.write(str(config.num_episodes_to_run)) # f.close() self.save_max_result_list_list = [] def step(self): """Runs a step in the game""" # print("(DDPG) into the step") # if self.video_mode: # f = open(self.file_name, mode = 'a') # f = open(self.file_name, 'a') # f.write("Episode" + str(self.episode_number) +"\n") # self.f = tables.open_file(self.file_name, mode='a') # self.atom = tables.Int64Atom() # self.array_c = self.f.create_earray(self.f.root, "Episode"+str(self.episode_number), self.atom, (0,self.height, self.width, self.channel)) record_video = self.video_mode and self.config.num_episodes_to_run - 10 <= self.episode_number if record_video: render_list = [] save_max_score_list = [] while not self.done: # print("State ", self.state.shape) self.action = self.pick_action() # print("picked action") """This is for the Cart-Pole environment""" if (self.get_environment_title() == "CartPole"): go_action = np.argmax(self.action) self.action = np.zeros(2) # print(self.action) self.action[go_action] = 1 # self.action = np.put(self.action, go_action, 1) # print(self.action) self.conduct_action(go_action) else: self.conduct_action(self.action) # print("(DDPG) action conducted! Rendering...") img = self.environment.render('rgb_array') if record_video: # f = open(self.file_name, mode='wb') render_list.append(img) # img = np.reshape(img, (1)).tolist() # f.write(str(img)) # f.write('\n') # img = np.reshape(img, (1, img.shape[0], img.shape[1], img.shape[2])) # print(type(img)) # print(img.shape) # print(self.array_c.shape) # print(img) # line = '\n' # f.write(img.tostring()) # f.write(line.encode("utf-8")) # f.close() # self.array_c.append(img) # self.render.append(img) save_max_score_list.append(img) # print("(DDPG)outside the loop") # print(self.time_for_critic_and_actor_to_learn()) # This is the learning part if self.time_for_critic_and_actor_to_learn(): # print("(DDPG) It is time to learn!!") for _ in range(self.hyperparameters[ "learning_updates_per_learning_session"]): states, actions, rewards, next_states, dones = self.sample_experiences( ) # print("(DDPG) running in range") self.critic_learn(states, actions, rewards, next_states, dones) self.actor_learn(states) # print("(DDPG)running in range complete") # print("(DDPG) outside of critic loop") self.save_experience() # print("(DDPG) saving experience") ###################### self.state = self.next_state #this is to set the state for the next iteration self.global_step_number += 1 # print("(DDPG) incrementing step number") if record_video: render_list = np.array(render_list) np.save(self.file_name + '/episode' + str(self.episode_number + 1), render_list) if self.total_episode_score_so_far > -0.2: if len(self.save_max_result_list_list) == 10: self.save_max_result_list_list.pop(0) self.save_max_result_list_list.append(save_max_score_list) if self.config.num_episodes_to_run == self.episode_number + 1: i = 1 for save_max_score_list in self.save_max_result_list_list: save_max_score_list = np.array(save_max_score_list) np.save(self.file_name + '/maxscore' + str(i), save_max_score_list) i += 1 self.episode_number += 1 # print("The epsiode end! rendering!!") # self.environment.render() def sample_experiences(self): return self.memory.sample() def pick_action(self, state=None): """Picks an action using the actor network and then adds some noise to it to ensure exploration""" if state is None: state = torch.from_numpy(self.state).float().unsqueeze(0).to( self.device) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() action = self.exploration_strategy.perturb_action_for_exploration_purposes( {"action": action}) return action.squeeze(0) def critic_learn(self, states, actions, rewards, next_states, dones): """Runs a learning iteration for the critic""" # print("(DDPG) inside the critic_learn()") loss = self.compute_loss(states, next_states, rewards, actions, dones) # print("(DDPG) critic learn loss: ",loss) self.take_optimisation_step( self.critic_optimizer, self.critic_local, loss, self.hyperparameters["Critic"]["gradient_clipping_norm"]) self.soft_update_of_target_network( self.critic_local, self.critic_target, self.hyperparameters["Critic"]["tau"]) def compute_loss(self, states, next_states, rewards, actions, dones): """Computes the loss for the critic""" # print("(DDPG) inside the compute_loss()") with torch.no_grad(): critic_targets = self.compute_critic_targets( next_states, rewards, dones) # print("(DDPG) after torch.no_grad") critic_expected = self.compute_expected_critic_values(states, actions) loss = functional.mse_loss(critic_expected, critic_targets) return loss def compute_critic_targets(self, next_states, rewards, dones): """Computes the critic target values to be used in the loss for the critic""" # print("(DDPG) inside the compute_critic_targets") critic_targets_next = self.compute_critic_values_for_next_states( next_states) critic_targets = self.compute_critic_values_for_current_states( rewards, critic_targets_next, dones) return critic_targets def compute_critic_values_for_next_states(self, next_states): """Computes the critic values for next states to be used in the loss for the critic""" # print("(DDPG) inside the compute_critic_values_for_next_states") with torch.no_grad(): # print("(DDPG) comput_critic_values_for_next_states()) inside the torch.no_grad()") # input() # print(self.actor_target) # print(next_states) # print(self.actor_target(next_states)) actions_next = self.actor_target(next_states) # input() # print("(DDPG comput_critic_values_for_next_states()) after calculating actor_target") critic_targets_next = self.critic_target( torch.cat((next_states, actions_next), 1)) # print("(DDPG compute_critic_values_for_next_states()) after torch.no_grad") return critic_targets_next def compute_critic_values_for_current_states(self, rewards, critic_targets_next, dones): """Computes the critic values for current states to be used in the loss for the critic""" critic_targets_current = rewards + ( self.hyperparameters["discount_rate"] * critic_targets_next * (1.0 - dones)) return critic_targets_current def compute_expected_critic_values(self, states, actions): """Computes the expected critic values to be used in the loss for the critic""" critic_expected = self.critic_local(torch.cat((states, actions), 1)) return critic_expected def time_for_critic_and_actor_to_learn(self): """Returns boolean indicating whether there are enough experiences to learn from and it is time to learn for the actor and critic""" return self.enough_experiences_to_learn_from( ) and self.global_step_number % self.hyperparameters[ "update_every_n_steps"] == 0 def actor_learn(self, states): """Runs a learning iteration for the actor""" if self.done: #we only update the learning rate at end of each episode self.update_learning_rate( self.hyperparameters["Actor"]["learning_rate"], self.actor_optimizer) actor_loss = self.calculate_actor_loss(states) self.take_optimisation_step( self.actor_optimizer, self.actor_local, actor_loss, self.hyperparameters["Actor"]["gradient_clipping_norm"]) self.soft_update_of_target_network( self.actor_local, self.actor_target, self.hyperparameters["Actor"]["tau"]) def calculate_actor_loss(self, states): """Calculates the loss for the actor""" actions_pred = self.actor_local(states) actor_loss = -self.critic_local(torch.cat( (states, actions_pred), 1)).mean() return actor_loss
class DQN(Base_Agent): """A deep Q learning agent""" agent_name = "DQN" def __init__(self, config): Base_Agent.__init__(self, config) self.memory = Replay_Buffer(self.hyperparameters["buffer_size"], self.hyperparameters["batch_size"], config.seed) self.q_network_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size) self.q_network_optimizer = optim.SGD( self.q_network_local.parameters(), lr=self.hyperparameters["learning_rate"], weight_decay=5e-4) self.exploration_strategy = Epsilon_Greedy_Exploration(config) def reset_game(self): super(DQN, self).reset_game() self.update_learning_rate(self.hyperparameters["learning_rate"], self.q_network_optimizer) def step(self): """Runs a step within a game including a learning step if required""" while not self.done: # print('state:', self.state) # self.environment.render() self.action = self.pick_action() self.conduct_action(self.action) if self.time_for_q_network_to_learn(): for _ in range(self.hyperparameters["learning_iterations"]): try: self.environment.pause() # print('pause') self.learn() self.environment.resume() # print('resume') except: self.learn() self.save_experience() self.state = self.next_state #this is to set the state for the next iteration self.global_step_number += 1 self.episode_number += 1 def pick_action(self, state=None): """Uses the local Q network and an epsilon greedy policy to pick an action""" # PyTorch only accepts mini-batches and not single observations so we have to use unsqueeze to add # a "fake" dimension to make it a mini-batch rather than a single observation if state is None: state = self.state if isinstance(state, np.int64) or isinstance(state, int): state = np.array([state]) state = torch.from_numpy(state).float().unsqueeze(0).to(self.device) if len(state.shape) < 2: state = state.unsqueeze(0) self.q_network_local.eval() #puts network in evaluation mode with torch.no_grad(): action_values = self.q_network_local(state) self.q_network_local.train() #puts network back in training mode force_explore = self.config.force_explore_mode and self.need_to_force_explore( ) if force_explore: print('explore...') action = self.exploration_strategy.perturb_action_for_exploration_purposes( { "action_values": action_values, "turn_off_exploration": self.turn_off_exploration, "episode_number": self.episode_number, "force_explore": force_explore }) # self.logger.info("Q values {} -- Action chosen {}".format(action_values, action)) return action def learn(self, experiences=None): """Runs a learning iteration for the Q network""" if experiences is None: states, actions, rewards, next_states, dones = self.sample_experiences( ) #Sample experiences else: states, actions, rewards, next_states, dones = experiences loss = self.compute_loss(states, next_states, rewards, actions, dones) actions_list = [action_X.item() for action_X in actions] self.logger.info("Action counts {}".format(Counter(actions_list))) self.take_optimisation_step( self.q_network_optimizer, self.q_network_local, loss, self.hyperparameters["gradient_clipping_norm"]) def compute_loss(self, states, next_states, rewards, actions, dones): """Computes the loss required to train the Q network""" with torch.no_grad(): Q_targets = self.compute_q_targets(next_states, rewards, dones) Q_expected = self.compute_expected_q_values(states, actions) # loss = F.mse_loss(Q_expected, Q_targets) loss = nn.MSELoss(size_average=False)(Q_expected, Q_targets) return loss def compute_q_targets(self, next_states, rewards, dones): """Computes the q_targets we will compare to predicted q values to create the loss to train the Q network""" Q_targets_next = self.compute_q_values_for_next_states(next_states) Q_targets = self.compute_q_values_for_current_states( rewards, Q_targets_next, dones) return Q_targets def compute_q_values_for_next_states(self, next_states): """Computes the q_values for next state we will use to create the loss to train the Q network""" Q_targets_next = self.q_network_local(next_states).detach().max( 1)[0].unsqueeze(1) return Q_targets_next def compute_q_values_for_current_states(self, rewards, Q_targets_next, dones): """Computes the q_values for current state we will use to create the loss to train the Q network""" Q_targets_current = rewards + (self.hyperparameters["discount_rate"] * Q_targets_next * (1 - dones)) return Q_targets_current def compute_expected_q_values(self, states, actions): """Computes the expected q_values we will use to create the loss to train the Q network""" Q_expected = self.q_network_local(states).gather(1, actions.long( )) #must convert actions to long so can be used as index return Q_expected def time_for_q_network_to_learn(self): """Returns boolean indicating whether enough steps have been taken for learning to begin and there are enough experiences in the replay buffer to learn from""" return self.right_amount_of_steps_taken( ) and self.enough_experiences_to_learn_from() def right_amount_of_steps_taken(self): """Returns boolean indicating whether enough steps have been taken for learning to begin""" return self.global_step_number % self.hyperparameters[ "update_every_n_steps"] == 0 def sample_experiences(self): """Draws a random sample of experience from the memory buffer""" experiences = self.memory.sample() states, actions, rewards, next_states, dones = experiences return states, actions, rewards, next_states, dones def locally_save_policy(self, best=True, episode=None): if self.agent_name != "DQN": state = { 'episode': self.episode_number, 'q_network_local': self.q_network_local.state_dict(), 'q_network_target': self.q_network_target.state_dict() } else: state = { 'episode': self.episode_number, 'q_network_local': self.q_network_local.state_dict() } model_root = os.path.join('Models', self.config.env_title, self.agent_name, self.config.log_base) if not os.path.exists(model_root): os.makedirs(model_root) if best: last_best_file = glob.glob( os.path.join(model_root, 'rolling_score*')) if last_best_file: os.remove(last_best_file[0]) save_name = model_root + "/rolling_score_%.4f.model" % ( self.rolling_results[-1]) torch.save(state, save_name) self.logger.info('Model-%s save success...' % (save_name)) else: save_name = model_root + "/%s_%d.model" % (self.agent_name, self.episode_number) torch.save(state, save_name) self.logger.info('Model-%s save success...' % (save_name)) def load_resume(self, resume_path): save = torch.load(resume_path) if self.agent_name != "DQN": q_network_local_dict = save['q_network_local'] q_network_target_dict = save['q_network_target'] self.q_network_local.load_state_dict(q_network_local_dict, strict=True) self.q_network_target.load_state_dict(q_network_target_dict, strict=True) else: q_network_local_dict = save['q_network_local'] self.q_network_local.load_state_dict(q_network_local_dict, strict=True) self.logger.info('load resume model success...') file_name = os.path.basename(resume_path) episode_str = re.findall(r"\d+\.?\d*", file_name)[0] episode_list = episode_str.split('.') if not episode_list[1]: episode = episode_list[0] else: episode = 0 if not self.config.retrain: self.episode_number = episode else: self.episode_number = 0
class DDQN_Wrapper(Base_Agent): def __init__(self, config, global_action_id_to_primitive_actions, action_length_reward_bonus, end_of_episode_symbol="/"): super().__init__(config) self.end_of_episode_symbol = end_of_episode_symbol self.global_action_id_to_primitive_actions = global_action_id_to_primitive_actions self.memory = Replay_Buffer(self.hyperparameters["buffer_size"], self.hyperparameters["batch_size"], config.seed) self.exploration_strategy = Epsilon_Greedy_Exploration(config) self.oracle = self.create_oracle() self.oracle_optimizer = optim.Adam( self.oracle.parameters(), lr=self.hyperparameters["learning_rate"]) self.q_network_local = self.create_NN(input_dim=self.state_size + 1, output_dim=self.action_size) self.q_network_local.print_model_summary() self.q_network_optimizer = optim.Adam( self.q_network_local.parameters(), lr=self.hyperparameters["learning_rate"]) self.q_network_target = self.create_NN(input_dim=self.state_size + 1, output_dim=self.action_size) Base_Agent.copy_model_over(from_model=self.q_network_local, to_model=self.q_network_target) self.action_length_reward_bonus = action_length_reward_bonus self.abandon_ship = config.hyperparameters["abandon_ship"] def create_oracle(self): """Creates the network we will use to predict the next state""" oracle_hyperparameters = copy.deepcopy(self.hyperparameters) oracle_hyperparameters["columns_of_data_to_be_embedded"] = [] oracle_hyperparameters["embedding_dimensions"] = [] oracle_hyperparameters["linear_hidden_units"] = [5, 5] oracle_hyperparameters["final_layer_activation"] = [None, "tanh"] oracle = self.create_NN(input_dim=self.state_size + 2, output_dim=[self.state_size + 1, 1], hyperparameters=oracle_hyperparameters) oracle.print_model_summary() return oracle def run_n_episodes(self, num_episodes, episodes_to_run_with_no_exploration): self.turn_on_any_epsilon_greedy_exploration() self.round_of_macro_actions = [] self.episode_actions_scores_and_exploration_status = [] num_episodes_to_get_to = self.episode_number + num_episodes while self.episode_number < num_episodes_to_get_to: self.reset_game() self.step() self.save_and_print_result() if num_episodes_to_get_to - self.episode_number == episodes_to_run_with_no_exploration: self.turn_off_any_epsilon_greedy_exploration() assert len(self.episode_actions_scores_and_exploration_status ) == num_episodes, "{} vs. {}".format( len(self.episode_actions_scores_and_exploration_status), num_episodes) assert len(self.episode_actions_scores_and_exploration_status[0]) == 3 assert self.episode_actions_scores_and_exploration_status[0][2] in [ True, False ] assert isinstance( self.episode_actions_scores_and_exploration_status[0][1], list) assert isinstance( self.episode_actions_scores_and_exploration_status[0][1][0], int) assert isinstance( self.episode_actions_scores_and_exploration_status[0][0], int) or isinstance( self.episode_actions_scores_and_exploration_status[0][0], float) return self.episode_actions_scores_and_exploration_status, self.round_of_macro_actions def step(self): """Runs a step within a game including a learning step if required""" step_number = 0.0 self.state = np.append( self.state, step_number / 200.0) #Divide by 200 because there are 200 steps in cart pole self.total_episode_score_so_far = 0 episode_macro_actions = [] while not self.done: surprised = False macro_action = self.pick_action() primitive_actions = self.global_action_id_to_primitive_actions[ macro_action] primitive_actions_conducted = 0 for ix, action in enumerate(primitive_actions): if self.abandon_ship and primitive_actions_conducted > 0: if self.abandon_macro_action(action): break step_number += 1 self.action = action self.next_state, self.reward, self.done, _ = self.environment.step( action) self.next_state = np.append( self.next_state, step_number / 200.0 ) #Divide by 200 because there are 200 steps in cart pole self.total_episode_score_so_far += self.reward if self.hyperparameters["clip_rewards"]: self.reward = max(min(self.reward, 1.0), -1.0) primitive_actions_conducted += 1 self.track_episodes_data() self.save_experience() if len(primitive_actions) > 1: surprised = self.am_i_surprised() self.state = self.next_state if self.time_for_q_network_to_learn(): for _ in range( self.hyperparameters["learning_iterations"]): self.q_network_learn() self.oracle_learn() if self.done or surprised: break episode_macro_actions.append(macro_action) self.round_of_macro_actions.append(macro_action) if random.random() < 0.1: print(Counter(episode_macro_actions)) self.save_episode_actions_with_score() self.episode_number += 1 self.logger.info("END OF EPISODE") def am_i_surprised(self): """Returns boolean indicating whether the next_state was a surprise or not""" with torch.no_grad(): state = torch.from_numpy(self.state).float().unsqueeze(0).to( self.device) action = torch.Tensor([[self.action]]) states_and_actions = torch.cat( (state, action), dim=1) #must change this for all games besides cart pole predictions = self.oracle(states_and_actions) predicted_next_state = predictions[0, :-1] difference = F.mse_loss(predicted_next_state, torch.Tensor(self.next_state)) if difference > 0.5: print("Surprise! Loss {} -- {} vs. {}".format( difference, predicted_next_state, self.next_state)) return True else: return False def abandon_macro_action(self, action): """Returns boolean indicating whether to abandon macro action or not""" state = torch.from_numpy(self.state).float().unsqueeze(0).to( self.device) with torch.no_grad(): primitive_q_values = self.calculate_q_values( state, local=True, primitive_actions_only=True) q_value_highest = torch.max(primitive_q_values) q_values_action = primitive_q_values[:, action] if q_value_highest > 0.0: multiplier = 0.7 else: multiplier = 1.3 if q_values_action < multiplier * q_value_highest: print("BREAKING Action {} -- Q Values {}".format( action, primitive_q_values)) return True else: return False def pick_action(self, state=None): """Uses the local Q network and an epsilon greedy policy to pick an action""" if state is None: state = self.state if isinstance(state, np.int64) or isinstance(state, int): state = np.array([state]) state = torch.from_numpy(state).float().unsqueeze(0).to(self.device) if len(state.shape) < 2: state = state.unsqueeze(0) self.q_network_local.eval() #puts network in evaluation mode with torch.no_grad(): action_values = self.calculate_q_values( state, local=True, primitive_actions_only=False) self.q_network_local.train() #puts network back in training mode action = self.exploration_strategy.perturb_action_for_exploration_purposes( { "action_values": action_values, "turn_off_exploration": self.turn_off_exploration, "episode_number": self.episode_number }) self.logger.info("Q values {} -- Action chosen {}".format( action_values, action)) return action def calculate_q_values(self, states, local, primitive_actions_only): """Calculates the q values using the local q network""" if local: primitive_q_values = self.q_network_local(states) else: primitive_q_values = self.q_network_target(states) num_actions = len(self.global_action_id_to_primitive_actions) if primitive_actions_only or num_actions <= self.action_size: return primitive_q_values extra_q_values = self.calculate_macro_action_q_values( states, num_actions) extra_q_values = torch.Tensor([extra_q_values]) all_q_values = torch.cat((primitive_q_values, extra_q_values), dim=1) return all_q_values def calculate_macro_action_q_values(self, state, num_actions): assert state.shape[0] == 1 q_values = [] for action_id in range(self.action_size, num_actions): macro_action = self.global_action_id_to_primitive_actions[ action_id] predicted_next_state = state cumulated_reward = 0 action_ix = 0 for action in macro_action[:-1]: predictions = self.oracle( torch.cat((predicted_next_state, torch.Tensor([[action]])), dim=1)) rewards = predictions[:, -1] predicted_next_state = predictions[:, :-1] cumulated_reward += ( rewards.item() + self.action_length_reward_bonus ) * self.hyperparameters["discount_rate"]**(action_ix) action_ix += 1 final_action = macro_action[-1] final_q_value = self.q_network_local(predicted_next_state)[ 0, final_action] total_q_value = cumulated_reward + final_q_value * self.hyperparameters[ "discount_rate"]**(action_ix) q_values.append(total_q_value) return q_values def time_for_q_network_to_learn(self): """Returns boolean indicating whether enough steps have been taken for learning to begin and there are enough experiences in the replay buffer to learn from""" return self.right_amount_of_steps_taken( ) and self.enough_experiences_to_learn_from() def right_amount_of_steps_taken(self): """Returns boolean indicating whether enough steps have been taken for learning to begin""" return self.global_step_number % self.hyperparameters[ "update_every_n_steps"] == 0 def q_network_learn(self, experiences=None): """Runs a learning iteration for the Q network""" if experiences is None: states, actions, rewards, next_states, dones = self.sample_experiences( ) #Sample experiences else: states, actions, rewards, next_states, dones = experiences loss = self.compute_loss(states, next_states, rewards, actions, dones) self.take_optimisation_step( self.q_network_optimizer, self.q_network_local, loss, self.hyperparameters["gradient_clipping_norm"]) self.soft_update_of_target_network(self.q_network_local, self.q_network_target, self.hyperparameters["tau"]) def sample_experiences(self): """Draws a random sample of experience from the memory buffer""" experiences = self.memory.sample() states, actions, rewards, next_states, dones = experiences return states, actions, rewards, next_states, dones def compute_loss(self, states, next_states, rewards, actions, dones): """Computes the loss required to train the Q network""" with torch.no_grad(): max_action_indexes = self.calculate_q_values( next_states, local=True, primitive_actions_only=True).detach().argmax(1) Q_targets_next = self.calculate_q_values( next_states, local=False, primitive_actions_only=True).gather( 1, max_action_indexes.unsqueeze(1)) Q_targets = rewards + (self.hyperparameters["discount_rate"] * Q_targets_next * (1 - dones)) Q_expected = self.calculate_q_values( states, local=True, primitive_actions_only=True).gather(1, actions.long( )) # must convert actions to long so can be used as index loss = F.mse_loss(Q_expected, Q_targets) return loss def save_episode_actions_with_score(self): self.episode_actions_scores_and_exploration_status.append([ self.total_episode_score_so_far, self.episode_actions + [self.end_of_episode_symbol], self.turn_off_exploration ]) def oracle_learn(self): states, actions, rewards, next_states, _ = self.sample_experiences( ) # Sample experiences states_and_actions = torch.cat( (states, actions), dim=1) #must change this for all games besides cart pole predictions = self.oracle(states_and_actions) loss = F.mse_loss(torch.cat((next_states, rewards), dim=1), predictions) / float(next_states.shape[1] + 1.0) self.take_optimisation_step( self.oracle_optimizer, self.oracle, loss, self.hyperparameters["gradient_clipping_norm"]) self.logger.info("Oracle Loss {}".format(loss))
class SAC(Base_Agent): """Soft Actor-Critic model based on the 2018 paper https://arxiv.org/abs/1812.05905 and on this github implementation https://github.com/pranz24/pytorch-soft-actor-critic. It is an actor-critic algorithm where the agent is also trained to maximise the entropy of their actions as well as their cumulative reward""" agent_name = "SAC" def __init__(self, config): Base_Agent.__init__(self, config) assert self.action_types == "CONTINUOUS", "Action types must be continuous. Use SAC Discrete instead for discrete actions" assert self.config.hyperparameters["Actor"][ "final_layer_activation"] != "Softmax", "Final actor layer must not be softmax" self.hyperparameters = config.hyperparameters self.critic_local = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") self.critic_local_2 = self.create_NN( input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic", override_seed=self.config.seed + 1) self.critic_optimizer = torch.optim.Adam( self.critic_local.parameters(), lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4) self.critic_optimizer_2 = torch.optim.Adam( self.critic_local_2.parameters(), lr=self.hyperparameters["Critic"]["learning_rate"], eps=1e-4) self.critic_target = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") self.critic_target_2 = self.create_NN(input_dim=self.state_size + self.action_size, output_dim=1, key_to_use="Critic") Base_Agent.copy_model_over(self.critic_local, self.critic_target) Base_Agent.copy_model_over(self.critic_local_2, self.critic_target_2) self.memory = Replay_Buffer( self.hyperparameters["Critic"]["buffer_size"], self.hyperparameters["batch_size"], self.config.seed, device=self.device) self.actor_local = self.create_NN(input_dim=self.state_size, output_dim=self.action_size * 2, key_to_use="Actor") self.actor_optimizer = torch.optim.Adam( self.actor_local.parameters(), lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4) self.automatic_entropy_tuning = self.hyperparameters[ "automatically_tune_entropy_hyperparameter"] if self.automatic_entropy_tuning: self.target_entropy = -torch.prod( torch.Tensor(self.environment.action_space.shape).to( self.device)).item() # heuristic value from the paper self.log_alpha = torch.zeros(1, requires_grad=True, device=self.device) self.alpha = self.log_alpha.exp() self.alpha_optim = Adam( [self.log_alpha], lr=self.hyperparameters["Actor"]["learning_rate"], eps=1e-4) else: self.alpha = self.hyperparameters["entropy_term_weight"] self.add_extra_noise = self.hyperparameters["add_extra_noise"] if self.add_extra_noise: self.noise = OU_Noise(self.action_size, self.config.seed, self.hyperparameters["mu"], self.hyperparameters["theta"], self.hyperparameters["sigma"]) self.do_evaluation_iterations = self.hyperparameters[ "do_evaluation_iterations"] def save_result(self): """Saves the result of an episode of the game. Overriding the method in Base Agent that does this because we only want to keep track of the results during the evaluation episodes""" if self.episode_number == 1 or not self.do_evaluation_iterations: self.game_full_episode_scores.extend( [self.total_episode_score_so_far]) self.rolling_results.append( np.mean( self.game_full_episode_scores[-1 * self.rolling_score_window:])) self.save_max_result_seen() elif (self.episode_number - 1) % TRAINING_EPISODES_PER_EVAL_EPISODE == 0: self.game_full_episode_scores.extend([ self.total_episode_score_so_far for _ in range(TRAINING_EPISODES_PER_EVAL_EPISODE) ]) self.rolling_results.extend([ np.mean( self.game_full_episode_scores[-1 * self.rolling_score_window:]) for _ in range(TRAINING_EPISODES_PER_EVAL_EPISODE) ]) self.save_max_result_seen() def reset_game(self): """Resets the game information so we are ready to play a new episode""" Base_Agent.reset_game(self) if self.add_extra_noise: self.noise.reset() def step(self, isEval=False): if not isEval: """Runs an episode on the game, saving the experience and running a learning step if appropriate""" eval_ep = self.episode_number % TRAINING_EPISODES_PER_EVAL_EPISODE == 0 and self.do_evaluation_iterations self.episode_step_number_val = 0 while not self.done: self.episode_step_number_val += 1 self.action = self.pick_action(eval_ep) self.conduct_action(self.action) if self.time_for_critic_and_actor_to_learn(): for _ in range(self.hyperparameters[ "learning_updates_per_learning_session"]): self.learn() mask = False if self.episode_step_number_val >= 1000. else self.done if not eval_ep: self.save_experience(experience=(self.state, self.action, self.reward, self.next_state, mask)) self.state = self.next_state self.global_step_number += 1 print(self.total_episode_score_so_far) if eval_ep: self.print_summary_of_latest_evaluation_episode() if eval_ep: iou_test = self.environment.iou() # iou_test=self.iou(self.environment.environment_memory,np.argmax(self.environment.one_hot)+1) print('\nEpodise: ', self.episode_number, '| Ep_reward_test:', self.reward) print('\nEpodise: ', self.episode_number, '| Ep_IOU_test: ', iou_test) self.reward_history_test.append(self.reward) self.iou_history_test.append(iou_test) self.episode_number += 1 else: eval_ep = True ''' ########################3d dynamic test################################ print("Testing") plan_choose = self.environment.plan_choose print(f"PLAN = {plan_choose}") iou_all_average = 0 iou_all_min = 1 for test_set in range(10): env = deep_mobile_printing_3d1r(plan_choose=plan_choose, test_set=test_set) def iou(environment_memory,environment_plan,HALF_WINDOW_SIZE,plan_height,plan_width): component1=environment_plan[HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_height,\ HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_width].astype(bool) component2=environment_memory[HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_height,\ HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_width].astype(bool) overlap = component1*component2 # Logical AND union = component1 + component2 # Logical OR IOU = overlap.sum()/float(union.sum()) return IOU print(test_set) N_iteration_test = 200 best_iou = 0 iou_test_total = 0 iou_min = 1 reward_test_total = 0 start_time_test = time.time() fig = plt.figure(figsize=[10, 5]) ax1 = fig.add_subplot(1, 2, 1, projection='3d') ax2 = fig.add_subplot(1, 2, 2) for ep in range(N_iteration_test): obs = env.reset() reward_test = 0 self.state = obs while True: self.action = self.pick_action(eval_ep) self.conduct_action(self.action) obs, r, done = env.step(self.action) self.state = obs reward_test += r if done: break iou_test = iou(env.environment_memory,env.plan,env.HALF_WINDOW_SIZE,env.plan_height,env.plan_width) iou_min = min(iou_min, iou_test) if iou_test > best_iou: best_iou = iou_test best_plan = env.plan best_step = env.count_step best_brick = env.count_brick best_tb = env.total_brick best_env = env.environment_memory env.render(ax1, ax2) save_path = "plots/" plt.savefig(save_path+"SAC_Plan"+str(test_set)+'_'+str(self.environment.plan_choose)+'_good.png') iou_test_total += iou_test reward_test_total += reward_test reward_test_total = reward_test_total / N_iteration_test iou_test_total = iou_test_total / N_iteration_test secs = int(time.time() - start_time_test) mins = secs // 60 secs = secs % 60 print(f"time = {mins} min {secs} sec") print(f"iou = {iou_test_total}") print(f"reward_test = {reward_test_total}") if best_iou>0: env.render(ax1,ax2,iou_average=iou_test_total,iou_min=iou_min,iter_times=N_iteration_test,best_env=best_env,best_iou=best_iou,best_step=best_step,best_brick=best_brick) else: env.render(ax1,ax2,iou_average=iou_test_total,iou_min=iou_min,iter_times=N_iteration_test) save_path = "plots/" plt.savefig(save_path+"SAC_Plan"+str(test_set)+'_'+str(self.environment.plan_choose)+'_summary.png') iou_all_average += iou_test_total iou_all_min = min(iou_min,iou_all_min) iou_all_average = iou_all_average/10 print('iou_all_average',iou_all_average) print('iou_all_min',iou_all_min) ''' ''' ########################3d static test################################ print(f"Testing plan {self.environment.plan_choose}") env = deep_mobile_printing_3d1r(plan_choose=self.environment.plan_choose) def iou(environment_memory,environment_plan,HALF_WINDOW_SIZE,plan_height,plan_width): component1=environment_plan[HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_height,\ HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_width].astype(bool) component2=environment_memory[HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_height,\ HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_width].astype(bool) overlap = component1*component2 # Logical AND union = component1 + component2 # Logical OR IOU = overlap.sum()/float(union.sum()) return IOU N_iteration_test = 500 best_iou = 0 iou_test_total = 0 iou_min = 1 reward_test_total = 0 start_time_test = time.time() fig = plt.figure(figsize=[10, 5]) ax1 = fig.add_subplot(1, 2, 1, projection='3d') ax2 = fig.add_subplot(1, 2, 2) for ep in range(N_iteration_test): obs = env.reset() reward_test = 0 self.state = obs while True: self.action = self.pick_action(eval_ep) self.conduct_action(self.action) # action, _ = test_agent.predict(obs) obs, r, done = env.step(self.action) self.state = obs # action, _ = test_agent.predict(obs) # obs, r, done, info = env.step(action) reward_test += r if done: break iou_test = iou(env.environment_memory,env.plan,env.HALF_WINDOW_SIZE,env.plan_height,env.plan_width) iou_min = min(iou_min, iou_test) if iou_test > best_iou: best_iou = iou_test best_plan = env.plan best_step = env.count_step best_brick = env.count_brick best_tb = env.total_brick best_env = env.environment_memory env.render(ax1, ax2) save_path = "plots/" plt.savefig(save_path+"SAC_Plan"+'_'+str(self.environment.plan_choose)+'_good.png') iou_test_total += iou_test reward_test_total += reward_test reward_test_total = reward_test_total / N_iteration_test iou_test_total = iou_test_total / N_iteration_test secs = int(time.time() - start_time_test) mins = secs // 60 secs = secs % 60 print(f"time = {mins} min {secs} sec") print(f"iou = {iou_test_total}") print(f"reward_test = {reward_test_total}") if best_iou>0: env.render(ax1,ax2,iou_average=iou_test_total,iou_min=iou_min,iter_times=N_iteration_test,best_env=best_env,best_iou=best_iou,best_step=best_step,best_brick=best_brick) else: env.render(ax1,ax2,iou_average=iou_test_total,iou_min=iou_min,iter_times=N_iteration_test) save_path = "plots/" plt.savefig(save_path+"SAC_Plan"+'_'+str(self.environment.plan_choose)+'_summary.png') ''' # ########################2d dynamic test################################ # print("Testing") # print(f"PLAN = {self.environment.plan_choose}") # iou_all_average = 0 # iou_all_min = 1 # for test_set in range(10): # test_set = 6 # env = deep_mobile_printing_2d1r(plan_choose=self.environment.plan_choose, test_set=test_set) # def iou(environment_memory,environment_plan,HALF_WINDOW_SIZE,plan_height,plan_width): # component1=environment_plan[HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_height,\ # HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_width].astype(bool) # component2=environment_memory[HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_height,\ # HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_width].astype(bool) # overlap = component1*component2 # Logical AND # union = component1 + component2 # Logical OR # IOU = overlap.sum()/float(union.sum()) # return IOU # print(test_set) # N_iteration_test = 200 # best_iou = 0 # iou_test_total = 0 # iou_min = 1 # reward_test_total = 0 # start_time_test = time.time() # fig = plt.figure(figsize=(5, 5)) # ax = fig.add_subplot(1, 1, 1) # for ep in range(N_iteration_test): # obs = env.reset() # reward_test = 0 # self.state = obs # while True: # self.action = self.pick_action(eval_ep) # self.conduct_action(self.action) # # action, _ = test_agent.predict(obs) # obs, r, done, info = env.step(self.action) # self.state = obs # reward_test += r # if done: # break # iou_test = iou(env.environment_memory,env.plan,env.HALF_WINDOW_SIZE,env.plan_height,env.plan_width) # iou_min = min(iou_min, iou_test) # if iou_test > best_iou: # best_iou = iou_test # best_plan = env.plan # best_tb = env.total_brick # env.render(ax) # save_path = "plots/" # plt.savefig(save_path+"SAC_Plan"+str(test_set)+'_'+str(self.environment.plan_choose)+'_good.png') # iou_test_total += iou_test # reward_test_total += reward_test # reward_test_total = reward_test_total / N_iteration_test # iou_test_total = iou_test_total / N_iteration_test # secs = int(time.time() - start_time_test) # mins = secs // 60 # secs = secs % 60 # print(f"time = {mins} min {secs} sec") # print(f"iou = {iou_test_total}") # print(f"reward_test = {reward_test_total}") # env.render(ax,iou_average=iou_test_total,iou_min=iou_min,iter_times=N_iteration_test) # iou_all_average += iou_test_total # iou_all_min = min(iou_min,iou_all_min) # save_path = "plots/" # plt.savefig(save_path+"SAC_Plan"+str(test_set)+'_'+str(self.environment.plan_choose)+'_summary.png') # iou_all_average = iou_all_average/10 # print('iou_all_average',iou_all_average) # print('iou_all_min',iou_all_min) ''' ########################2d static test################################ def iou(environment_memory,environment_plan,HALF_WINDOW_SIZE,plan_height,plan_width): component1=environment_plan[HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_height,\ HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_width].astype(bool) component2=environment_memory[HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_height,\ HALF_WINDOW_SIZE:HALF_WINDOW_SIZE+plan_width].astype(bool) overlap = component1*component2 # Logical AND union = component1 + component2 # Logical OR IOU = overlap.sum()/float(union.sum()) return IOU env = deep_mobile_printing_2d1r(plan_choose=self.environment.plan_choose) N_iteration_test = 500 best_iou = 0 iou_test_total = 0 iou_min = 1 reward_test_total = 0 start_time_test = time.time() fig = plt.figure(figsize=(5, 5)) ax = fig.add_subplot(1, 1, 1) for ep in range(N_iteration_test): obs = env.reset() reward_test = 0 self.state = obs while True: self.action = self.pick_action(eval_ep) self.conduct_action(self.action) obs, r, done = env.step(self.action) self.state = obs # action, _ = test_agent.predict(obs) # obs, r, done, info = env.step(action) reward_test += r if done: break iou_test = iou(env.environment_memory,env.plan,env.HALF_WINDOW_SIZE,env.plan_height,env.plan_width) iou_min = min(iou_min, iou_test) if iou_test > best_iou: best_iou = iou_test best_plan = env.plan best_tb = env.total_brick env.render(ax) save_path = "plots/" plt.savefig(save_path+"SAC_Plan"+str(self.environment.plan_choose)+'_good.png') iou_test_total += iou_test reward_test_total += reward_test reward_test_total = reward_test_total / N_iteration_test iou_test_total = iou_test_total / N_iteration_test secs = int(time.time() - start_time_test) mins = secs // 60 secs = secs % 60 print(f"time = {mins} min {secs} sec") print(f"iou = {iou_test_total}") print(f"reward_test = {reward_test_total}") env.render(ax,iou_average=iou_test_total,iou_min=iou_min,iter_times=N_iteration_test) save_path = "plots/" plt.savefig(save_path+"SAC_Plan"+str(self.environment.plan_choose)+'_summary.png') ''' ''' ########################1d dynamic test################################ iou_all_average = 0 iou_all_min = 1 for plan_choose in range(10): plan_choose = 8 print(plan_choose) env = deep_mobile_printing_1d1r(plan_choose=plan_choose) N_iteration_test = 200 best_iou = 0 iou_test_total = 0 iou_min = 1 reward_test_total = 0 start_time_test = time.time() fig = plt.figure(figsize=(5, 5)) ax = fig.add_subplot(1, 1, 1) for ep in range(N_iteration_test): obs = env.reset() reward_test = 0 self.state = obs while True: self.action = self.pick_action(eval_ep) self.conduct_action(self.action) obs, r, done = env.step(self.action) self.state = obs reward_test += r if done: break iou_test = env.iou() iou_min = min(iou_min, iou_test) if iou_test > best_iou: best_iou = iou_test best_plan = env.plan best_tb = env.total_brick env.render(ax) save_path = "plots/" plt.savefig(save_path+"SAC_Plan"+str(plan_choose)+'_good.png') iou_test_total += iou_test reward_test_total += reward_test reward_test_total = reward_test_total / N_iteration_test iou_test_total = iou_test_total / N_iteration_test secs = int(time.time() - start_time_test) mins = secs // 60 secs = secs % 60 print(f"time = {mins} min {secs} sec") print(f"iou = {iou_test_total}") print(f"reward_test = {reward_test_total}") env.render(ax,iou_average=iou_test_total,iou_min=iou_min,iter_times=N_iteration_test) iou_all_average += iou_test_total iou_all_min = min(iou_min,iou_all_min) save_path = "plots/" plt.savefig(save_path+"SAC_Plan"+str(plan_choose)+'_summary.png') iou_all_average = iou_all_average/10 print('iou_all_average',iou_all_average) print('iou_all_min',iou_all_min) ''' ########################1d static test################################ plan = self.environment.plan_choose print(f"Testing plan {plan}") N_iteration_test = 1 best_iou = 0 iou_test_total = 0 iou_min = 1 reward_test_total = 0 start_time_test = time.time() fig = plt.figure(figsize=(5, 5)) ax = fig.add_subplot(1, 1, 1) for ep in range(N_iteration_test): obs = self.environment.reset() reward_test = 0 while True: self.action = self.pick_action(eval_ep) self.conduct_action(self.action) self.state = self.next_state obs, r, done = self.environment.step(self.action) reward_test += r if done: break # self.environment.render(ax) # plt.show() iou_test = self.environment.iou() iou_min = min(iou_min, iou_test) if iou_test > best_iou: best_iou = iou_test best_plan = self.environment.plan best_tb = self.environment.total_brick self.environment.render(ax) save_path = self.config.save_model_path plt.savefig(save_path + "SAC_Plan" + str(plan) + '_good.png') iou_test_total += iou_test reward_test_total += reward_test reward_test_total = reward_test_total / N_iteration_test iou_test_total = iou_test_total / N_iteration_test secs = int(time.time() - start_time_test) mins = secs // 60 secs = secs % 60 print(f"time = {mins} min {secs} sec") print(f"iou = {iou_test_total}") print(f"reward_test = {reward_test_total}") self.environment.render(ax, iou_average=iou_test_total, iou_min=iou_min, iter_times=N_iteration_test) save_path = self.config.save_model_path plt.savefig(save_path + "SAC_Plan" + str(plan) + '_summary.png') ########################Initial test################################ # fig = plt.figure(figsize=(5, 5)) # ax = fig.add_subplot(1, 1, 1) # print(self.environment.total_brick) # print(self.environment.one_hot) # step = self.environment.total_step # ax.clear() # while not self.done: # self.action = self.pick_action(eval_ep) # self.conduct_action(self.action) # self.state = self.next_state # self.environment.render(ax) # self.environment.iou() # plt.pause(1e-6) # plt.show() exit() def pick_action(self, eval_ep, state=None): """Picks an action using one of three methods: 1) Randomly if we haven't passed a certain number of steps, 2) Using the actor in evaluation mode if eval_ep is True 3) Using the actor in training mode if eval_ep is False. The difference between evaluation and training mode is that training mode does more exploration""" if state is None: state = self.state # print(self.state) if eval_ep: action = self.actor_pick_action(state=state, eval=True) elif self.global_step_number < self.hyperparameters[ "min_steps_before_learning"]: # print('pick_action') action = np.random.randint(0, self.environment.action_dim) #action = self.environment.action_space.sample() print("Picking random action ", action) else: action = self.actor_pick_action(state=state) if self.add_extra_noise: action += self.noise.sample() return action def actor_pick_action(self, state=None, eval=False): """Uses actor to pick an action in one of two ways: 1) If eval = False and we aren't in eval mode then it picks an action that has partly been randomly sampled 2) If eval = True then we pick the action that comes directly from the network and so did not involve any random sampling""" if state is None: state = self.state state = torch.FloatTensor([state]).to(self.device) if len(state.shape) == 1: state = state.unsqueeze(0) if eval == False: action, _, _ = self.produce_action_and_action_info(state) else: with torch.no_grad(): _, z, action = self.produce_action_and_action_info(state) action = action.detach().cpu().numpy() return action[0] def produce_action_and_action_info(self, state): """Given the state, produces an action, the log probability of the action, and the tanh of the mean action""" actor_output = self.actor_local(state) mean, log_std = actor_output[:, :self. action_size], actor_output[:, self. action_size:] std = log_std.exp() normal = Normal(mean, std) x_t = normal.rsample( ) #rsample means it is sampled using reparameterisation trick action = torch.tanh(x_t) log_prob = normal.log_prob(x_t) log_prob -= torch.log(1 - action.pow(2) + EPSILON) log_prob = log_prob.sum(1, keepdim=True) return action, log_prob, torch.tanh(mean) def time_for_critic_and_actor_to_learn(self): """Returns boolean indicating whether there are enough experiences to learn from and it is time to learn for the actor and critic""" return self.global_step_number > self.hyperparameters["min_steps_before_learning"] and \ self.enough_experiences_to_learn_from() and self.global_step_number % self.hyperparameters["update_every_n_steps"] == 0 def learn(self): """Runs a learning iteration for the actor, both critics and (if specified) the temperature parameter""" state_batch, action_batch, reward_batch, next_state_batch, mask_batch = self.sample_experiences( ) qf1_loss, qf2_loss = self.calculate_critic_losses( state_batch, action_batch, reward_batch, next_state_batch, mask_batch) policy_loss, log_pi = self.calculate_actor_loss(state_batch) if self.automatic_entropy_tuning: alpha_loss = self.calculate_entropy_tuning_loss(log_pi) else: alpha_loss = None self.update_all_parameters(qf1_loss, qf2_loss, policy_loss, alpha_loss) def sample_experiences(self): return self.memory.sample() def calculate_critic_losses(self, state_batch, action_batch, reward_batch, next_state_batch, mask_batch): """Calculates the losses for the two critics. This is the ordinary Q-learning loss except the additional entropy term is taken into account""" with torch.no_grad(): next_state_action, next_state_log_pi, _ = self.produce_action_and_action_info( next_state_batch) qf1_next_target = self.critic_target( torch.cat((next_state_batch, next_state_action), 1)) qf2_next_target = self.critic_target_2( torch.cat((next_state_batch, next_state_action), 1)) min_qf_next_target = torch.min( qf1_next_target, qf2_next_target) - self.alpha * next_state_log_pi next_q_value = reward_batch + ( 1.0 - mask_batch) * self.hyperparameters["discount_rate"] * ( min_qf_next_target) qf1 = self.critic_local(torch.cat((state_batch, action_batch), 1)) qf2 = self.critic_local_2(torch.cat((state_batch, action_batch), 1)) qf1_loss = F.mse_loss(qf1, next_q_value) qf2_loss = F.mse_loss(qf2, next_q_value) return qf1_loss, qf2_loss def calculate_actor_loss(self, state_batch): """Calculates the loss for the actor. This loss includes the additional entropy term""" action, log_pi, _ = self.produce_action_and_action_info(state_batch) qf1_pi = self.critic_local(torch.cat((state_batch, action), 1)) qf2_pi = self.critic_local_2(torch.cat((state_batch, action), 1)) min_qf_pi = torch.min(qf1_pi, qf2_pi) policy_loss = ((self.alpha * log_pi) - min_qf_pi).mean() return policy_loss, log_pi def calculate_entropy_tuning_loss(self, log_pi): """Calculates the loss for the entropy temperature parameter. This is only relevant if self.automatic_entropy_tuning is True.""" alpha_loss = -(self.log_alpha * (log_pi + self.target_entropy).detach()).mean() return alpha_loss def update_all_parameters(self, critic_loss_1, critic_loss_2, actor_loss, alpha_loss): """Updates the parameters for the actor, both critics and (if specified) the temperature parameter""" self.take_optimisation_step( self.critic_optimizer, self.critic_local, critic_loss_1, self.hyperparameters["Critic"]["gradient_clipping_norm"]) self.take_optimisation_step( self.critic_optimizer_2, self.critic_local_2, critic_loss_2, self.hyperparameters["Critic"]["gradient_clipping_norm"]) self.take_optimisation_step( self.actor_optimizer, self.actor_local, actor_loss, self.hyperparameters["Actor"]["gradient_clipping_norm"]) self.soft_update_of_target_network( self.critic_local, self.critic_target, self.hyperparameters["Critic"]["tau"]) self.soft_update_of_target_network( self.critic_local_2, self.critic_target_2, self.hyperparameters["Critic"]["tau"]) if alpha_loss is not None: self.take_optimisation_step(self.alpha_optim, None, alpha_loss, None) self.alpha = self.log_alpha.exp() def print_summary_of_latest_evaluation_episode(self): """Prints a summary of the latest episode""" print(" ") print("----------------------------") print("Episode score {} ".format(self.total_episode_score_so_far)) print("----------------------------")