class ACAgent(BaseAgent): def __init__(self, env, agent_params): super().__init__() self.env = env self.agent_params = agent_params self.gamma = self.agent_params['gamma'] self.standardize_advantages = self.agent_params[ 'standardize_advantages'] self.actor = MLPPolicyAC( self.agent_params['ac_dim'], self.agent_params['ob_dim'], self.agent_params['n_layers'], self.agent_params['size'], self.agent_params['discrete'], self.agent_params['learning_rate'], ) self.critic = BootstrappedContinuousCritic(self.agent_params) self.replay_buffer = ReplayBuffer() def train(self, ob_no, ac_na, re_n, next_ob_no, terminal_n): # TODO Implement the following pseudocode: # for agent_params['num_critic_updates_per_agent_update'] steps, # update the critic ob_no = ptu.from_numpy(ob_no) next_ob_no = ptu.from_numpy(next_ob_no) re_n = ptu.from_numpy(re_n) ac_na = ptu.from_numpy(ac_na) terminal_n = ptu.from_numpy(terminal_n) for _ in range( self.agent_params['num_critic_updates_per_agent_update']): critic_loss = self.critic.update(ob_no, ac_na, next_ob_no, re_n, terminal_n) # advantage = estimate_advantage(...) advantage = self.estimate_advantage(ob_no, next_ob_no, re_n, terminal_n) # for agent_params['num_actor_updates_per_agent_update'] steps, # update the actor for _ in range( self.agent_params['num_actor_updates_per_agent_update']): actor_loss = self.actor.update(ob_no, ac_na, advantage) loss = OrderedDict() loss['Critic_Loss'] = critic_loss loss['Actor_Loss'] = actor_loss return loss def estimate_advantage(self, ob_no, next_ob_no, re_n, terminal_n): # TODO Implement the following pseudocode: # 1) query the critic with ob_no, to get V(s) # 2) query the critic with next_ob_no, to get V(s') # 3) estimate the Q value as Q(s, a) = r(s, a) + gamma*V(s') # HINT: Remember to cut off the V(s') term (ie set it to 0) at terminal states (ie terminal_n=1) # 4) calculate advantage (adv_n) as A(s, a) = Q(s, a) - V(s) # ob_no = torch.from_numpy(ob_no) # next_ob_no = torch.from_numpy(next_ob_no) # re_n = torch.from_numpy(re_n) q = re_n + self.gamma * self.critic.forward(next_ob_no) * (1 - terminal_n) adv_n = q - self.critic.forward(ob_no) adv_n = adv_n.detach().cpu().numpy() # v = self.critic(ob_no) # v_next = self.critic(next_ob_no)*(1-terminal_n) # q = re_n + self.gamma*v_next # adv_n = q - v # adv_n = adv_n.detach().cpu().numpy() if self.standardize_advantages: adv_n = (adv_n - np.mean(adv_n)) / (np.std(adv_n) + 1e-8) return adv_n def add_to_replay_buffer(self, paths): self.replay_buffer.add_rollouts(paths) def sample(self, batch_size): return self.replay_buffer.sample_recent_data(batch_size)
class ACAgent(BaseAgent): def __init__(self, sess, env, agent_params): super(ACAgent, self).__init__() self.env = env self.sess = sess self.agent_params = agent_params self.gamma = self.agent_params['gamma'] self.standardize_advantages = self.agent_params['standardize_advantages'] self.actor = MLPPolicyAC(sess, self.agent_params['ac_dim'], self.agent_params['ob_dim'], self.agent_params['n_layers'], self.agent_params['size'], discrete=self.agent_params['discrete'], learning_rate=self.agent_params['learning_rate'], ) self.critic = BootstrappedContinuousCritic(sess, self.agent_params) self.replay_buffer = ReplayBuffer() def estimate_advantage(self, ob_no, next_ob_no, re_n, terminal_n): # TODO Implement the following pseudocode: # 1) query the critic with ob_no, to get V(s) # 2) query the critic with next_ob_no, to get V(s') # 3) estimate the Q value as Q(s, a) = r(s, a) + gamma*V(s') # HINT: Remember to cut off the V(s') term (ie set it to 0) at terminal states (ie terminal_n=1) # 4) calculate advantage (adv_n) as A(s, a) = Q(s, a) - V(s) V_s, V_s1 = self.critic.forward(ob_no), self.critic.forward(next_ob_no) Q_value = re_n + self.gamma * V_s1 * (1 - terminal_n) adv_n = Q_value - V_s if self.standardize_advantages: adv_n = (adv_n - np.mean(adv_n)) / (np.std(adv_n) + 1e-8) return adv_n def train(self, ob_no, ac_na, re_n, next_ob_no, terminal_n): # TODO Implement the following pseudocode: # for agent_params['num_critic_updates_per_agent_update'] steps, # update the critic # advantage = estimate_advantage(...) # for agent_params['num_actor_updates_per_agent_update'] steps, # update the actor for iter in range(self.agent_params['num_critic_updates_per_agent_update']): critic_loss = self.critic.update(ob_no, next_ob_no, re_n, terminal_n) advantage = self.estimate_advantage(ob_no, next_ob_no, re_n, terminal_n) for iter in range(self.agent_params['num_actor_updates_per_agent_update']): actor_loss = self.actor.update(ob_no, ac_na, advantage) # print('critic_loss: {}, actor_loss: {}'.format(critic_loss, actor_loss)) loss = OrderedDict() loss['Critic_Loss'] = critic_loss # put final critic loss here loss['Actor_Loss'] = actor_loss # put final actor loss here return loss def add_to_replay_buffer(self, paths): self.replay_buffer.add_rollouts(paths) def sample(self, batch_size): return self.replay_buffer.sample_recent_data(batch_size)
class ACAgent(BaseAgent): def __init__(self, env, agent_params): super(ACAgent, self).__init__() self.env = env self.agent_params = agent_params self.gamma = self.agent_params['gamma'] self.standardize_advantages = self.agent_params[ 'standardize_advantages'] self.actor = MLPPolicyAC( self.agent_params['ac_dim'], self.agent_params['ob_dim'], self.agent_params['n_layers'], self.agent_params['size'], self.agent_params['discrete'], self.agent_params['learning_rate'], ) self.critic = BootstrappedContinuousCritic(self.agent_params) self.replay_buffer = ReplayBuffer() def train(self, ob_no, ac_na, re_n, next_ob_no, terminal_n): ob_no = ptu.from_numpy(ob_no) next_ob_no = ptu.from_numpy(next_ob_no) terminal_n = ptu.from_numpy(terminal_n) re_n = ptu.from_numpy(re_n) ac_na = ptu.from_numpy(ac_na) loss_critic = 0. for i in range( self.agent_params['num_critic_updates_per_agent_update']): loss_critic += self.critic.update(ob_no, ac_na, next_ob_no, re_n, terminal_n) # advantage = estimate_advantage(...) : adv_n = self.estimate_advantage(ob_no, next_ob_no, re_n, terminal_n) # a tensor is returned loss_actor = 0. for i in range( self.agent_params['num_actor_updates_per_agent_update']): loss_actor += self.actor.update(ob_no, ac_na, adv_n) loss = OrderedDict() loss['Critic_Loss'] = loss_critic loss[ 'Actor_Loss'] = loss_actor # in TensorBoard, loss_actor actually increases as we actually minimize -loss_actor return loss def estimate_advantage(self, ob_no, next_ob_no, re_n, terminal_n): # TODO Implement the following pseudocode: # 1) query the critic with ob_no, to get V(s) # 2) query the critic with next_ob_no, to get V(s') # 3) estimate the Q value as Q(s, a) = r(s, a) + gamma*V(s') # HINT: Remember to cut off the V(s') term (ie set it to 0) at terminal states (ie terminal_n=1) # 4) calculate advantage (adv_n) as A(s, a) = Q(s, a) - V(s) # V_s_prime = self.critic.critic_network(next_ob_no) # V_s_prime = V_s_prime.squeeze() # mask = (terminal_n == 1.) # V_s_prime= V_s_prime.masked_fill(mask, 0.) # # V_s = self.critic.critic_network(ob_no) # V_s = V_s.squeeze() # # assert V_s_prime.ndim == V_s.ndim # TODO-assert enable this assert in debug # adv_n2 = re_n + self.gamma * V_s_prime - V_s # another way to calculate: V_s_prime = re_n + ( 1 - terminal_n) * self.gamma * self.critic.forward(next_ob_no) adv_n = V_s_prime - self.critic.forward(ob_no) # assert adv_n2 == adv_n if self.standardize_advantages: adv_n = (adv_n - torch.mean(adv_n)) / (torch.std(adv_n) + 1e-8) return adv_n def add_to_replay_buffer(self, paths): self.replay_buffer.add_rollouts(paths) def sample(self, batch_size): return self.replay_buffer.sample_recent_data(batch_size)