class AgentDDPG(AgentBase): def __init__(self, net_dim, state_dim, action_dim, learning_rate=1e-4): super().__init__() self.explore_noise = 0.05 # explore noise of action self.act = Actor(net_dim, state_dim, action_dim).to(self.device) self.act_target = deepcopy(self.act) self.cri = Critic(net_dim, state_dim, action_dim).to(self.device) self.cri_target = deepcopy(self.cri) self.criterion = torch.nn.SmoothL1Loss() self.optimizer = torch.optim.Adam([{ 'params': self.act.parameters(), 'lr': learning_rate }, { 'params': self.cri.parameters(), 'lr': learning_rate }]) def select_actions(self, states): # states = (state, ...) states = torch.as_tensor(states, dtype=torch.float32, device=self.device) actions = self.act(states) actions = (actions + torch.randn_like(actions) * self.explore_noise).clamp( -1, 1) return actions.detach().cpu().numpy() def update_policy(self, buffer, max_step, batch_size, repeat_times): buffer.update__now_len__before_sample() obj_critic = obj_actor = None # just for print return for _ in range(int(max_step * repeat_times)): with torch.no_grad(): reward, mask, action, state, next_s = buffer.random_sample( batch_size) next_q = self.cri_target(next_s, self.act_target(next_s)) q_label = reward + mask * next_q q_value = self.cri(state, action) obj_critic = self.criterion(q_value, q_label) q_value_pg = self.act(state) # policy gradient obj_actor = -self.cri_target(state, q_value_pg).mean() obj_united = obj_actor + obj_critic # objective self.optimizer.zero_grad() obj_united.backward() self.optimizer.step() soft_target_update(self.cri_target, self.cri) soft_target_update(self.act_target, self.act) return obj_actor.item(), obj_critic.item()
class AgentDDPG(AgentBase): def __init__(self): super().__init__() self.ou_explore_noise = 0.3 # explore noise of action self.ou_noise = None def init(self, net_dim, state_dim, action_dim, if_per=False): self.ou_noise = OrnsteinUhlenbeckNoise(size=action_dim, sigma=self.ou_explore_noise) # I don't recommend to use OU-Noise self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.cri = Critic(net_dim, state_dim, action_dim).to(self.device) self.cri_target = deepcopy(self.cri) self.cri_optimizer = torch.optim.Adam(self.cri.parameters(), lr=self.learning_rate) self.act = Actor(net_dim, state_dim, action_dim).to(self.device) self.act_target = deepcopy(self.act) self.act_optimizer = torch.optim.Adam(self.act.parameters(), lr=self.learning_rate) self.criterion = torch.nn.SmoothL1Loss(reduction='none' if if_per else 'mean') if if_per: self.get_obj_critic = self.get_obj_critic_per else: self.get_obj_critic = self.get_obj_critic_raw def select_action(self, state) -> np.ndarray: states = torch.as_tensor((state,), dtype=torch.float32, device=self.device).detach_() action = self.act(states)[0].cpu().numpy() return (action + self.ou_noise()).clip(-1, 1) def update_net(self, buffer, target_step, batch_size, repeat_times) -> (float, float): buffer.update_now_len_before_sample() obj_critic = obj_actor = None # just for print return for _ in range(int(target_step * repeat_times)): obj_critic, state = self.get_obj_critic(buffer, batch_size) self.cri_optimizer.zero_grad() obj_critic.backward() self.cri_optimizer.step() self.soft_update(self.cri_target, self.cri, self.soft_update_tau) q_value_pg = self.act(state) # policy gradient obj_actor = -self.cri_target(state, q_value_pg).mean() # obj_actor self.act_optimizer.zero_grad() obj_actor.backward() self.act_optimizer.step() self.soft_update(self.act_target, self.act, self.soft_update_tau) return obj_actor.item(), obj_critic.item() def get_obj_critic_raw(self, buffer, batch_size): with torch.no_grad(): reward, mask, action, state, next_s = buffer.sample_batch(batch_size) next_q = self.cri_target(next_s, self.act_target(next_s)) q_label = reward + mask * next_q q_value = self.cri(state, action) obj_critic = self.criterion(q_value, q_label) return obj_critic, state def get_obj_critic_per(self, buffer, batch_size): with torch.no_grad(): reward, mask, action, state, next_s, is_weights = buffer.sample_batch(batch_size) next_q = self.cri_target(next_s, self.act_target(next_s)) q_label = reward + mask * next_q q_value = self.cri(state, action) obj_critic = (self.criterion(q_value, q_label) * is_weights).mean() td_error = (q_label - q_value.detach()).abs() buffer.td_error_update(td_error) return obj_critic, state