class Agent(): """Interacts with and learns from the environment.""" memory = None actor_local = None actor_target = None actor_optimizer = None critic_local = None critic_target = None critic_optimizer = None instances = [] def __init__(self, state_size, action_size, random_seed): """Initialize an Agent object. Params ====== state_size (int): dimension of each state action_size (int): dimension of each action random_seed (int): random seed """ self.state_size = state_size self.action_size = action_size self.seed = random.seed(random_seed) # Actor Network (w/ Target Network) self.actor_local = Actor(state_size, action_size, random_seed).to(device) self.actor_target = Actor(state_size, action_size, random_seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR) # initialize Class level Actor Network # if Agent.actor_local is None: # Agent.actor_local = Actor(state_size, action_size, random_seed).to(device) # if Agent.actor_target is None: # Agent.actor_target = Actor(state_size, action_size, random_seed).to(device) # if Agent.actor_optimizer is None: # Agent.actor_optimizer = optim.Adam(Agent.actor_local.parameters(), lr=LR_ACTOR) # self.actor_local = Agent.actor_local # self.actor_target = Agent.actor_target # self.actor_optimizer = Agent.actor_optimizer # Critic Network (w/ Target Network) # self.critic_local = Critic(state_size, action_size, random_seed).to(device) # self.critic_target = Critic(state_size, action_size, random_seed).to(device) # self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) # Initilise Class levell Critic Network if Agent.critic_local is None: Agent.critic_local = Critic(state_size, action_size, random_seed).to(device) if Agent.critic_target is None: Agent.critic_target = Critic(state_size, action_size, random_seed).to(device) if Agent.critic_optimizer is None: Agent.critic_optimizer = optim.Adam( Agent.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) self.critic_local = Agent.critic_local self.critic_target = Agent.critic_target self.critic_optimizer = Agent.critic_optimizer # Noise process self.noise = OUNoise(action_size, random_seed) # Replay memory - only intitialise once per class if Agent.memory is None: print("Initialising ReplayBuffer") Agent.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed) # else: # print("Sharing ReplayBuffer %s", Agent.memory) # Add this instances - we need to access all agent states whilst learning self.agent_num = len(Agent.instances) Agent.instances.append(self) print("Appended to Agent.instances agent {}".format(self.agent_num)) def step(self, time_step, state, action, reward, next_state, done): """Save experience in replay memory, and use random sample from buffer to learn.""" # Save experience / reward Agent.memory.add(state, action, reward, next_state, done) # only learn every n_time_steps if time_step % N_TIME_STEPS != 0: return # Learn, if enough samples are available in memory if len(Agent.memory) > BATCH_SIZE: for i in range(N_LEARN_UPDATES): experiences = Agent.memory.sample() self.learn(experiences, GAMMA) def act(self, state, add_noise=True, noise_amplitude=0.0): """Returns actions for given state as per current policy.""" state = torch.from_numpy(state).float().to(device) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() if add_noise: action += self.noise.sample() * noise_amplitude return np.clip(action, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences, gamma): """Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Params ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor """ states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # # Get predicted next-state actions and Q values from target models actions_next = self.actor_target(next_states) Q_targets_next = self.critic_target(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)) # Compute critic loss Q_expected = self.critic_local(states, actions) critic_loss = F.mse_loss(Q_expected, Q_targets) # Minimize the loss self.critic_optimizer.zero_grad() critic_loss.backward() torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1) self.critic_optimizer.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred = self.actor_local(states) actor_loss = -self.critic_local(states, actions_pred).mean() # Minimize the loss self.actor_optimizer.zero_grad() actor_loss.backward() self.actor_optimizer.step() # ----------------------- update target networks ----------------------- # self.soft_update(self.critic_local, self.critic_target, TAU) self.soft_update(self.actor_local, self.actor_target, TAU) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model: PyTorch model (weights will be copied from) target_model: PyTorch model (weights will be copied to) tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class DDPGAgent(): """DDPG agent that interacts with and learns from the environment. The agents model is implemented in 'ddpg_model.py'. It consists of two neural networks; one for the actor, and one for the critic. The DDPGAgent class makes use of two other classes: ReplayBuffer, OUNoise """ def __init__(self, state_size, action_size, num_agents, random_seed): """Initialize an Agent object. Arguments: state_size (int) -- dimension of each state action_size (int) -- dimension of each action num_agents (int) -- number of agents (brains) random_seed (int) -- random seed """ self.state_size = state_size self.action_size = action_size self.num_agents = num_agents self.seed = random.seed(random_seed) ### Make neural networks (local and target) for both actor and critic, and set optimizers # Actor Network (w/ Target Network) self.actor_local = Actor(state_size, action_size, random_seed).to(device) self.actor_target = Actor(state_size, action_size, random_seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR) # Critic Network (w/ Target Network) self.critic_local = Critic(state_size, action_size, random_seed).to(device) self.critic_target = Critic(state_size, action_size, random_seed).to(device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) # Noise process self.noise = OUNoise((num_agents, action_size), random_seed) # Initialize replay memory ### self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed) def step(self, state, action, reward, next_state, done, timestep): """Save experience in replay memory, and use random sample from buffer to learn.""" # Save experience in memory for i in range(self.num_agents): self.memory.add(state[i, :], action[i, :], reward[i], next_state[i, :], done[i]) # Learn every UPDATE_EVERY time steps if timestep % UPDATE_EVERY == 0: # If we have collected enough experience in our memory i.e. more # than the mini-batch size, then call the self.learn() function if len(self.memory) > BATCH_SIZE: # Number of updates per timestep for _ in range(NUM_UPDATES): experiences = self.memory.sample() self.learn(experiences, GAMMA) def act(self, state, add_noise=True): """Returns actions for given state as per current policy. Arguments: state {[type]} -- Current state add_noise {bool} -- Add noise (exploration) to the actions (default: {True}) Returns: [float] -- Actions """ # Convert 'state' numpy array to pytorch tensor using the current device # i.e. GPU or CPU. state = torch.from_numpy(state).float().to(device) # Set the module in evaluation mode. self.actor_local.eval() with torch.no_grad(): # Evaluate the network with the current state action = self.actor_local(state).cpu().data.numpy() # Set the module in training mode. self.actor_local.train() if add_noise: # Add noise to the actions to add exploration action += self.noise.sample() # Return the clipped actions return np.clip(action, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences, gamma): """Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Arguments: experiences {Tuple[torch.Tensor]} -- tuple of (s, a, r, s', done) tuples gamma {float} -- discount factor """ # Experiences, mini-batch of 128 states, actions, rewards, next_states, dones = experiences # ---------------------------- Update critic ---------------------------- # # Get predicted next-state actions and Q values from target models actions_next = self.actor_target(next_states) Q_targets_next = self.critic_target(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)) # Compute critic loss Q_expected = self.critic_local(states, actions) critic_loss = F.mse_loss(Q_expected, Q_targets) # Minimize the loss self.critic_optimizer.zero_grad() critic_loss.backward() # Clip the gradients torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1) # Take one step with the optimizer self.critic_optimizer.step() # ---------------------------- Update actor ---------------------------- # # Compute actor loss actions_pred = self.actor_local(states) actor_loss = -self.critic_local(states, actions_pred).mean() # Minimize the loss self.actor_optimizer.zero_grad() actor_loss.backward() self.actor_optimizer.step() # ----------------------- Update target networks ----------------------- # self.soft_update(self.critic_local, self.critic_target, TAU) self.soft_update(self.actor_local, self.actor_target, TAU) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Arguments: local_model -- PyTorch model (weights will be copied from) target_model -- PyTorch model (weights will be copied to) tau (float) -- interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class DDPG_Agent(): """Interacts with and learns from the environment.""" def __init__(self, state_size, action_size, random_seed): """Initialize an Agent object. Params ====== state_size (int): dimension of each state action_size (int): dimension of each action random_seed (int): random seed """ self.state_size = state_size self.action_size = action_size self.seed = random.seed(random_seed) # Actor Network (w/ Target Network) self.actor_local = Actor(state_size, action_size, random_seed).to(device) self.actor_target = Actor(state_size, action_size, random_seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR) # Critic Network (w/ Target Network) self.critic_local = Critic(state_size, action_size, random_seed).to(device) self.critic_target = Critic(state_size, action_size, random_seed).to(device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) # Noise process self.noise = OUNoise(action_size, random_seed) # Replay memory self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed) def step(self, state, action, reward, next_state, done): """Save experience in replay memory, and use random sample from buffer to learn.""" # Save experience / reward self.memory.add(state, action, reward, next_state, done) # Learn, if enough samples are available in memory if len(self.memory) > BATCH_SIZE: experiences = self.memory.sample() self.learn(experiences, GAMMA) def act(self, state, add_noise=True): """Returns actions for given state as per current policy.""" state = torch.from_numpy(state).float().to(device) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() if add_noise: action += self.noise.sample() return np.clip(action, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences, gamma): """Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Params ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor """ states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # # Get predicted next-state actions and Q values from target models actions_next = self.actor_target(next_states) Q_targets_next = self.critic_target(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)) # Compute critic loss Q_expected = self.critic_local(states, actions) critic_loss = F.mse_loss(Q_expected, Q_targets) # Minimize the loss self.critic_optimizer.zero_grad() critic_loss.backward() self.critic_optimizer.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred = self.actor_local(states) actor_loss = -self.critic_local(states, actions_pred).mean() # Minimize the loss self.actor_optimizer.zero_grad() actor_loss.backward() self.actor_optimizer.step() # ----------------------- update target networks ----------------------- # self.soft_update(self.critic_local, self.critic_target, TAU) self.soft_update(self.actor_local, self.actor_target, TAU) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model: PyTorch model (weights will be copied from) target_model: PyTorch model (weights will be copied to) tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class Agent(): def __init__(self, n_state, n_action, n_agents, random_seed, device="cpu"): """Initialize an Agent object. Params ------ n_state : int dimension of each state n_action : int dimension of each action random_seed : int random seed device : which device is used, cpu or cuda. """ self.n_state = n_state self.n_action = n_action self.n_agents = n_agents self.random_seed = np.random.seed(random_seed) self.device = device # Networks for the first agent # Local Actor, Local Critic, Target Actor, Target Critic self.actor_local1 = Actor(self.n_state, self.n_action, self.random_seed).to(self.device) self.actor_local1.apply(initialize_weights) self.critic_local1 = Critic(self.n_state * self.n_agents, self.n_action * self.n_agents, self.random_seed).to(self.device) self.critic_local1.apply(initialize_weights) self.actor_target1 = Actor(self.n_state, self.n_action, self.random_seed).to(self.device) self.actor_target1.apply(initialize_weights) self.actor_target1.eval() self.critic_target1 = Critic(self.n_state * self.n_agents, self.n_action * self.n_agents, self.random_seed).to(self.device) self.critic_target1.apply(initialize_weights) self.critic_target1.eval() # Networks for the second agent # Local Actor, Local Critic, Target Actor, Target Critic self.actor_local2 = Actor(self.n_state, self.n_action, self.random_seed).to(self.device) self.actor_local2.apply(initialize_weights) self.critic_local2 = Critic(self.n_state * self.n_agents, self.n_action * self.n_agents, self.random_seed).to(self.device) self.critic_local2.apply(initialize_weights) self.actor_target2 = Actor(self.n_state, self.n_action, self.random_seed).to(self.device) self.actor_target2.apply(initialize_weights) self.actor_target2.eval() self.critic_target2 = Critic(self.n_state * self.n_agents, self.n_action * self.n_agents, self.random_seed).to(self.device) self.actor_target2.apply(initialize_weights) self.critic_target2.eval() # optimizers self.actor_optimizer1 = optim.Adam(self.actor_local1.parameters(), lr=LR_ACTOR) self.actor_optimizer2 = optim.Adam(self.actor_local2.parameters(), lr=LR_ACTOR) self.critic_optimizer1 = optim.Adam(self.critic_local1.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) self.critic_optimizer2 = optim.Adam(self.critic_local2.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) # Noise process self.noise = OUNoise(n_action * 2, random_seed + 1, mu=0., theta=THETA, sigma=SIGMA) # Replay Buffer self.memory = ReplayBuffer(n_action, BUFFER_SIZE, BATCH_SIZE, random_seed + 2, self.device) # Initialize time step (for updating every UPDATE_EVERY steps) self.t_step = 0 def step(self, state, action, reward, next_state, done): pass # Save experience / reward self.memory.add(state, action, reward, next_state, done) self.t_step = (self.t_step + 1) % UPDATE_EVERY # Learn, if enough samples are available in memory if self.t_step == 0 and len(self.memory) > BATCH_SIZE: for _ in range(N_LEARNING): experiences = self.memory.sample() self.learn(experiences, GAMMA) def act(self, state, add_noise=True): state0 = torch.from_numpy(state[0]).unsqueeze(dim=0).float().to( self.device) state1 = torch.from_numpy(state[1]).unsqueeze(dim=0).float().to( self.device) self.actor_local1.eval() self.actor_local2.eval() with torch.no_grad(): action0 = self.actor_local1(state0).cpu().data.numpy() action1 = self.actor_local2(state1).cpu().data.numpy() action = np.vstack([action0, action1]) self.actor_local1.train() self.actor_local2.train() if add_noise: action += self.noise.sample() return np.clip(action, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences, gamma): states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # # Get predicted next-state actions and Q values from target models with torch.no_grad(): actions_next1 = self.actor_target1(next_states[:, 0:24]) actions_next2 = self.actor_target2(next_states[:, 24:]) actions_next = torch.cat((actions_next1, actions_next2), dim=1) Q_targets_next1 = self.critic_target1(next_states, actions_next) Q_targets_next2 = self.critic_target2(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets1 = rewards[:, 0].unsqueeze( dim=1) + (gamma * Q_targets_next1 * (1 - dones[:, 0].unsqueeze(dim=1))) Q_targets2 = rewards[:, 1].unsqueeze( dim=1) + (gamma * Q_targets_next2 * (1 - dones[:, 1].unsqueeze(dim=1))) # Compute critic loss Q_expected1 = self.critic_local1(states, actions) Q_expected2 = self.critic_local2(states, actions) critic_loss1 = F.mse_loss(Q_expected1, Q_targets1.detach()) critic_loss2 = F.mse_loss(Q_expected2, Q_targets2.detach()) # Minimize the loss self.critic_optimizer1.zero_grad() critic_loss1.backward() torch.nn.utils.clip_grad_norm_(self.critic_local1.parameters(), 1) self.critic_optimizer1.step() self.critic_optimizer2.zero_grad() critic_loss2.backward() torch.nn.utils.clip_grad_norm_(self.critic_local2.parameters(), 1) self.critic_optimizer2.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred1 = self.actor_local1(states[:, 0:24]) actions_pred2 = self.actor_local2(states[:, 24:]) actions_pred = torch.cat((actions_pred1, actions_pred2), dim=1) actor_loss1 = -self.critic_local1(states, actions_pred).mean() # Minimize the loss self.actor_optimizer1.zero_grad() actor_loss1.backward(retain_graph=True) self.actor_optimizer1.step() actor_loss2 = -self.critic_local2(states, actions_pred).mean() self.actor_optimizer2.zero_grad() actor_loss2.backward(retain_graph=True) self.actor_optimizer2.step() # ----------------------- update target networks ----------------------- # self.soft_update(self.critic_local1, self.critic_target1, TAU) self.soft_update(self.actor_local1, self.actor_target1, TAU) self.soft_update(self.critic_local2, self.critic_target2, TAU) self.soft_update(self.actor_local2, self.actor_target2, TAU) def soft_update(self, local_model, target_model, tau): """Soft update model parameters Arguments """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class DDPGAgent(): """Interacts with and learns from the environment.""" def __init__(self, state_size, action_size, memory, device='cpu', params=None): """Initialize an Agent object. Params ====== state_size (int): dimension of each state action_size (int): dimension of each action memory (obj): Memory buffer to sample device (str): device string between cuda:0 and cpu params (dict): hyper-parameters """ self.state_size = state_size self.action_size = action_size self.device = device self.step_t = 0 self.update_every = params['update_every'] # Set parameters self.gamma = params['gamma'] self.tau = params['tau'] self.seed = random.seed(params['seed']) # Actor Network (w/ Target Network) self.actor_local = Actor(state_size, action_size, params['seed'], params['actor_units'][0], params['actor_units'][1]).to(device) self.actor_target = Actor(state_size, action_size, params['seed'], params['actor_units'][0], params['actor_units'][1]).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=params['lr_actor']) # Critic Network (w/ Target Network) self.critic_local = Critic(state_size, action_size, params['seed'], params['critic_units'][0], params['critic_units'][1]).to(device) self.critic_target = Critic(state_size, action_size, params['seed'], params['critic_units'][0], params['critic_units'][1]).to(device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=params['lr_critic'], weight_decay=params['weight_decay']) # Noise process self.noise = OUNoise(action_size, params['seed'], theta=params['noise_theta'], sigma=params['noise_sigma']) # Replay memory self.memory = memory def store_weights(self, filenames): """Store weights of Actor/Critic Params ====== filenames (list): string of filename to store weights of actor and critic filenames[0] = actor weights filenames[1] = critic weights """ torch.save(self.actor_local.state_dict(), filenames[0]) torch.save(self.critic_local.state_dict(), filenames[1]) def load_weights(self, filenames): """Load weights of Actor/Critic Params ====== filenames (list): string of filename to load weights of actor and critic filenames[0] = actor weights filenames[1] = critic weights """ self.actor_local.load_state_dict(torch.load(filenames[0])) self.critic_local.load_state_dict(torch.load(filenames[1])) def step(self, state, action, reward, next_state, done): """Save experience in replay memory, and use random sample from buffer to learn.""" # Save experience / reward self.memory.add(state, action, reward, next_state, done) self.step_t = (self.step_t + 1) % self.update_every # Learn, if enough samples are available in memory if self.step_t == 0 and len( self.memory) > self.memory.get_batch_size(): experiences = self.memory.sample() self.learn(experiences, self.gamma) def act(self, state, add_noise=True): """Returns actions for given state as per current policy.""" state = torch.from_numpy(state).float().to(self.device) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() if add_noise: action += self.noise.sample() return np.clip(action, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences, gamma): """Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Params ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor """ states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # # Get predicted next-state actions and Q values from target models actions_next = self.actor_target(next_states) Q_targets_next = self.critic_target(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)) # Compute critic loss Q_expected = self.critic_local(states, actions) critic_loss = F.mse_loss(Q_expected, Q_targets) # Minimize the loss self.critic_optimizer.zero_grad() critic_loss.backward() self.critic_optimizer.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred = self.actor_local(states) actor_loss = -self.critic_local(states, actions_pred).mean() # Minimize the loss self.actor_optimizer.zero_grad() actor_loss.backward() self.actor_optimizer.step() # ----------------------- update target networks ----------------------- # self.soft_update(self.critic_local, self.critic_target, self.tau) self.soft_update(self.actor_local, self.actor_target, self.tau) @staticmethod def soft_update(local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model: PyTorch model (weights will be copied from) target_model: PyTorch model (weights will be copied to) tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class Agent(): '''Interact with and learn from environment.''' def __init__(self, state_size, action_size, seed): self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) self.t_step = 0 # counter for activating learning every few steps self.TAU = 1e-2 self.gamma = 0.99 self.BUFFER_SIZE = int(1e6) self.BATCH_SIZE = 1024 self.LR_CRITIC = 1e-3 self.LR_ACTOR = 1e-3 self.WEIGHT_DECAY = 0.0 self.EPSILON = 1.0 self.EPSILON_DECAY = 0.99 # Actor network (w/ target network) self.actor_local = Actor(self.state_size, self.action_size, seed).to(device) self.actor_target = Actor(self.state_size, self.action_size, seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=self.LR_ACTOR) # Critic network (w/ target network) self.critic_local = Critic(self.state_size, self.action_size, seed).to(device) self.critic_target = Critic(self.state_size, self.action_size, seed).to(device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=self.LR_CRITIC, weight_decay=self.WEIGHT_DECAY) # Noise process self.noise = OUNoise(self.action_size, self.seed) def act(self, state, add_noise=True): """ Given a state choose an action Params ====== state (float ndarray): state of the environment """ state = torch.from_numpy(state).unsqueeze(0).float().to(device) self.actor_local.eval( ) # set network on eval mode, this has any effect only on certain modules (Dropout, BatchNorm, etc.) with torch.no_grad(): action = self.actor_local(state).cpu().squeeze(0).data.numpy() self.actor_local.train() # set nework on train mode if add_noise: action += self.noise.sample() * self.EPSILON return np.clip(action, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences): """ Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Policy loss = (1/n)*Q_local(s,a) -> for deterministic policy (no log prob) Params ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor is_weights (tensor array): importance-sampling weights for prioritized experience replay leaf_idxes (numpy array): indexes for update priorities in SumTree """ states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # # Get predicted next-state actions and Q values from target models actions_next = self.actor_target(next_states) Q_targets_next = self.critic_target(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets = rewards + (self.gamma * Q_targets_next * (1 - dones)) # Compute critic loss Q_expected = self.critic_local(states, actions) critic_loss = F.mse_loss(Q_expected, Q_targets) # Minimize the loss self.critic_optimizer.zero_grad() critic_loss.backward() self.critic_optimizer.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred = self.actor_local(states) actor_loss = -self.critic_local(states, actions_pred).mean() # Minimize the loss self.actor_optimizer.zero_grad() actor_loss.backward() self.actor_optimizer.step() # ----------------------- update target networks ----------------------- # self.soft_update(self.critic_local, self.critic_target, self.TAU) self.soft_update(self.actor_local, self.actor_target, self.TAU) self.EPSILON *= self.EPSILON_DECAY self.noise.reset() def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model: PyTorch model (weights will be copied from) target_model: PyTorch model (weights will be copied to) tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class Agent(): '''Interact with and learn from environment.''' def __init__(self, state_size, action_size, seed): self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) self.t_step = 0 # counter for activating learning every few steps self.running_c_loss = 0 self.running_a_loss = 0 self.training_cnt = 0 # Actor network (w/ target network) self.actor_local = Actor(state_size, action_size, seed).to(device) self.actor_target = Actor(state_size, action_size, seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR) # Critic network (w/ target network) self.critic_local = Critic(state_size, action_size, seed).to(device) self.critic_target = Critic(state_size, action_size, seed).to(device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) # Noise process self.noise = OUNoise(action_size, seed) # Prioritized replay memory self.prioritized_memory = PrioritizedMemory(BATCH_SIZE, BUFFER_SIZE, seed) def act(self, state, mode): '''Returns actions for given state as per current policy. Params ====== state (array): current state mode (string): train or test epsilon (float): for epsilon-greedy action selection ''' state = torch.from_numpy(state).unsqueeze(0).float().to( device) # shape of state (1, state_size) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() if mode == 'test': return np.clip(action, -1, 1) elif mode == 'train': # if train, then add OUNoise in action action += self.noise.sample() return np.clip(action, -1, 1) def step(self, state, action, reward, next_state, done): # add new experience in memory self.prioritized_memory.add(state, action, reward, next_state, done) # activate learning every few steps self.t_step = self.t_step + 1 if self.t_step % LEARN_EVERY_STEP == 0: # If enough samples are available in memory, get random subset and learn if len(self.prioritized_memory) >= BUFFER_SIZE: for _ in range(10): # update 10 times per learning idxes, experiences, is_weights = self.prioritized_memory.sample( device) self.learn(experiences, GAMMA, is_weights=is_weights, leaf_idxes=idxes) def reset(self): self.noise.reset() def learn(self, experiences, gamma, is_weights, leaf_idxes): """ Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Policy loss = (1/n)*Q_local(s,a) -> for deterministic policy (no log prob) Params ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor is_weights (tensor array): importance-sampling weights for prioritized experience replay leaf_idxes (numpy array): indexes for update priorities in SumTree """ states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # rewards = rewards # TODO: rewards are clipped to be in [-1,1] actions_next = self.actor_target(next_states) Q_targets_next = self.critic_target(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)) # Compute critic loss Q_expected = self.critic_local(states, actions) td_errors = ( Q_targets - Q_expected).tanh() # TD-errors are clipped to be in [-1,1] abs_errors = td_errors.abs().cpu().data.numpy() # pull back to cpu self.prioritized_memory.batch_update( leaf_idxes, abs_errors) # update priorities in SumTree c_loss = (is_weights * (td_errors**2)).mean( ) # adjust squared TD loss by Importance-Sampling Weights self.running_c_loss += float(c_loss.cpu().data.numpy()) self.training_cnt += 1 # Minimize the loss self.critic_optimizer.zero_grad() c_loss.backward() torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1) # clip gradient to max 1 self.critic_optimizer.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred = self.actor_local(states) a_loss = self.critic_local(states, actions_pred) a_loss = -a_loss.mean() self.running_a_loss += float(a_loss.cpu().data.numpy()) # Minimize the loss self.actor_optimizer.zero_grad() a_loss.backward() torch.nn.utils.clip_grad_norm_(self.actor_local.parameters(), 1) # clip gradient to max 1 self.actor_optimizer.step() # ------------------- update target network ------------------- # self.soft_update(self.critic_local, self.critic_target, TAU) self.soft_update(self.actor_local, self.actor_target, TAU) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model: PyTorch model (weights will be copied from) target_model: PyTorch model (weights will be copied to) tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class Agent(): '''Interact with and learn from environment.''' def __init__(self, state_size, action_size, seed): self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) self.t_step = 0 # counter for activating learning every few steps self.running_c_loss = 0 self.running_a_loss = 0 self.training_cnt = 0 # Actor network (w/ target network) self.actor_local = Actor(state_size, action_size, seed).to(device) self.actor_target = Actor(state_size, action_size, seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR) # Critic network (w/ target network) self.critic_local = Critic(state_size, action_size, seed).to(device) self.critic_target = Critic(state_size, action_size, seed).to(device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) # Noise process self.noise = OUNoise(action_size, seed) # Replay memory self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed) def act(self, state, mode): '''Returns actions for given state as per current policy. Params ====== state (array): current state mode (string): train or test epsilon (float): for epsilon-greedy action selection ''' state = torch.from_numpy(state).unsqueeze(0).float().to( device) # shape of state (1, state_size) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() if mode == 'test': return np.clip(action, -1, 1) elif mode == 'train': # if train, then add OUNoise in action action += self.noise.sample() return np.clip(action, -1, 1) def step(self, state, action, reward, next_state, done): """Save experience in replay memory, and use random sample from buffer to learn.""" # Save experience / reward self.memory.add(state, action, reward, next_state, done) # activate learning every few steps self.t_step = self.t_step + 1 if self.t_step % LEARN_EVERY_STEP == 0: # Learn, if enough samples are available in memory if len(self.memory) > BATCH_SIZE: for _ in range(10): # update 10 times per learning experiences = self.memory.sample() self.learn(experiences, GAMMA) def reset(self): self.noise.reset() def learn(self, experiences, gamma): """Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Params ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor """ states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # # Get predicted next-state actions and Q values from target models actions_next = self.actor_target(next_states) Q_targets_next = self.critic_target(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)) # Compute critic loss Q_expected = self.critic_local(states, actions) critic_loss = F.mse_loss(Q_expected, Q_targets) self.running_c_loss += float(critic_loss.cpu().data.numpy()) self.training_cnt += 1 # Minimize the loss self.critic_optimizer.zero_grad() critic_loss.backward() torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1) self.critic_optimizer.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred = self.actor_local(states) actor_loss = -self.critic_local(states, actions_pred).mean() self.running_a_loss += float(actor_loss.cpu().data.numpy()) # Minimize the loss self.actor_optimizer.zero_grad() actor_loss.backward() #torch.nn.utils.clip_grad_norm_(self.actor_local.parameters(), 1) # clip gradient to max 1 self.actor_optimizer.step() # ------------------- update target network ------------------- # self.soft_update(self.critic_local, self.critic_target, TAU) self.soft_update(self.actor_local, self.actor_target, TAU) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model: PyTorch model (weights will be copied from) target_model: PyTorch model (weights will be copied to) tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class Agent(): """Interacts with and learns from the environment.""" def __init__(self, num_agents, state_size, action_size, random_seed, buffer_size, batch_size, gamma, TAU, lr_actor, lr_critic, weight_decay, a_hidden_sizes, c_hidden_sizes): """Initialize an Agent object. Params ====== state_size (int): dimension of each state action_size (int): dimension of each action random_seed (int): random seed """ self.state_size = state_size self.action_size = action_size self.seed = random.seed(random_seed) # Hyperparameters self.BUFFER_SIZE = buffer_size self.BATCH_SIZE = batch_size self.GAMMA = gamma self.TAU = TAU self.LR_ACTOR = lr_actor self.LR_CRITIC = lr_critic self.WEIGHT_DECAY = weight_decay self.ACTOR_HL_SIZE = a_hidden_sizes self.CRITIC_HL_SIZE = c_hidden_sizes self.num_agents = num_agents # Actor Network (w/ Target Network) self.actor_local_1 = Actor(state_size, action_size, random_seed, self.ACTOR_HL_SIZE).to(device) self.actor_target_1 = Actor(state_size, action_size, random_seed, self.ACTOR_HL_SIZE).to(device) self.actor_optimizer_1 = optim.Adam(self.actor_local_1.parameters(), lr=self.LR_ACTOR) # Critic Network (w/ Target Network) self.critic_local_1 = Critic(state_size, action_size, random_seed, self.CRITIC_HL_SIZE).to(device) self.critic_target_1 = Critic(state_size, action_size, random_seed, self.CRITIC_HL_SIZE).to(device) self.critic_optimizer_1 = optim.Adam(self.critic_local_1.parameters(), lr=self.LR_CRITIC, weight_decay=self.WEIGHT_DECAY) # Actor Network (w/ Target Network) self.actor_local_2 = Actor(state_size, action_size, random_seed, self.ACTOR_HL_SIZE).to(device) self.actor_target_2 = Actor(state_size, action_size, random_seed, self.ACTOR_HL_SIZE).to(device) self.actor_optimizer_2 = optim.Adam(self.actor_local_2.parameters(), lr=self.LR_ACTOR) # Critic Network (w/ Target Network) self.critic_local_2 = Critic(state_size, action_size, random_seed, self.CRITIC_HL_SIZE).to(device) self.critic_target_2 = Critic(state_size, action_size, random_seed, self.CRITIC_HL_SIZE).to(device) self.critic_optimizer_2 = optim.Adam(self.critic_local_2.parameters(), lr=self.LR_CRITIC, weight_decay=self.WEIGHT_DECAY) # Noise process self.noise = OUNoise(action_size, random_seed) # Replay memory self.memory = ReplayBuffer(action_size, self.BUFFER_SIZE, self.BATCH_SIZE, random_seed) def step(self, states, actions, rewards, next_states, dones): """Save experience in replay memory, and use random sample from buffer to learn.""" # Save experience / reward for i in range(states.shape[0]): self.memory.add(states[i], actions[i], rewards[i], next_states[i], dones[i]) if len(self.memory) > self.BATCH_SIZE: experiences = self.memory.sample() self.learn(experiences, self.GAMMA) def act(self, states, add_noise=True): """Returns actions for given state as per current policy.""" states = torch.from_numpy(states).float().to(device) self.actor_local_1.eval() self.actor_local_2.eval() action_values = [states.shape[0], self.action_size] with torch.no_grad(): action_values[0] = self.actor_local_1(states[0]).cpu().data.numpy() action_values[1] = self.actor_local_2(states[1]).cpu().data.numpy() self.actor_local_1.train() self.actor_local_2.train() #print (action_values) if add_noise: action_values += self.noise.sample() #print (action_values) #print (np.clip(action_values, -1, 1)) return np.clip(action_values, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences, gamma): """Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Params ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor """ states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # # Get predicted next-state actions and Q values from target models actions_next_1 = self.actor_target_1(next_states) actions_next_2 = self.actor_target_2(next_states) Q_targets_next_1 = self.critic_target_1(next_states, actions_next_1.detach()) Q_targets_next_2 = self.critic_target_2(next_states, actions_next_2.detach()) # Compute Q targets for current states (y_i) Q_targets_1 = rewards + (gamma * Q_targets_next_1 * (1 - dones)) Q_targets_2 = rewards + (gamma * Q_targets_next_2 * (1 - dones)) # Compute critic loss Q_expected_1 = self.critic_local_1(states, actions) Q_expected_2 = self.critic_local_2(states, actions) critic_loss_1 = F.mse_loss(Q_expected_1, Q_targets_1.detach()) critic_loss_2 = F.mse_loss(Q_expected_2, Q_targets_2.detach()) # Minimize the loss self.critic_optimizer_1.zero_grad() self.critic_optimizer_2.zero_grad() critic_loss_1.backward() critic_loss_2.backward() #torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1) # adds gradient clipping to stabilize learning self.critic_optimizer_1.step() self.critic_optimizer_2.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred_1 = self.actor_local_1(states) actions_pred_2 = self.actor_local_2(states) actor_loss_1 = -self.critic_local_1(states, actions_pred_1).mean() actor_loss_2 = -self.critic_local_2(states, actions_pred_2).mean() # Minimize the loss self.actor_optimizer_1.zero_grad() self.actor_optimizer_2.zero_grad() actor_loss_1.backward() actor_loss_2.backward() self.actor_optimizer_1.step() self.actor_optimizer_2.step() # ----------------------- update target networks ----------------------- # self.soft_update(self.critic_local_1, self.critic_target_1, self.TAU) self.soft_update(self.critic_local_2, self.critic_target_2, self.TAU) self.soft_update(self.actor_local_1, self.actor_target_1, self.TAU) self.soft_update(self.actor_local_2, self.actor_target_2, self.TAU) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model: PyTorch model (weights will be copied from) target_model: PyTorch model (weights will be copied to) tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class Agent: def __init__(self, state_size=OBS_DIM, action_size=ACT_DIM, random_seed=0): """Initialize an Agent object. Params ===== state_size (int): dimension of all observation action_size (int): dimension of each action random_seed (int): random seed """ self.state_size = state_size self.action_size = action_size self.seed = random.seed(random_seed) self.epsilon = EPSILON self.actor_local = Actor(state_size, action_size, random_seed).to(device) self.actor_target = Actor(state_size, action_size, random_seed).to(device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR) self.critic_local = Critic(state_size, action_size, random_seed).to(device) self.critic_target = Critic(state_size, action_size, random_seed).to(device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC) self.noise = OUNoise(action_size, random_seed) self.memory = ReplayBuffer(BUFFER_SIZE, BATCH_SIZE, random_seed) def step(self, state, action, reward, next_state): """Save an experience in replay buffer and use random samples from buffer to learn.""" self.memory.add(state, action, reward, next_state) if len(self.memory ) > BATCH_SIZE: # begin to learn when replay buffer is full experiences = self.memory.sample() self.learn(experiences, GAMMA) def act(self, state, add_noise=True): """Return actions for given state as per current policy.""" state = state[None, :] state = torch.from_numpy(state).float().to(device) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() if add_noise: action += self.epsilon * self.noise.sample() return np.clip(action, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences, gamma): """Update policy and value parameters using given batch of experience tuples Q_target = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q_value Params ===== experiences (Tuple[torch.Tensor]): tuple of (s,a,r,s',done) gamma (float): discount factor """ states, actions, rewards, next_states = experiences # ----------------- update critic network weights ---------------- # # get predicted next_state actions and Q_values from target models actions_next = self.actor_target(next_states) q_targets_next = self.critic_target(next_states, actions_next) # compute Q targets for current states q_targets = rewards + gamma * q_targets_next # compute critic loss q_expected = self.critic_local(states, actions) critic_loss = F.mse_loss(q_expected, q_targets) # minimize the loss self.critic_optimizer.zero_grad() critic_loss.backward() torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1) self.critic_optimizer.step() # ---------------- update actor network weights ---------------- # # compute the loss actions_pred = self.actor_local(states) actor_loss = -self.critic_local(states, actions_pred).mean() # minimize the loss self.actor_optimizer.zero_grad() actor_loss.backward() self.actor_optimizer.step() # ----------------- update target networks ------------------ # self.soft_update(self.critic_local, self.critic_target, TAU) self.soft_update(self.actor_local, self.actor_target, TAU) # ----------------- update noise -------------------- # self.epsilon -= EPSILON_DECAY self.noise.reset() @staticmethod def soft_update(local_model, target_model, tau): """Soft update model parameters θ_target = τ * θ_local + (1 - τ) * θ_target Params ===== local_model: Network weights to be copied from target_model: Network weights to be copied to tau(float): interpolation parameter """ for local_param, target_param in zip(local_model.parameters(), target_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data) def restore(self, save_path): actor_checkpoint = torch.load(save_path + '/checkpoint_actor.pth') self.actor_local.load_state_dict(actor_checkpoint) critic_checkpoint = torch.load(save_path + '/checkpoint_critic.pth') self.actor_local.load_state_dict(critic_checkpoint) print('Successfully load network weights!')
class DDPG_Agent(): """Interacts with and learns from the environment.""" def __init__(self, state_size, action_size, brain_name, seed, params=default_params, device=None): """Initialize an Agent object. Params ====== state_size (int): dimension of each state action_size (int): dimension of each action seed (int): random seed """ params = self._fill_params(params) # implementation and identity self.device = device if device is not None else torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") self.name = params['name'] self.brain_name = brain_name # set environment information self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) # Actor Network (w/ Target Network) self.actor_local = Actor(state_size, action_size, seed, fc1_units=params['layers_actor'][0], fc2_units=params['layers_actor'][1]).to( self.device) self.actor_target = Actor(state_size, action_size, seed, fc1_units=params['layers_actor'][0], fc2_units=params['layers_actor'][1]).to( self.device) self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=params['lr_actor']) # Critic Network (w/ Target Network) self.critic_local = Critic(state_size, action_size, seed, fcs1_units=params['layers_critic'][0], fc2_units=params['layers_critic'][1]).to( self.device) self.critic_target = Critic(state_size, action_size, seed, fcs1_units=params['layers_critic'][0], fc2_units=params['layers_critic'][1]).to( self.device) self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=params['lr_critic'], weight_decay=params['weight_decay']) # Noise process self.noise = OUNoise(action_size, seed) # Replay memory self.memory = ReplayBuffer(action_size, params['buffer_size'], params['batch_size'], seed, device=self.device) # save params self.params = params def _fill_params(self, src_params): params = { 'name': self._get_param_or_default('name', src_params, default_params), 'buffer_size': self._get_param_or_default('buffer_size', src_params, default_params), 'batch_size': self._get_param_or_default('batch_size', src_params, default_params), 'layers_actor': self._get_param_or_default('layers_actor', src_params, default_params), 'layers_critic': self._get_param_or_default('layers_critic', src_params, default_params), 'lr_actor': self._get_param_or_default('lr_actor', src_params, default_params), 'lr_critic': self._get_param_or_default('lr_critic', src_params, default_params), 'gamma': self._get_param_or_default('gamma', src_params, default_params), 'tau': self._get_param_or_default('tau', src_params, default_params), 'weight_decay': self._get_param_or_default('weight_decay', src_params, default_params) } return params def display_params(self, force_print=False): if force_print: print(self.params) return self.params def _get_param_or_default(self, key, src_params, default_params): if key in src_params: return src_params[key] else: return default_params[key] def step(self, state, action, reward, next_state, done): """Save experience in replay memory, and use random sample from buffer to learn.""" # Save experience / reward self.memory.add(state, action, reward, next_state, done) def start_learn(self): # Learn, if enough samples are available in memory # decoupled from step method to allow multiple steps per learning pass if len(self.memory) > self.params['batch_size']: experiences = self.memory.sample() self.learn(experiences, self.params['gamma']) def act(self, state, add_noise=True): """Returns actions for given state as per current policy.""" state = torch.from_numpy(state).float().to(self.device) self.actor_local.eval() with torch.no_grad(): action = self.actor_local(state).cpu().data.numpy() self.actor_local.train() if add_noise: action += self.noise.sample() return np.clip(action, -1, 1) def reset(self): self.noise.reset() def learn(self, experiences, gamma): """Update policy and value parameters using given batch of experience tuples. Q_targets = r + γ * critic_target(next_state, actor_target(next_state)) where: actor_target(state) -> action critic_target(state, action) -> Q-value Params ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor """ states, actions, rewards, next_states, dones = experiences # ---------------------------- update critic ---------------------------- # # Get predicted next-state actions and Q values from target models actions_next = self.actor_target(next_states) Q_targets_next = self.critic_target(next_states, actions_next) # Compute Q targets for current states (y_i) Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)) # Compute critic loss Q_expected = self.critic_local(states, actions) critic_loss = F.mse_loss(Q_expected, Q_targets) # Minimize the loss self.critic_optimizer.zero_grad() critic_loss.backward() torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1) self.critic_optimizer.step() # ---------------------------- update actor ---------------------------- # # Compute actor loss actions_pred = self.actor_local(states) actor_loss = -self.critic_local(states, actions_pred).mean() # Minimize the loss self.actor_optimizer.zero_grad() actor_loss.backward() self.actor_optimizer.step() # ----------------------- update target networks ----------------------- # self.soft_update(self.critic_local, self.critic_target, self.params['tau']) self.soft_update(self.actor_local, self.actor_target, self.params['tau']) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model: PyTorch model (weights will be copied from) target_model: PyTorch model (weights will be copied to) tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class DDPGAgent: def __init__(self, action_size=4, state_size=33, num_agents=20, max_steps=1000, seed=0, train_mode=True): self.train_mode = train_mode self.action_size = action_size self.state_size = state_size self.num_agents = num_agents self.max_steps = max_steps self.step_count = 0 self.scores = np.zeros(self.num_agents) self.states, self.actions, self.rewards, self.next_states, self.dones = None, None, None, None, None self.noise = OUNoise(self.action_size, seed) self.memory = AgentMemory(batch_size=BATCH_SIZE, buffer_size=MEMORY_BUFFER, seed=seed) self.actor = Actor(self.state_size, self.action_size, seed) self.critic = Critic(self.state_size, self.action_size, seed) self.target_actor = Actor(self.state_size, self.action_size, seed) self.target_critic = Critic(self.state_size, self.action_size, seed) self.actor_opt = optim.Adam(self.actor.parameters(), lr=LR_ACTOR) self.critic_opt = optim.Adam(self.critic.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY) hard_update(self.actor, self.target_actor) hard_update(self.critic, self.target_critic) def reset(self): self.noise.reset() self.step_count = 0 self.scores = np.zeros(self.num_agents) self.states, self.actions, self.rewards, self.next_states, self.dones = None, None, None, None, None def step(self): self.scores += np.array(self.rewards) self.step_count += 1 self.memory.add(self.states, self.actions, self.rewards, self.next_states, self.dones) if self.memory.has_enough_memory(): for i in range(UPDATE_FREQUENCY_PER_STEP): states, actions, rewards, next_states, dones = self.memory.sample( ) self.learn(states, actions, rewards, next_states, dones) self.soft_update() def act(self, add_noise=True): states = array_to_tensor(self.states) self.actor.eval() with torch.no_grad(): actions = self.actor(states) actions = actions.cpu().data.numpy() self.actor.train() if add_noise: noise = self.noise.sample() actions += noise actions = np.clip(actions, -1, 1) return actions def learn(self, states, actions, rewards, next_states, dones): # Update critic self.critic_opt.zero_grad() critic_loss = ddpg_compute_critic_loss(states, actions, rewards, next_states, dones, self.target_actor, self.target_critic, self.critic) critic_loss.backward() torch.nn.utils.clip_grad_norm_(self.critic.parameters(), 1) self.critic_opt.step() # Update actor self.actor_opt.zero_grad() actor_loss = ddpg_compute_actor_loss(states, self.actor, self.critic) actor_loss.backward() torch.nn.utils.clip_grad_norm_(self.actor.parameters(), 1) self.actor_opt.step() # Update target nets self.soft_update() def soft_update(self): soft_update(self.actor, self.target_actor, TAU) soft_update(self.critic, self.target_critic, TAU)