class Agent(): def __init__(self, state_size, action_size, seed): self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) # Q-Network self.qnetwork_local = QNetwork(state_size, action_size, seed) self.qnetwork_target = QNetwork(state_size, action_size, seed) self.qnetwork_local.load_model("./dqn_LL_model data.pickle") self.qnetwork_target.load_model("./dqn_LL_model data.pickle") # Replay memory self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed) # Initialize time step (for updating every UPDATE_EVERY steps) self.t_step = 0 self.loss = 0 self.loss_list = [] def step(self, state, action, reward, next_state, done, t_step): # Save experience in replay memory self.memory.add(state, action, reward, next_state, done) # Learn every UPDATE_EVERY time steps. self.t_step = t_step if self.t_step % UPDATE_EVERY == 0: # If enough samples are available in memory, get random subset and learn if len(self.memory) > 100 * BATCH_SIZE: experiences = self.memory.sample() self.learn(experiences, GAMMA) def act(self, state, eps=0.): """Returns actions for given state as per current policy. """ action_values = self.qnetwork_local.forward(state) # Epsilon-greedy action selection if random.random() > eps: return np.argmax(action_values) else: return random.choice(np.arange(self.action_size)) def learn(self, experiences, gamma): """Update value parameters using given batch of experience tuples. ====== experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples """ states, actions, rewards, next_states, dones = experiences for time in range(BATCH_SIZE): # compute Q_target from the target network inputing next_state Q_target_av = np.max( self.qnetwork_target.forward(next_states[time])) Q_target = rewards[time] + gamma * (Q_target_av) * ( 1 - dones[time]) # if done, than the second will not be added # compute the Q_expected Q_expected = self.qnetwork_local.forward( states[time] ) # get q value for corrosponding action along dimension 1 of 64,4 matrix self.qnetwork_local.backward(Q_target, "MSE", actions[time]) self.loss_list.append((Q_target - Q_expected[actions[time]])**2) self.loss = np.mean(self.loss_list) self.qnetwork_local.step() self.loss_list.clear() # update target network # if self.t_step % UPDATE_FREQUENCY == 0: self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = tau*θ_local + (1 - tau)*θ_target """ self.qnetwork_target.soft_update(local_model, TAU)
class Agent(): """Interacts with and learns from the environment.""" def __init__(self, state_size, action_size, seed, compute_weights=False): """Initialize an Agent object. Params ====== state_size (int): dimension of each state action_size (int): dimension of each action seed (int): random seed """ self.state_size = state_size self.action_size = action_size self.seed = random.seed(seed) self.compute_weights = compute_weights # Algorithms to enable during training self.PrioritizedReplayBuffer = True # Use False to enable uniform sampling self.HardTargetUpdate = True # Use False to enable soft target update # building the policy and target Q-networks for the agent, such that the target Q-network is kept frozen to avoid the training instability issues # Q-Network self.qnetwork_local = QNetwork(state_size, action_size, seed).to(device) # main policy network self.qnetwork_target = QNetwork(state_size, action_size, seed).to(device) # target network self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR) self.criterion = nn.MSELoss() # Replay memory # building the experience replay memory used to avoid training instability issues # Below: PER self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, EXPERIENCES_PER_SAMPLING, seed, compute_weights) # Below: Uniform by method defined in this script #self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed) # Initialize time step (for updating every UPDATE_NN_EVERY steps) self.t_step_nn = 0 # Initialize time step (for updating every UPDATE_MEM_PAR_EVERY steps) self.t_step_mem_par = 0 # Initialize time step (for updating every UPDATE_MEM_EVERY steps) self.t_step_mem = 0 def step(self, state, action, reward, next_state, done): # Save experience in replay memory self.memory.add(state, action, reward, next_state, done) # Learn every UPDATE_NN_EVERY time steps. self.t_step_nn = (self.t_step_nn + 1) % UPDATE_NN_EVERY self.t_step_mem = (self.t_step_mem + 1) % UPDATE_MEM_EVERY self.t_step_mem_par = (self.t_step_mem_par + 1) % UPDATE_MEM_PAR_EVERY if self.t_step_mem_par == 0: self.memory.update_parameters() if self.t_step_nn == 0: # If enough samples are available in memory, get random subset and learn if self.memory.experience_count > EXPERIENCES_PER_SAMPLING: sampling = self.memory.sample() self.learn(sampling, GAMMA) if self.t_step_mem == 0: self.memory.update_memory_sampling() def act(self, state, eps=0.): """A function to select an action based on the Epsilon greedy policy. Epislon percent of times the agent will select a random action while 1-Epsilon percent of the time the agent will select the action with the highest Q value as predicted by the neural network. Params ====== state (array_like): current state eps (float): epsilon, for epsilon-greedy action selection """ state = torch.from_numpy(state).float().unsqueeze(0).to(device) # here we calculate action values (Q values) self.qnetwork_local.eval( ) # model deactivate norm, dropout etc. layers as it is expected with torch.no_grad(): action_values = self.qnetwork_local(state) self.qnetwork_local.train( ) # model.train() sets the modules in the network in training mode # Epsilon-greedy action selection if random.random() > eps: return np.argmax(action_values.cpu().data.cpu().numpy()) else: return random.choice(np.arange(self.action_size)) def learn(self, sampling, gamma): """Update value parameters using given batch of experience tuples. Function for training the neural network. The function will update the weights of the newtwork Params ====== sampling (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples gamma (float): discount factor """ states, actions, rewards, next_states, dones, weights, indices = sampling # Target (absolute) Q values from target Q network q_target = self.qnetwork_target(next_states).detach().max( 1)[0].unsqueeze(1) # Predictions from local Q network expected_values = rewards + gamma * q_target * (1 - dones) output = self.qnetwork_local(states).gather(1, actions) # computing the loss loss = F.mse_loss(output, expected_values) # Loss Function: Mean Square Error if self.compute_weights: with torch.no_grad(): weight = sum(np.multiply(weights, loss.data.cpu().numpy())) loss *= weight # Minimizing the loss by optimizer self.optimizer.zero_grad() loss.backward() self.optimizer.step() # ------------------- update target network ------------------- # self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU) # ------------------- update priorities ------------------- # delta = abs(expected_values - output.detach()).cpu().numpy() #print("delta", delta) self.memory.update_priorities(delta, indices) def soft_update(self, local_model, target_model, tau): """Soft update model parameters. θ_target = τ*θ_local + (1 - τ)*θ_target Params ====== local_model (PyTorch model): weights will be copied from target_model (PyTorch model): weights will be copied to tau (float): interpolation parameter """ for target_param, local_param in zip(target_model.parameters(), local_model.parameters()): target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data) # def hard_update(self): # """ This hard_update method performs direct update of target network # weight update from local network weights instantly""" # Write the algorithm here def load_models(self, policy_net_filename, target_net_filename): """ Function to load the parameters of the policy and target models """ print('Loading model...') self.qnetwork_local.load_model(policy_net_filename) self.qnetwork_target.load_model(target_net_filename)