# Loop until mission starts: print("Waiting for the mission to start ", end=' ') world_state = agent_host.getWorldState() while not world_state.has_mission_begun: print(".", end="") time.sleep(0.1) world_state = agent_host.getWorldState() for error in world_state.errors: print("Error:", error.text) print() print("Mission running ", end=' ') agent_host.sendCommand("chat /time set day") agent = ag.Agent(agent_host) # Loop until mission ends: while not agent.finished: state = transform_farm(copy.deepcopy(agent.state)) action = agent.select_action(state, net) reward = agent.run(action.item()) memory.push(state, action, transform_farm(copy.deepcopy(agent.state)), torch.tensor([reward]).long()) cnn.train(net, memory) print() print("Mission ended") agent_host.sendCommand("chat /kill @p") # Mission has ended.
class DQN: def __init__(self, env, hparams): self.hparams = hparams self.env = env self.n = env.action_space.n self.Q = DCNN(4, self.n) self.T = DCNN(4, self.n) self.T.load_state_dict(self.Q.state_dict()) self.T.eval() self.memory = ReplayMemory(hparams.memory_size) self.steps = 0 self.state = env.reset() self.optimizer = torch.optim.RMSprop(self.Q.parameters(), lr=hparams.lr, momentum=hparams.momentum) self.n_episodes = 0 @torch.no_grad() def select_action(self): hparams = self.hparams start = hparams.eps_start end = hparams.eps_end time = hparams.eps_time steps = self.steps self.steps += 1 if steps < time: epsilon = start - (start - end) * steps / time else: epsilon = end sample = random.random() if sample > epsilon: return self.Q(s2t(self.state).to(device)).max(1)[1].item() else: return self.env.action_space.sample() def sample_step(self, fs_min=2, fs_max=6): """repeats a single action between fs_min and fs_max (inclusive) times""" fs = random.randint(fs_min, fs_max) action = self.select_action() r = 0 for _ in range(fs): new_state, reward, done, _ = self.env.step(action) self.memory.push(self.state, action, new_state if not done else None, reward) r += reward self.state = self.env.reset() if done else new_state if done: self.n_episodes += 1 return r def optimize(self): hparams = self.hparams transitions = self.memory.sample(hparams.batch_size) batch = Transition(*zip(*transitions)) states = torch.cat([s2t(state) for state in batch.state]).to(device) actions = torch.tensor(batch.action).unsqueeze(1).to(device) target_values = torch.tensor( batch.reward).unsqueeze(1).to(device).float() non_terminal_next_states = torch.cat([ s2t(state) for state in batch.next_state if state is not None ]).to(device) non_terminal_mask = torch.tensor([ state is not None for state in batch.next_state ]).to(device).unsqueeze(1) values = self.Q(states).gather(1, actions).float() target_values[non_terminal_mask] += hparams.gamma * self.T( non_terminal_next_states).detach().max(1)[0].float() #print(values.dtype,target_values.dtype) loss = F.smooth_l1_loss(values, target_values) self.optimizer.zero_grad() loss.backward() for param in self.Q.parameters(): param.grad.data.clamp_(-1, 1) # maybe try sign_? self.optimizer.step() return loss
class Agent: def __init__(self, env, env_w, device, config: Config): self.env = env self.env_w = env_w self.device = device self.cfg = config self.n_actions = config.n_actions self.policy_net = config.policy_net self.target_net = config.target_net self.target_net.load_state_dict(self.policy_net.state_dict()) self.target_net.eval() self.optimizer = optim.RMSprop(self.policy_net.parameters()) self.memory = ReplayMemory(10000) self.steps_done = 0 self.episode_durations = [] def select_action(self, state): self.steps_done += 1 sample = random.random() eps_threshold = self.cfg.EPS_END + (self.cfg.EPS_START - self.cfg.EPS_END) * \ math.exp(-1. * self.steps_done / self.cfg.EPS_DECAY) if sample < eps_threshold: with torch.no_grad(): # t.max(1) will return largest column value of each row. # second column on max result is index of where max element was # found, so we pick action with the larger expected reward. # action = self.policy_net(state).max(1)[1] action = self.policy_net(state).argmax() % self.n_actions else: action = random.randrange(self.n_actions) return torch.tensor([[action]], device=self.device, dtype=torch.long) def optimize_model(self): if len(self.memory) < self.cfg.BATCH_SIZE: return transitions = self.memory.sample(self.cfg.BATCH_SIZE) # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for # detailed explanation). This converts batch-array of Transitions # to Transition of batch-arrays. batch = Transition(*zip(*transitions)) # Compute a mask of non-final states and concatenate the batch elements # (a final state would've been the one after which simulation ended) non_final_mask = torch.tensor(tuple(map(lambda s: s is not None, batch.next_state)), device=self.device, dtype=torch.bool) non_final_next_states = torch.cat([s for s in batch.next_state if s is not None]) state_batch = torch.cat(batch.state) action_batch = torch.cat(batch.action) reward_batch = torch.cat(batch.reward) # Compute Q(s_t, a) - the model computes Q(s_t), then we select the # columns of actions taken. These are the actions which would've been taken # for each batch state according to policy_net state_action_values = self.policy_net(state_batch).gather(1, action_batch) # Compute V(s_{t+1}) for all next states. # Expected values of actions for non_final_next_states are computed based # on the "older" target_net; selecting their best reward with max(1)[0]. # This is merged based on the mask, such that we'll have either the expected # state value or 0 in case the state was final. next_state_values = torch.zeros(self.cfg.BATCH_SIZE, device=self.device) next_state_values[non_final_mask] = self.target_net(non_final_next_states).max(1)[0].detach() # Compute the expected Q values expected_state_action_values = (next_state_values * self.cfg.GAMMA) + reward_batch # Compute Huber loss loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1)) # Optimize the model self.optimizer.zero_grad() loss.backward() for param in self.policy_net.parameters(): param.grad.data.clamp_(-1, 1) self.optimizer.step() def step(self, i_episode): # Initialize the environment and state self.env.reset() last_screen = self.env_w.get_screen() current_screen = self.env_w.get_screen() state = current_screen - last_screen for t in count(): # Select and perform an action action = self.select_action(state) obs, reward, done, obs_ = self.env.step(action.item()) # reward = torch.tensor([reward], device=self.device) reward = torch.tensor([-abs(obs[2])], device=self.device, dtype=torch.float) # Observe new state last_screen = current_screen current_screen = self.env_w.get_screen() if not done: next_state = current_screen - last_screen else: next_state = None # Store the transition in memory self.memory.push(state, action, next_state, reward) # Move to the next state state = next_state # Perform one step of the optimization (on the target network) self.optimize_model() if done: self.episode_durations.append(t + 1) self.env_w.plot_durations(self.episode_durations) break # Update the target network, copying all weights and biases in DQN if i_episode % self.cfg.TARGET_UPDATE == 0: self.target_net.load_state_dict(self.policy_net.state_dict())
while not world_state.has_mission_begun: print(".", end="") time.sleep(0.1) world_state = agent_host.getWorldState() for error in world_state.errors: print("Error:", error.text) print() print("Mission running ", end=' ') agent_host.sendCommand("chat /time set day") agent = ag.Agent(agent_host) # Loop until mission ends: while not agent.finished: state = agent.state.copy() action = agent.select_action(state) reward = agent.run(action) memory.push(state, action, agent.state.copy(), reward) net.train(memory) print(agent.state) memory.print_replay() #net.forward(agent.state) print() print("Mission ended") agent_host.sendCommand("chat /kill @p") # Mission has ended.