class VAE_DQNAgent(agent): def __init__(self, model, opt, learning=True): super().__init__() self.memory = PrioritizedReplayBuffer(100000, 0.6) self.previous_state = None self.previous_action = None self.previous_legal_actions = None self.step = 0 self.model_vae = model[0] self.model_dqn = model[1] self.model_dqn_target = model[2] self.opt_vae = opt[0] self.opt_dqn = opt[1] self.loss_vae = 0 self.loss_dqn = 0 self.batch_size = 32 self.max_tile = 0 self.totalCorrect = 0 self.total = 0 self.acc = 0 self.beta = 0.7 #self.test_q = 0 self.epsilon_schedule = LinearSchedule(500000, initial_p=0.99, final_p=0.01) self.learning = learning def should_explore(self): self.epsilon = self.epsilon_schedule.value(self.step) return random.random() < self.epsilon def action(self): if self.learning: self.step += 1 legalActions = self.legal_actions(deepcopy(self.gb.board)) board = deepcopy(self.gb.board) board = oneHotMap(board) if self.learning and self.should_explore(): q_values = None action = random.choice(legalActions) choice = self.actions[action] else: #mark state = torch.from_numpy(board).type( torch.FloatTensor).cuda().view(-1, 17, 4, 4) action, q_values = self.predict(state, legalActions) choice = self.actions[action] if self.learning: reward = self.gb.currentReward if reward != 0: reward = np.log2(reward) if (self.previous_state is not None and self.previous_action is not None): self.memory.add(self.previous_state, self.previous_action, self.previous_legal_actions, reward, legalActions, board, 0) self.previous_state = board self.previous_action = action self.previous_legal_actions = legalActions if not self.learning: state = torch.from_numpy(board).type( torch.FloatTensor).cuda().view(-1, 17, 4, 4) recon_batch, _, _ = self.model_vae(state) target_board = reverseOneHotMap(recon_batch.data.cpu().numpy()) #print(target_board.shape) self.totalCorrect += np.sum(self.gb.board == target_board) self.total += 16 self.acc = self.totalCorrect / self.total if self.learning: self.update() return choice def enableLearning(self): self.model_vae.train() self.model_dqn.train() self.learning = True self.max_tile = 0 self.reset() def disableLearning(self): self.model_vae.eval() self.model_dqn.eval() self.totalCorrect = 0 self.total = 0 self.acc = 0 self.learning = False def end_episode(self): if not self.learning: m = np.max(self.gb.board) if m > self.max_tile: self.max_tile = m return #print(self.gb.board) board = deepcopy(self.gb.board) board = oneHotMap(board) #legalActions = self.legal_actions(deepcopy(self.gb.board)) #print(legalActions) self.memory.add(self.previous_state, self.previous_action, self.previous_legal_actions, self.gb.currentReward, [], board, 1) self.reset() def reset(self): self.previous_state = None self.previous_action = None self.previous_legal_actions = None def update(self): if self.step < self.batch_size: return batch = self.memory.sample(self.batch_size, self.beta) (states, actions, legal_actions, reward, next_legal_actions, next_states, is_terminal, weights, batch_idxes) = batch batch_idx = 1 terminal = torch.tensor(is_terminal).type(torch.cuda.FloatTensor) reward = torch.tensor(reward).type(torch.cuda.FloatTensor) states = torch.from_numpy(states).type(torch.FloatTensor).cuda().view( -1, 17, 4, 4) next_states = torch.from_numpy(next_states).type( torch.FloatTensor).cuda().view(-1, 17, 4, 4) # Current Q Values q_actions, q_values, mu, logvar = self.predict_batch(states) batch_index = torch.arange(self.batch_size, dtype=torch.long) #print(actions) #print(q_values) #self.test_q = q_values q_values = q_values[batch_index, actions] #print(q_values) # Calculate target q_actions_next, q_values_next, _, _ = self.predict_batch_target( next_states, legalActions=next_legal_actions) q_max = q_values_next.max(1)[0].detach() q_max = (1 - terminal) * q_max q_target = reward + 0.99 * q_max recon_batch, mu, logvar = self.model_vae(states) self.opt_vae.zero_grad() self.opt_dqn.zero_grad() loss_vae = self.model_vae.loss_function(recon_batch, states, mu, logvar) loss_dqn = self.model_dqn.loss_function(q_target, q_values) loss_vae.backward() loss_dqn.backward() self.opt_vae.step() self.opt_dqn.step() #train_loss = loss_vae.item() + loss_dqn.item() self.loss_vae += loss_vae.item() / len(states) self.loss_dqn += loss_dqn.item() / len(states) # Update priorities td_errors = q_values - q_target new_priorities = torch.abs(td_errors) + 1e-6 # prioritized_replay_eps self.memory.update_priorities(batch_idxes, new_priorities.data) if self.step % 4000 == 0: self.model_dqn_target.load_state_dict(self.model_dqn.state_dict()) def predict_batch(self, input, legalActions=None): q_values, mu, logvar = self.model_dqn(input) if legalActions is None: values, q_actions = q_values.max(1) else: q_values_true = torch.full((self.batch_size, 4), -100000000).cuda() for i, action in enumerate(legalActions): q_values_true[i, action] = q_values[i, action] values, q_actions = q_values_true.max(1) q_values = q_values_true #print(q_values_true) return q_actions, q_values, mu, logvar def predict_batch_target(self, input, legalActions=None): q_values, mu, logvar = self.model_dqn_target(input) if legalActions is None: values, q_actions = q_values.max(1) else: q_values_true = torch.full((self.batch_size, 4), -100000000).cuda() for i, action in enumerate(legalActions): q_values_true[i, action] = q_values[i, action] values, q_actions = q_values_true.max(1) q_values = q_values_true #print(q_values_true) return q_actions, q_values, mu, logvar def predict(self, input, legalActions): q_values, mu, logvar = self.model_dqn(input) for action in range(4): if action not in legalActions: q_values[0, action] = -100000000 action = torch.argmax(q_values) if int(action.item()) not in legalActions: print(legalActions, q_values, action) print("!!!!!!!!!!!!!!!!!!!!!!!!!") return action.item(), q_values def legal_actions(self, copy_gb): legalActions = [] for i in range(4): try_gb = gameboard(4, deepcopy(copy_gb)) changed = try_gb.takeAction(self.actions[i]) if changed: legalActions.append(i) return legalActions
class DQNAgent_Vanila(agent): def __init__(self, model, opt, learning=True): super().__init__() self.memory = PrioritizedReplayBuffer(3000, 0.6) self.previous_state = None self.previous_action = None self.previous_legal_actions = None self.step = 0 self.model = model self.opt = opt self.loss = 0 self.batch_size = 32 self.test_q = 0 self.max_tile = 0 self.beta = 0.7 self.reconver_step = 0 #self.test_q = 0 self.epsilon_schedule = LinearSchedule(100000, initial_p=0.99, final_p=0.01) self.learning = learning def should_explore(self): self.epsilon = self.epsilon_schedule.value(self.step + self.reconver_step) return random.random() < self.epsilon def action(self): if self.learning: self.step += 1 legalActions = self.legal_actions(deepcopy(self.gb.board)) board = deepcopy(self.gb.board) board = oneHotMap(board) if self.learning and self.should_explore(): q_values = None action = random.choice(legalActions) choice = self.actions[action] else: #mark state = torch.from_numpy(board).type( torch.FloatTensor).cuda().view(-1, 17, 4, 4) action, q_values = self.predict(state, legalActions) choice = self.actions[action] if self.learning: reward = self.gb.currentReward if reward != 0: reward = np.log2(reward) if (self.previous_state is not None and self.previous_action is not None): self.memory.add(self.previous_state, self.previous_action, self.previous_legal_actions, reward, legalActions, board, 0) self.previous_state = board self.previous_action = action self.previous_legal_actions = legalActions if self.learning: self.update() return choice def enableLearning(self): self.model.train() self.learning = True self.max_tile = 0 self.reset() def disableLearning(self): self.model.eval() self.learning = False def end_episode(self): if not self.learning: m = np.max(self.gb.board) if m > self.max_tile: self.max_tile = m return #print(self.gb.board) board = deepcopy(self.gb.board) board = oneHotMap(board) #legalActions = self.legal_actions(deepcopy(self.gb.board)) #print(legalActions) self.memory.add(self.previous_state, self.previous_action, self.previous_legal_actions, self.gb.currentReward, [], board, 1) self.reset() def reset(self): self.previous_state = None self.previous_action = None self.previous_legal_actions = None def update(self): if self.step < self.batch_size: return batch = self.memory.sample(self.batch_size, self.beta) (states, actions, legal_actions, reward, next_legal_actions, next_states, is_terminal, weights, batch_idxes) = batch terminal = torch.tensor(is_terminal).type(torch.cuda.FloatTensor) reward = torch.tensor(reward).type(torch.cuda.FloatTensor) states = torch.from_numpy(states).type(torch.FloatTensor).cuda().view( -1, 17, 4, 4) next_states = torch.from_numpy(next_states).type( torch.FloatTensor).cuda().view(-1, 17, 4, 4) # Current Q Values _, q_values = self.predict_batch(states) batch_index = torch.arange(self.batch_size, dtype=torch.long) #print(actions) #print(q_values) q_values = q_values[batch_index, actions] #print(q_values) # Calculate target q_actions_next, q_values_next = self.predict_batch( next_states, legalActions=next_legal_actions) #print(q_values_next) q_max = q_values_next.max(1)[0].detach() q_max = (1 - terminal) * q_max # if sum(terminal == 1) > 0: # print(reward) # print( (terminal == 1).nonzero()) # print(terminal) # print(next_legal_actions) # print(q_max) # input() q_target = reward + 0.99 * q_max self.opt.zero_grad() loss = self.model.loss_function(q_target, q_values) loss.backward() self.opt.step() #train_loss = loss_vae.item() + loss_dqn.item() self.loss += loss.item() / len(states) # Update priorities td_errors = q_values - q_target new_priorities = torch.abs(td_errors) + 1e-6 # prioritized_replay_eps self.memory.update_priorities(batch_idxes, new_priorities.data) def predict_batch(self, input, legalActions=None): q_values = self.model(input) if legalActions is None: values, q_actions = q_values.max(1) else: q_values_true = torch.full((self.batch_size, 4), -100000000).cuda() for i, action in enumerate(legalActions): q_values_true[i, action] = q_values[i, action] values, q_actions = q_values_true.max(1) q_values = q_values_true #print(q_values_true) return q_actions, q_values def predict(self, input, legalActions): q_values = self.model(input) for action in range(4): if action not in legalActions: q_values[0, action] = -100000000 action = torch.argmax(q_values) return action.item(), q_values def legal_actions(self, copy_gb): legalActions = [] for i in range(4): try_gb = gameboard(4, deepcopy(copy_gb)) changed = try_gb.takeAction(self.actions[i]) if changed: legalActions.append(i) return legalActions '''
class train_DQN(): def __init__(self, env_id, max_step = 1e5, prior_alpha = 0.6, prior_beta_start = 0.4, epsilon_start = 1.0, epsilon_final = 0.01, epsilon_decay = 500, batch_size = 32, gamma = 0.99, target_update_interval=1000, save_interval = 1e4, ): self.prior_beta_start = prior_beta_start self.max_step = int(max_step) self.batch_size = batch_size self.gamma = gamma self.target_update_interval = target_update_interval self.save_interval = save_interval self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") self.env = gym.make(env_id) self.model = DuelingDQN(self.env).to(self.device) self.target_model = DuelingDQN(self.env).to(self.device) self.target_model.load_state_dict(self.model.state_dict()) self.replay_buffer = PrioritizedReplayBuffer(100000,alpha=prior_alpha) self.optimizer = optim.Adam(self.model.parameters()) self.writer = SummaryWriter(comment="-{}-learner".format(self.env.unwrapped.spec.id)) # decay function self.scheduler = optim.lr_scheduler.StepLR(self.optimizer,step_size=1000,gamma=0.99) self.beta_by_frame = lambda frame_idx: min(1.0, self.prior_beta_start + frame_idx * (1.0 - self.prior_beta_start) / 1000) self.epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay) def update_target(self,current_model, target_model): target_model.load_state_dict(current_model.state_dict()) def compute_td_loss(self,batch_size, beta): state, action, reward, next_state, done, weights, indices = self.replay_buffer.sample(batch_size, beta) state = torch.FloatTensor(state).to(self.device) next_state = torch.FloatTensor(next_state).to(self.device) action = torch.LongTensor(action).to(self.device) reward = torch.FloatTensor(reward).to(self.device) done = torch.FloatTensor(done).to(self.device) weights = torch.FloatTensor(weights).to(self.device) batch = (state, action, reward, next_state, done, weights) # q_values = self.model(state) # next_q_values = self.target_model(next_state) # q_value = q_values.gather(1, action.unsqueeze(1)).squeeze(1) # next_q_value = next_q_values.max(1)[0] # expected_q_value = reward + self.gamma * next_q_value * (1 - done) # td_error = torch.abs(expected_q_value.detach() - q_value) # loss = (td_error).pow(2) * weights # prios = loss+1e-5#0.9 * torch.max(td_error)+(1-0.9)*td_error # loss = loss.mean() loss, prios = utils.compute_loss(self.model,self.target_model, batch,1) self.optimizer.zero_grad() loss.backward() self.scheduler.step() self.replay_buffer.update_priorities(indices, prios) self.optimizer.step() return loss def train(self): losses = [] all_rewards = [] episode_reward = 0 episode_idx = 0 episode_length = 0 state = self.env.reset() for frame_idx in range(self.max_step): epsilon = self.epsilon_by_frame(frame_idx) action,_ = self.model.act(torch.FloatTensor((state)).to(self.device), epsilon) next_state, reward, done, _ = self.env.step(action) self.replay_buffer.add(state, action, reward, next_state, done) state = next_state episode_reward += reward episode_length += 1 if done: state = self.env.reset() all_rewards.append(episode_reward) self.writer.add_scalar("actor/episode_reward", episode_reward, episode_idx) self.writer.add_scalar("actor/episode_length", episode_length, episode_idx) # print("episode: ",episode_idx, " reward: ", episode_reward) episode_reward = 0 episode_length = 0 episode_idx += 1 if len(self.replay_buffer) > self.batch_size: beta = self.beta_by_frame(frame_idx) loss = self.compute_td_loss(self.batch_size, beta) losses.append(loss.item()) self.writer.add_scalar("learner/loss", loss, frame_idx) if frame_idx % self.target_update_interval == 0: print("update target...") self.update_target(self.model, self.target_model) if frame_idx % self.save_interval == 0 or frame_idx == self.max_step-1: print("save model...") self.save_model(frame_idx) def save_model(self, idx): torch.save(self.model.state_dict(), "./model{}.pth".format(idx)) def load_model(self,idx): with open("model{}.pth".format(idx), "rb") as f: print("loading weights_{}".format(idx)) self.model.load_state_dict(torch.load(f,map_location="cpu"))
def learn(env, q_func, lr=5e-4, max_timesteps=100000, buffer_size=50000, exploration_fraction=0.1, exploration_final_eps=0.02, train_freq=1, batch_size=32, print_freq=100, checkpoint_freq=10000, learning_starts=1000, gamma=1.0, target_network_update_freq=500, prioritized_replay=False, prioritized_replay_alpha=0.6, prioritized_replay_beta0=0.4, prioritized_replay_beta_iters=None, prioritized_replay_eps=1e-6, param_noise=False, callback=None): """Train a deepq model. Parameters ------- env: gym.Env environment to train on q_func: (tf.Variable, int, str, bool) -> tf.Variable the model that takes the following inputs: observation_in: object the output of observation placeholder num_actions: int number of actions scope: str reuse: bool should be passed to outer variable scope and returns a tensor of shape (batch_size, num_actions) with values of every action. lr: float learning rate for adam optimizer max_timesteps: int number of env steps to optimizer for buffer_size: int size of the replay buffer exploration_fraction: float fraction of entire training period over which the exploration rate is annealed exploration_final_eps: float final value of random action probability train_freq: int update the model every `train_freq` steps. set to None to disable printing batch_size: int size of a batched sampled from replay buffer for training print_freq: int how often to print out training progress set to None to disable printing checkpoint_freq: int how often to save the model. This is so that the best version is restored at the end of the training. If you do not wish to restore the best version at the end of the training set this variable to None. learning_starts: int how many steps of the model to collect transitions for before learning starts gamma: float discount factor target_network_update_freq: int update the target network every `target_network_update_freq` steps. prioritized_replay: True if True prioritized replay buffer will be used. prioritized_replay_alpha: float alpha parameter for prioritized replay buffer prioritized_replay_beta0: float initial value of beta for prioritized replay buffer prioritized_replay_beta_iters: int number of iterations over which beta will be annealed from initial value to 1.0. If set to None equals to max_timesteps. prioritized_replay_eps: float epsilon to add to the TD errors when updating priorities. callback: (locals, globals) -> None function called at every steps with state of the algorithm. If callback returns true training stops. Returns ------- act: ActWrapper Wrapper over act function. Adds ability to save it and load it. See header of baselines/deepq/categorical.py for details on the act function. """ # Create all the functions necessary to train the model sess = tf.Session() sess.__enter__() # capture the shape outside the closure so that the env object is not serialized # by cloudpickle when serializing make_obs_ph observation_space_shape = env.observation_space.shape def make_obs_ph(name): return BatchInput(observation_space_shape, name=name) act, train, update_target, debug = deepq.build_train( make_obs_ph=make_obs_ph, q_func=q_func, num_actions=env.action_space.n, optimizer=tf.train.AdamOptimizer(learning_rate=lr), gamma=gamma, grad_norm_clipping=10, param_noise=param_noise) act_params = { 'make_obs_ph': make_obs_ph, 'q_func': q_func, 'num_actions': env.action_space.n, } act = ActWrapper(act, act_params) # Create the replay buffer if prioritized_replay: replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha) if prioritized_replay_beta_iters is None: prioritized_replay_beta_iters = max_timesteps beta_schedule = LinearSchedule(prioritized_replay_beta_iters, initial_p=prioritized_replay_beta0, final_p=1.0) else: replay_buffer = ReplayBuffer(buffer_size) beta_schedule = None # Create the schedule for exploration starting from 1. exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps), initial_p=1.0, final_p=exploration_final_eps) # Initialize the parameters and copy them to the target network. U.initialize() update_target() episode_rewards = [0.0] saved_mean_reward = None obs = env.reset() reset = True with tempfile.TemporaryDirectory() as td: model_saved = False model_file = os.path.join(td, "model") for t in range(max_timesteps): if callback is not None: if callback(locals(), globals()): break # Take action and update exploration to the newest value kwargs = {} if not param_noise: update_eps = exploration.value(t) update_param_noise_threshold = 0. else: update_eps = 0. # Compute the threshold such that the KL divergence between perturbed and non-perturbed # policy is comparable to eps-greedy exploration with eps = exploration.value(t). # See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017 # for detailed explanation. update_param_noise_threshold = -np.log(1. - exploration.value( t) + exploration.value(t) / float(env.action_space.n)) kwargs['reset'] = reset kwargs[ 'update_param_noise_threshold'] = update_param_noise_threshold kwargs['update_param_noise_scale'] = True action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0] env_action = action reset = False new_obs, rew, done, _ = env.step(env_action) # Store transition in the replay buffer. replay_buffer.add(obs, action, rew, new_obs, float(done)) obs = new_obs episode_rewards[-1] += rew if done: obs = env.reset() episode_rewards.append(0.0) reset = True if t > learning_starts and t % train_freq == 0: # Minimize the error in Bellman's equation on a batch sampled from replay buffer. if prioritized_replay: experience = replay_buffer.sample( batch_size, beta=beta_schedule.value(t)) (obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience else: obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample( batch_size) weights, batch_idxes = np.ones_like(rewards), None td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights) if prioritized_replay: new_priorities = np.abs(td_errors) + prioritized_replay_eps replay_buffer.update_priorities(batch_idxes, new_priorities) if t > learning_starts and t % target_network_update_freq == 0: # Update target network periodically. update_target() mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1) num_episodes = len(episode_rewards) if done and print_freq is not None and len( episode_rewards) % print_freq == 0: logger.record_tabular("steps", t) logger.record_tabular("episodes", num_episodes) logger.record_tabular("mean 100 episode reward", mean_100ep_reward) logger.record_tabular("% time spent exploring", int(100 * exploration.value(t))) logger.dump_tabular() if (checkpoint_freq is not None and t > learning_starts and num_episodes > 100 and t % checkpoint_freq == 0): if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward: if print_freq is not None: logger.log( "Saving model due to mean reward increase: {} -> {}" .format(saved_mean_reward, mean_100ep_reward)) save_state(model_file) model_saved = True saved_mean_reward = mean_100ep_reward if model_saved: if print_freq is not None: logger.log("Restored model with mean reward: {}".format( saved_mean_reward)) load_state(model_file) return act