def main(_config): env = gym.make(_config.ENV_NAME) agent = DQN(env, _config) print("[*] --- Begin Emulator Training ---") for episode in range(_config.EPISODE): obs = env.reset() # === Emulator === for i in range(_config.STEP): action = agent.pick_action(obs) obs_next, reward, done, _ = env.step(action) # agent will store the newest experience into replay buffer, and training with mini-batch and off-policy agent.perceive(obs, action, reward, done) if done: break obs = obs_next # == train == agent.train(episode) if (episode + 1) % agent.save_every == 0: agent.save(step=episode) # == test == print("\n[*] === Enter TEST module ===") test(env, _config.STEP, agent) agent.record()
class Agent: """ The intelligent agent of the simulation. Set the model of the neural network used and general parameters. It is responsible to select the actions, optimize the neural network and manage the models. """ def __init__(self, action_set, train=True, load_path=None): #1. Initialize agent params self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.action_set = action_set self.action_number = len(action_set) self.steps_done = 0 self.epsilon = Config.EPS_START self.episode_durations = [] #2. Build networks self.policy_net = DQN().to(self.device) self.target_net = DQN().to(self.device) self.optimizer = optim.RMSprop(self.policy_net.parameters(), lr=Config.LEARNING_RATE) if not train: self.optimizer = optim.RMSprop(self.policy_net.parameters(), lr=0) self.policy_net.load(load_path, optimizer=self.optimizer) self.policy_net.eval() self.target_net.load_state_dict(self.policy_net.state_dict()) self.target_net.eval() #3. Create Prioritized Experience Replay Memory self.memory = Memory(Config.MEMORY_SIZE) def append_sample(self, state, action, next_state, reward): """ save sample (error,<s,a,s',r>) to the replay memory """ # Define if is the end of the simulation done = True if next_state is None else False # Compute Q(s_t, a) - the model computes Q(s_t), then we select the columns of actions taken state_action_values = self.policy_net(state) state_action_values = state_action_values.gather(1, action.view(-1,1)) if not done: # Compute argmax Q(s', a; θ) next_state_actions = self.policy_net(next_state).max(1)[1].detach().unsqueeze(1) # Compute Q(s', argmax Q(s', a; θ), θ-) next_state_values = self.target_net(next_state).gather(1, next_state_actions).squeeze(1).detach() # Compute the expected Q values expected_state_action_values = (next_state_values * Config.GAMMA) + reward else: expected_state_action_values = reward error = abs(state_action_values - expected_state_action_values).data.cpu().numpy() self.memory.add(error, state, action, next_state, reward) def select_action(self, state, train=True): """ Selet the best action according to the Q-values outputed from the neural network Parameters ---------- state: float ndarray The current state on the simulation train: bool Define if we are evaluating or trainning the model Returns ------- a.max(1)[1]: int The action with the highest Q-value a.max(0): float The Q-value of the action taken """ global steps_done sample = random.random() #1. Perform a epsilon-greedy algorithm #a. set the value for epsilon self.epsilon = Config.EPS_END + (Config.EPS_START - Config.EPS_END) * \ math.exp(-1. * self.steps_done / Config.EPS_DECAY) self.steps_done += 1 #b. make the decision for selecting a random action or selecting an action from the neural network if sample > self.epsilon or (not train): # select an action from the neural network with torch.no_grad(): # a <- argmax Q(s, theta) a = self.policy_net(state) return a.max(1)[1].view(1, 1), a.max(0) else: # select a random action print('random action') return torch.tensor([[random.randrange(2)]], device=self.device, dtype=torch.long), None """ def select_action(self, state, train=True): Selet the best action according to the Q-values outputed from the neural network Parameters ---------- state: float ndarray The current state on the simulation train: bool Define if we are evaluating or trainning the model Returns ------- a.max(1)[1]: int The action with the highest Q-value a.max(0): float The Q-value of the action taken global steps_done sample = random.random() #1. Perform a epsilon-greedy algorithm #a. set the value for epsilon self.epsilon = Config.EPS_END + (Config.EPS_START - Config.EPS_END) * \ math.exp(-1. * self.steps_done / Config.EPS_DECAY) self.steps_done += 1 #b. make the decision for selecting a random action or selecting an action from the neural network if sample > self.epsilon or (not train): # select an action from the neural network with torch.no_grad(): # a <- argmax Q(s, theta) #set the network to train mode is important to enable dropout self.policy_net.train() output_list = [] # Retrieve the outputs from neural network feedfoward n times to build a statistic model for i in range(Config.STOCHASTIC_PASSES): #print(agent.policy_net(data)) output_list.append(torch.unsqueeze(F.softmax(self.policy_net(state)), 0)) #print(output_list[i]) self.policy_net.eval() # The result of the network is the mean of n passes output_mean = torch.cat(output_list, 0).mean(0) q_value = output_mean.data.cpu().numpy().max() action = output_mean.max(1)[1].view(1, 1) uncertainty = torch.cat(output_list, 0).var(0).mean().item() return action, q_value, uncertainty else: # select a random action print('random action') return torch.tensor([[random.randrange(2)]], device=self.device, dtype=torch.long), None, None """ def optimize_model(self): """ Perform one step of optimization on the neural network """ if self.memory.tree.n_entries < Config.BATCH_SIZE: return transitions, idxs, is_weights = self.memory.sample(Config.BATCH_SIZE) # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for detailed explanation). batch = Transition(*zip(*transitions)) # Compute a mask of non-final states and concatenate the batch elements non_final_mask = torch.tensor(tuple(map(lambda s: s is not None, batch.next_state)), device=self.device, dtype=torch.uint8) non_final_next_states = torch.cat([s for s in batch.next_state if s is not None]) state_batch = torch.cat(batch.state) action_batch = torch.cat(batch.action) reward_batch = torch.cat(batch.reward) # Compute Q(s_t, a) - the model computes Q(s_t), then we select the columns of actions taken state_action_values = self.policy_net(state_batch).gather(1, action_batch) # Compute argmax Q(s', a; θ) next_state_actions = self.policy_net(non_final_next_states).max(1)[1].detach().unsqueeze(1) # Compute Q(s', argmax Q(s', a; θ), θ-) next_state_values = torch.zeros(Config.BATCH_SIZE, device=self.device) next_state_values[non_final_mask] = self.target_net(non_final_next_states).gather(1, next_state_actions).squeeze(1).detach() # Compute the expected Q values expected_state_action_values = (next_state_values * Config.GAMMA) + reward_batch # Update priorities errors = torch.abs(state_action_values.squeeze() - expected_state_action_values).data.cpu().numpy() # update priority for i in range(Config.BATCH_SIZE): idx = idxs[i] self.memory.update(idx, errors[i]) # Compute Huber loss loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1)) loss_return = loss.item() # Optimize the model self.optimizer.zero_grad() loss.backward() for param in self.policy_net.parameters(): param.grad.data.clamp_(-1, 1) self.optimizer.step() return loss_return def save(self, step, logs_path, label): """ Save the model on hard disc Parameters ---------- step: int current step on the simulation logs_path: string path to where we will store the model label: string label that will be used to store the model """ os.makedirs(logs_path + label, exist_ok=True) full_label = label + str(step) + '.pth' logs_path = os.path.join(logs_path, label, full_label) self.policy_net.save(logs_path, step=step, optimizer=self.optimizer) def restore(self, logs_path): """ Load the model from hard disc Parameters ---------- logs_path: string path to where we will store the model """ self.policy_net.load(logs_path) self.target_net.load(logs_path)
import time from model import DQN from rewards import CustomReward # CONFIG wave = True learn_timesteps = 1 * int(1e3) if wave: import arlie env = arlie.make("LunarLander", render_mode=False, reward=CustomReward()) else: import gym env = gym.make("LunarLander-v2") model = DQN(env) print("Training...") _t = time.time() model.learn(total_timesteps=learn_timesteps) t = time.time() - _t str_t = time.strftime("%H h, %M m, %S s", time.gmtime(t)) print("Trained in {} during {} timesteps".format(str_t, learn_timesteps)) model.save("{}-trained-model".format("wave" if wave else "gym")) env.close()
rewards = np.array(winner_rewards + loser_rewards) boards = np.concatenate([boards[winner], boards[loser]]) else: #tie one_rewards = [0] * len(boards[1]) two_rewards = [0] * len(boards[2]) rewards = np.array(one_rewards + two_rewards) boards = np.concatenate([boards[1], boards[2]]) rewards = rewards.reshape(rewards.shape[0], -1) model.train(boards, rewards) if games % EPOCH == 0: gamma *= 1.0 win_rate.append(test_against_random(model)) debug_run(model) if games % TEST_FRQ == 0: print(win_rate) plt.plot(win_rate) plt.ylabel('Winning Percentage') plt.xlabel('Epochs') plt.show() model.save('./dqn_no/') exit()
# Transition transition.append(state) if len(transition) > 4: transition.pop(0) if model.step > model.train_start_step and model.step % model.train_step_interval: model.train() if model.step % model.target_update_interval == 0: model.update_target() if is_render: env.render() if done: writer.add_scalar('reward/accum', accum_reward, model.step) writer.add_scalar('data/epsilon', model.epsilon, model.step) writer.add_scalar('data/x_pos', info['x_pos'], model.step) print( "Episode : %5d\t\tSteps : %10d\t\tReward : %7d\t\tX_step : %4d\t\tEpsilon : %.3f" % (model.episode, model.step, accum_reward, info['x_pos'], model.epsilon)) if save_model and model.episode % 100 == 0: model.save() break env.close()
def main(): ## arguments ## parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-d', '--device', default='cuda') parser.add_argument( '--checkpoint', action='store_true', help='save model every epoch, ER buffer will only be saved as one file' ) parser.add_argument('--logdir', default='log/dqn') # train parser.add_argument('--warmup', default=10000, type=int) parser.add_argument('--epochs', default=1200, type=int) parser.add_argument('--steps', default=1000000, type=int, help='steps per epoch') parser.add_argument('--capacity', default=1000000, type=int) parser.add_argument('--batch_size', default=32, type=int) parser.add_argument('--lr', default=0.00001, type=float) parser.add_argument('--momentum', default=0.95, type=float) parser.add_argument('--epsilon', default=1000000, type=int, help='decay steps from eps_max to eps_min') parser.add_argument('--eps_min', default=.1, type=float) parser.add_argument('--gamma', default=.99, type=float) parser.add_argument('--freq', default=4, type=int) parser.add_argument('--target_freq', default=10000, type=int) # test parser.add_argument('--test_only', action='store_true') parser.add_argument('--test_steps', type=int, default=500000) parser.add_argument('--render', action='store_true') parser.add_argument('--test_epsilon', default=0.05, type=float) # average parser.add_argument( '-k', '--k', type=int, default=1, help='number of average target network, if k = 1, perform vanilla DQN') # resume training parser.add_argument('--resume', action='store_true', help='resume training') parser.add_argument('-m', '--model', default=None, help='model path') # DDQN parser.add_argument('--ddqn', action='store_true', help='perform Double-DQN') args = parser.parse_args() ## main ## env = gym.make('BreakoutNoFrameskip-v4') # frame stack and preprocessing env = AtariPreprocessing(env, noop_max=30, frame_skip=4) env = FrameStack(env, 4) agent = DQN(args, env) #writer = SummaryWriter(args.logdir) if not args.test_only: train(args, env, agent) agent.save(args.model)
class Agent: """ The intelligent agent of the simulation. Set the model of the neural network used and general parameters. It is responsible to select the actions, optimize the neural network and manage the models. """ def __init__(self, action_set, train=True, load_path=None): #1. Initialize agent params self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.action_set = action_set self.action_number = len(action_set) self.steps_done = 0 self.epsilon = Config.EPS_START self.episode_durations = [] print('LOAD PATH -- agent.init:', load_path) time.sleep(2) #2. Build networks self.policy_net = DQN().to(self.device) self.target_net = DQN().to(self.device) self.optimizer = optim.RMSprop(self.policy_net.parameters(), lr=Config.LEARNING_RATE) if not train: print('entrou no not train') self.optimizer = optim.RMSprop(self.policy_net.parameters(), lr=0) self.policy_net.load(load_path, optimizer=self.optimizer) self.policy_net.eval() self.target_net.load_state_dict(self.policy_net.state_dict()) self.target_net.eval() self.memory = ReplayMemory(1000) def select_action(self, state, train=True): """ Selet the best action according to the Q-values outputed from the neural network Parameters ---------- state: float ndarray The current state on the simulation train: bool Define if we are evaluating or trainning the model Returns ------- a.max(1)[1]: int The action with the highest Q-value a.max(0): float The Q-value of the action taken """ global steps_done sample = random.random() #1. Perform a epsilon-greedy algorithm #a. set the value for epsilon self.epsilon = Config.EPS_END + (Config.EPS_START - Config.EPS_END) * \ math.exp(-1. * self.steps_done / Config.EPS_DECAY) self.steps_done += 1 #b. make the decision for selecting a random action or selecting an action from the neural network if sample > self.epsilon or (not train): # select an action from the neural network with torch.no_grad(): # a <- argmax Q(s, theta) a = self.policy_net(state) return a.max(1)[1].view(1, 1), a.max(0) else: # select a random action print('random action') return torch.tensor([[random.randrange(2)]], device=self.device, dtype=torch.long), None def optimize_model(self): """ Perform one step of optimization on the neural network """ if len(self.memory) < Config.BATCH_SIZE: return transitions = self.memory.sample(Config.BATCH_SIZE) # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for detailed explanation). batch = Transition(*zip(*transitions)) # Compute a mask of non-final states and concatenate the batch elements non_final_mask = torch.tensor(tuple(map(lambda s: s is not None, batch.next_state)), device=self.device, dtype=torch.uint8) non_final_next_states = torch.cat([s for s in batch.next_state if s is not None]) state_batch = torch.cat(batch.state) action_batch = torch.cat(batch.action) reward_batch = torch.cat(batch.reward) # Compute Q(s_t, a) - the model computes Q(s_t), then we select the columns of actions taken state_action_values = self.policy_net(state_batch).gather(1, action_batch) # Compute argmax Q(s', a; θ) next_state_actions = self.policy_net(non_final_next_states).max(1)[1].detach().unsqueeze(1) # Compute Q(s', argmax Q(s', a; θ), θ-) next_state_values = torch.zeros(Config.BATCH_SIZE, device=self.device) next_state_values[non_final_mask] = self.target_net(non_final_next_states).gather(1, next_state_actions).squeeze(1).detach() # Compute the expected Q values expected_state_action_values = (next_state_values * Config.GAMMA) + reward_batch # Compute Huber loss loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1)) # Optimize the model self.optimizer.zero_grad() loss.backward() for param in self.policy_net.parameters(): param.grad.data.clamp_(-1, 1) self.optimizer.step() def save(self, step, logs_path, label): """ Save the model on hard disc Parameters ---------- step: int current step on the simulation logs_path: string path to where we will store the model label: string label that will be used to store the model """ os.makedirs(logs_path + label, exist_ok=True) full_label = label + str(step) + '.pth' logs_path = os.path.join(logs_path, label, full_label) self.policy_net.save(logs_path, step=step, optimizer=self.optimizer) def restore(self, logs_path): """ Load the model from hard disc Parameters ---------- logs_path: string path to where we will store the model """ self.policy_net.load(logs_path) self.target_net.load(logs_path)
class Agent: """ Class representing a learning agent acting in an environment. """ def __init__(self, buffer_size, batch_size, alpha, gamma, epsilon, epsilon_min, epsilon_decay, lr, game="CartPole-v1", mean_bound=5, reward_bound=495.0, sync_model=1000, save_model=10): """ Constructor of the agent class. - game="CartPole-v1" : Name of the game environment - mean_bound=5 : Number of last acquired rewards considered for mean reward - reward_bound=495.0 : Reward acquired for completing an episode properly - sync_model=1000 : Interval for synchronizing model and target model - save_model=10 : Interval for saving model - buffer_size : Replay buffer size of the DQN model - batch_size : Batch size of the DQN model - alpha : Learning rate for Q-Learning - gamma : Discount factor for Q-Learning - epsilon : Threshold for taking a random action - epsilon_min : Minimal value allowed for epsilon - epsilon_decay : Decay rate for epsilon - lr : Learning rate for the DQN model """ # Environment variables self.game = game self.env = gym.make(self.game) self.num_states = self.env.observation_space.shape[0] self.num_actions = self.env.action_space.n # Agent variables self.buffer_size = buffer_size self.batch_size = batch_size self.buffer = ReplayBuffer(self.buffer_size, self.batch_size) self.alpha = alpha self.gamma = gamma self.epsilon = epsilon self.epsilon_min = epsilon_min self.epsilon_decay = epsilon_decay self.mean_bound = mean_bound self.reward_bound = reward_bound # DQN variables self.lr = lr self.model = DQN(self.num_states, self.num_actions, self.lr) self.target_model = DQN(self.num_states, self.num_actions, self.lr) self.target_model.update(self.model) self.sync_model = sync_model self.save_model = save_model # File paths dirname = os.path.dirname(__file__) self.path_model = os.path.join(dirname, "../models/dqn.h5") self.path_plot = os.path.join(dirname, "../plots/dqn.png") # Load model, if it already exists try: self.model.load(self.path_model) self.target_model.update(self.model) except: print("Model does not exist! Create new model...") def reduce_epsilon(self): """ Reduces the parameter epsilon up to a given minimal value where the speed of decay is controlled by some given parameter. """ epsilon = self.epsilon * self.epsilon_decay if epsilon >= self.epsilon_min: self.epsilon = epsilon else: self.epsilon = self.epsilon_min def get_action(self, state): """ Returns an action for a given state, based on the current policy. - state : Current state of the agent """ if np.random.random() < self.epsilon: action = self.env.action_space.sample() else: action = np.argmax(self.model.predict(state)) return action def train(self, num_episodes, report_interval): """ Trains the DQN model for a given number of episodes. Outputting report information is controlled by a given time interval. - num_episodes : Number of episodes to train - report_interval : Interval for outputting report information of training """ step = 0 total_rewards = [] for episode in range(1, num_episodes + 1): if episode % self.save_model == 0: self.model.save(self.path_model) state = self.env.reset() state = state.reshape((1, self.num_states)) total_reward = 0.0 while True: step += 1 action = self.get_action(state) next_state, reward, done, _ = self.env.step(action) next_state = next_state.reshape((1, self.num_states)) # Penalize agent if pole could not be balanced until end of episode if done and reward < 499.0: reward = -100.0 self.buffer.remember(state, action, reward, next_state, done) self.replay() self.reduce_epsilon() state = next_state total_reward += reward if step % self.sync_model == 0: self.target_model.update(self.model) if done: total_reward += 100.0 total_rewards.append(total_reward) mean_reward = np.mean(total_rewards[-self.mean_bound:]) if episode % report_interval == 0: print(f"Episode: {episode}/{num_episodes}" f"\tStep: {step}" f"\tMemory Size: {len(self.memory)}" f"\tEpsilon: {self.epsilon : .3f}" f"\tReward: {total_reward}" f"\tLast 5 Mean: {mean_reward : .2f}") self.plot_rewards(total_rewards) if mean_reward > self.reward_bound: self.model.save(self.path_model) return break self.model.save(self.path_model) def replay(self): """ Samples training data from the replay buffer and fits the DQN model. """ sample_size, states, actions, rewards, next_states, dones = self.memory.sample( ) q_values = self.model.predict(states) next_q_values = self.target_model.predict(next_states) for i in range(sample_size): action = actions[i] done = dones[i] if done: q_target = rewards[i] else: q_target = rewards[i] + self.gamma * np.max(next_q_values[i]) q_values[i][action] = (1 - self.alpha) * \ q_values[i][action] + self.alpha * q_target self.model.fit(states, q_values) def play(self, num_episodes): """ Renders the trained agent for a given number of episodes. - num_episodes : Number of episodes to render """ self.epsilon = self.epsilon_min for episode in range(1, num_episodes + 1): state = self.env.reset() state = state.reshape((1, self.num_states)) total_reward = 0.0 while True: self.env.render() action = self.get_action(state) next_state, reward, done, _ = self.env.step(action) next_state = next_state.reshape((1, self.num_states)) state = next_state total_reward += reward if done: print(f"Episode: {episode}/{num_episodes}" f"\tTotal Reward: {total_reward : .2f}") break def plot_rewards(self, total_rewards): """ Plots the rewards the agent has acquired during training. - total_rewards : Rewards the agent has gained per episode """ x = range(len(total_rewards)) y = total_rewards slope, intercept, _, _, _ = linregress(x, y) plt.plot(x, y, linewidth=0.8) plt.plot(x, slope * x + intercept, color="red", linestyle="-.") plt.xlabel("Episode") plt.ylabel("Reward") plt.title("DQN-Learning") plt.savefig(self.path_plot)
s_[3]) # 增加当局的reward reward_counter += r if do_train: # 保存memory dqn.store(s, a, r, done, s_) # train network losses = dqn.train_net() s = s_ # 帧数自增 last_frame_counter += 1 if last_frame_counter >= 200: done = True # 如果达到保存条件,则保存模型 if do_save and do_train: dqn.save(save_path) do_save = False # 如果结束,打印当局数据,判断是否保存模型 if done: mean_array[mean_counter % mean_size] = reward_counter mean_counter += 1 # 如果mean满数据,并且达到目标,则保存模型,并且开启保存模型 if mean_counter > mean_size and np.mean( mean_array) > target_reward and do_train: render_show = True do_save = True # 打印当局的数据 epilist.append(episode) meanrdlist.append(np.mean(mean_array)) rewardlist.append(reward_counter) print(
# -abs term代表鼓勵agent不要去移動車子, # 一直維持在中間才能獲得很高的獎賞! # 2. r2得到的是角度的資訊, # 儘量讓棒子跟垂直線的角度愈小愈好(就是讓棒子立正), # 角度愈小獎賞愈高 # 最後扣0.5是讓獎勵區間分布於[-1~1]之間 # ---------------------------------------------------- x, x_dot, theta, theta_dot = s_ r1 = (env.x_threshold - abs(x)) / env.x_threshold - 0.8 r2 = (env.theta_threshold_radians - abs(theta)) / env.theta_threshold_radians - 0.5 r = r1 + r2 # Sotre the transition and update the network net.store_path(s, a, r, s_) net.update() # Judge if finish an episode (and decay the epsilon) if finish: print("Episode: %d \t Total reward: %d \t Eps: %f" % (i, total_reward, net.epsilon)) reward_list.append(total_reward) if net.epsilon > 0.01: net.episodeDecay() break # Update the current state as the future state of previous state s = s_ net.save() plt.plot(range(len(reward_list)), reward_list, '-') plt.show()
targets[1] = [0] * len(rewards[1]) targets[1][-1] = float(rewards[1][-1]) for i in reversed(range(len(rewards[1]) - 1)): targets[1][i] = float(lam * vals[1][i + 1]) targets[2] = [0] * len(rewards[2]) targets[2][-1] = float(rewards[2][-1]) for i in reversed(range(len(rewards[2]) - 1)): targets[2][i] = float(lam * vals[2][i + 1]) boards = np.array(boards[1] + boards[2]) targets = np.expand_dims(np.array(targets[1] + targets[2]), 1) model.train(boards, targets) if games % EPOCH == 0: gamma *= .99 win_rate.append(test_against_random(model)) debug_run(model) if games % TEST_FRQ == 0: print(win_rate) plt.plot(win_rate) plt.ylabel('Winning Percentage') plt.xlabel('Epochs') plt.show() model.save('./temp/') exit()
def main(opt): dqn = DQN(NUM_STATES, NUM_ACTIONS, opt.eps, opt) episodes = opt.num_episodes print("Collecting Experience....") reward_list = [] reward_list_mean = [] win_list = [] win_list_mean = [] test_reward_rand = [] test_win_rand = [] test_reward_max = [] test_win_max = [] test_reward_self = [] test_win_self = [] test_reward_exact = [] test_win_exact = [] plt.ion() fig, ax = plt.subplots() fig2, ax2 = plt.subplots() fig3, ax3 = plt.subplots() for i in range(episodes): dqn.ep_decay(episodes, i) ep_reward, ep_win = train_ep(dqn, opt.opp_policy) r = copy.copy(ep_reward) reward_list.append(r) reward_list_mean.append(np.mean(reward_list[-10:])) win_list.append(ep_win) win_list_mean.append(np.mean(win_list[-10:])) ax.set_xlim(0, episodes) print( "episode: {} , the episode reward is {}, average of last 10 eps is {}, win = {}, win_mean = {}" .format(i, ep_reward, reward_list_mean[-1], win_list[-1], win_list_mean[-1])) if i % 100 == 99 or i == 0: if i < episodes - 50: test_epsilon = 0.05 else: test_epsilon = 1e-2 t_reward_rand, t_win_rand = test_ep(dqn, 'random', opt.num_test, test_epsilon) t_reward_max, t_win_max = test_ep(dqn, 'max', opt.num_test, test_epsilon) t_reward_self, t_win_self = test_ep(dqn, 'self', opt.num_test, test_epsilon) t_reward_exact, t_win_exact = test_ep(dqn, 'exact', opt.num_test, test_epsilon) # t_reward,t_win = test_ep(dqn, 'max', opt.num_test) print('[random policy] test: {}, test_reward: {}, test_win: {}'. format(i, t_reward_rand, t_win_rand)) print( '[max policy] test: {}, test_reward: {}, test_win: {}'.format( i, t_reward_max, t_win_max)) print( '[self policy] test: {}, test_reward: {}, test_win: {}'.format( i, t_reward_self, t_win_self)) print('[exact policy] test: {}, test_reward: {}, test_win: {}'. format(i, t_reward_exact, t_win_exact)) test_reward_rand.append(t_reward_rand) test_win_rand.append(t_win_rand * 100) test_reward_max.append(t_reward_max) test_win_max.append(t_win_max * 100) test_reward_self.append(t_reward_self) test_win_self.append(t_win_self * 100) test_reward_exact.append(t_reward_exact) test_win_exact.append(t_win_exact * 100) # SAVE s = os.path.join(opt.save_path, 'p1-' + str(i).zfill(5)) print("saving model at episode %i in save_path=%s" % (i, s)) dqn.save(s + '.pth') np.savez(s + '-test_reward_rand', test_reward_rand) np.savez(s + '-test_win_rand', test_win_rand) np.savez(s + '-test_reward_max', test_reward_max) np.savez(s + '-test_win_max', test_win_max) np.savez(s + '-test_reward_max', test_reward_self) np.savez(s + '-test_win_max', test_win_self) np.savez(s + '-test_reward_max', test_reward_exact) np.savez(s + '-test_win_max', test_win_exact) np.savez(s + '-train_reward', reward_list) np.savez(s + '-train_reward_mean', reward_list_mean) # PLOT fig.suptitle('[Train] Score over Number of Episodes') ax.set_xlabel('Number of Training Episodes') ax.set_ylabel('Score') ax.plot(reward_list, 'g-', label='current') ax.plot(reward_list_mean, 'r-', label='ema') # ax.plot(np.array(win_list_mean)*100, 'b-', label='win_rate') fig2.suptitle('[Test] Score over Number of Episodes') ax2.set_xlabel('Number of Training Episodes (Hundreds)') ax2.set_ylabel('Mean Score') ax2.plot(test_reward_rand, 'r-', label='vs random policy') ax2.plot(test_reward_max, 'g-', label='vs max policy') ax2.plot(test_reward_self, 'b-', label='vs self policy') ax2.plot(test_reward_exact, 'y-', label='vs exact policy') fig3.suptitle('[Test] Win rate of agent') ax3.set_xlabel('Number of Training Episodes (Hundreds)') ax3.set_ylabel('Win rate (%)') ax3.plot(test_win_rand, 'r-', label='vs random policy') ax3.plot(test_win_max, 'g-', label='vs max policy') ax3.plot(test_win_self, 'b-', label='vs self policy') ax3.plot(test_win_exact, 'y-', label='vs exact policy') plt.pause(0.001) if i == 0: fig.legend(loc='lower right', bbox_to_anchor=(0.9, 0.11)) fig2.legend(loc='lower right', bbox_to_anchor=(0.9, 0.11)) fig3.legend(loc='lower right', bbox_to_anchor=(0.9, 0.11)) fig.savefig(os.path.join(opt.save_path, 'p1-train-rewards.png')) fig2.savefig(os.path.join(opt.save_path, 'p1-test-rewards.png')) fig3.savefig(os.path.join(opt.save_path, 'p1-win-rates.png')) print('vs. random policy') _, _ = test_ep(dqn, 'random', 1, 1e-2, render=True) print('vs. max policy') _, _ = test_ep(dqn, 'max', 1, 1e-2, render=True) print('vs. self') _, _ = test_ep(dqn, 'self', 1, 1e-2, render=True)