def evaluate(policy_model, action_space, episode_true, epsilon=0.01, num_episode=10): """ Evaluate the performance of the agent policy_model: the agent model action_space: action space dimension of game episode_true: the training episodes epsilon : the probability of agent choose random action num_episode : the test episodes of evaluation process """ env = atari_wrapper.make_atari('RiverraidNoFrameskip-v4') env = atari_wrapper.wrap_deepmind(env, clip_rewards=False, frame_stack=True, pytorch_img=True) test_scores = [] score = 0 episode = 0 while episode < num_episode: observation = env.reset() done = False while not done: t_observation = trace.from_numpy(observation).float() / 255 t_observation = t_observation.view( 1, t_observation.shape[0], # t_observation.shape:torch.Size([1, 4, 84, 84]) t_observation.shape[1], t_observation.shape[2]) if random.random() > epsilon: # choose action by epsilon-greedy q_value = policy_model(t_observation) action = q_value.argmax(1).data.cpu().numpy().astype(int)[0] else: action = random.sample(range(len(action_space)), 1)[0] next_observation, reward, done, info = env.step( action_space[action]) observation = next_observation score += reward if info['ale.lives'] == 0: test_scores.append(score) episode += 1 score = 0 f = open("file.txt", 'a') f.write("%f, %d, %d\n" % (float(sum(test_scores)) / float(num_episode), episode_true, num_episode)) f.close() mean_reward = float(sum(test_scores)) / float(num_episode) return mean_reward
def sub_env_creator(): sub_env = make_atari(env_name) sub_env.seed(seed + env_num) if env_num == 0 and num_envs > 1: # Wrap first env in default monitor for video output # Results will be transformed into baselines monitor style at the end of the run sub_env = gym.wrappers.Monitor(sub_env, results_save_dir) else: # Wrap every other env in the baselines monitor for equivalent plotting. sub_env = Monitor(sub_env, join(results_save_dir, str(env_num))) sub_env = wrap_deepmind(sub_env, frame_stack=True, scale=True) return sub_env
def main(argv): env = make_atari(FLAGS.env) env = wrap_deepmind(env, frame_stack=True) if FLAGS.agent == 'Rainbow': FLAGS.network = 'Dueling_Net' FLAGS.multi_step = 3 FLAGS.category = True FLAGS.noise = True message = OrderedDict({ "Env": env, "Agent": FLAGS.agent, "Network": FLAGS.network, "Episode": FLAGS.n_episode, "Max_Step": FLAGS.step, "Categorical": FLAGS.category, "init_model": FLAGS.model }) out_dim = set_output_dim(FLAGS, env.action_space.n) agent = eval(FLAGS.agent)(model=set_model(outdim=out_dim), n_actions=env.action_space.n, n_features=env.observation_space.shape, learning_rate=0, e_greedy=0, reward_decay=0, replace_target_iter=0, e_greedy_increment=0, optimizer=None, network=FLAGS.network, trainable=False, is_categorical=FLAGS.category, is_noise=FLAGS.noise, gpu=find_gpu()) if FLAGS.agent == 'PolicyGradient': trainer = PolicyTrainer(agent=agent, env=env, n_episode=FLAGS.n_episode, max_step=FLAGS.step, replay_size=0, data_size=0, n_warmup=0, priority=None, multi_step=0, render=FLAGS.render, test_episode=5, test_interval=0, test_frame=FLAGS.rec, test_render=FLAGS.test_render, metrics=message, init_model_dir=FLAGS.model) elif FLAGS.agent == 'A3C' or FLAGS.agent == 'Ape_X': trainer = DistributedTrainer(agent=agent, n_workers=0, env=env, n_episode=FLAGS.n_episode, max_step=FLAGS.step, replay_size=0, data_size=0, n_warmup=0, priority=None, multi_step=0, render=False, test_episode=5, test_interval=0, test_frame=FLAGS.rec, test_render=FLAGS.test_render, metrics=message, init_model_dir=FLAGS.model) else: trainer = Trainer(agent=agent, env=env, n_episode=FLAGS.n_episode, max_step=FLAGS.step, replay_size=0, data_size=0, n_warmup=0, priority=None, multi_step=0, render=FLAGS.render, test_episode=5, test_interval=0, test_frame=FLAGS.rec, test_render=FLAGS.test_render, metrics=message, init_model_dir=FLAGS.model) trainer.test() return
def main(): use_cuda = torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") print("use_cuda: ", use_cuda) print("Device: ", device) env = atari_wrapper.make_atari('RiverraidNoFrameskip-v4') env = atari_wrapper.wrap_deepmind(env, clip_rewards=False, frame_stack=True, pytorch_img=True) action_space = [a for a in range(env.action_space.n)] n_action = len(action_space) # DQN Model and optimizer: policy_model = DQNModel().to(device) target_model = DQNModel().to(device) target_model.load_state_dict(policy_model.state_dict()) optimizer = torch.optim.RMSprop(policy_model.parameters(), lr=lr, alpha=alpha) # Initialize the Replay Buffer replay_buffer = ReplayBuffer(rep_buf_size) while len(replay_buffer) < rep_buf_ini: observation = env.reset() done = False while not done: with torch.no_grad(): t_observation = torch.from_numpy(observation).float().to( device) t_observation = t_observation.view(1, t_observation.shape[0], t_observation.shape[1], t_observation.shape[2]) action = random.sample(range(len(action_space)), 1)[0] next_observation, reward, done, info = env.step( action_space[action]) replay_buffer.push(observation, action, reward, next_observation, done) observation = next_observation print('Experience Replay buffer initialized') # Use log to record the performance logger = logging.getLogger('dqn_Riverraid') logger.setLevel(logging.INFO) logger_handler = logging.FileHandler('./dqn_Riverraid.log') logger.addHandler(logger_handler) # Training part env.reset() score = 0 episode_score = [] mean_episode_score = [] episode_true = 0 num_frames = 0 episode = 0 last_100episode_score = deque(maxlen=100) while episode < max_episodes: observation = env.reset() done = False # import time # start=time.time() while not done: with torch.no_grad(): t_observation = torch.from_numpy(observation).float().to( device) / 255 t_observation = t_observation.view(1, t_observation.shape[0], t_observation.shape[1], t_observation.shape[2]) epsilon = epsilon_by_frame(num_frames) if random.random() > epsilon: q_value = policy_model(t_observation) action = q_value.argmax(1).data.cpu().numpy().astype( int)[0] else: action = random.sample(range(len(action_space)), 1)[0] next_observation, reward, done, info = env.step( action_space[action]) num_frames += 1 score += reward replay_buffer.push(observation, action, reward, next_observation, done) observation = next_observation # Update policy if len(replay_buffer ) > batch_size and num_frames % skip_frame == 0: observations, actions, rewards, next_observations, dones = replay_buffer.sample( batch_size) observations = torch.from_numpy(np.array(observations) / 255).float().to(device) actions = torch.from_numpy( np.array(actions).astype(int)).float().to(device) actions = actions.view(actions.shape[0], 1) rewards = torch.from_numpy( np.array(rewards)).float().to(device) rewards = rewards.view(rewards.shape[0], 1) next_observations = torch.from_numpy( np.array(next_observations) / 255).float().to(device) dones = torch.from_numpy( np.array(dones).astype(int)).float().to(device) dones = dones.view(dones.shape[0], 1) q_values = policy_model(observations) next_q_values = target_model(next_observations) q_value = q_values.gather(1, actions.long()) next_q_value = next_q_values.max(1)[0].unsqueeze(1) expected_q_value = rewards + gamma * next_q_value * (1 - dones) loss = huber_loss(q_value, expected_q_value) optimizer.zero_grad() loss.backward() optimizer.step() for target_param, policy_param in zip( target_model.parameters(), policy_model.parameters()): target_param.data.copy_(TAU * policy_param.data + (1 - TAU) * target_param.data) episode += 1 # episode_score.append(score) # end=time.time() # print("Running time ( %i episode): %.3f Seconds "%(episode ,end-start)) if info['ale.lives'] == 0: # episode_score.append(score) mean_score = score episode_true += 1 score = 0 # if episode % 20 == 0: # mean_score = np.mean(episode_score) mean_episode_score.append(mean_score) last_100episode_score.append(mean_score) # episode_score = [] logger.info('Frame: ' + str(num_frames) + ' / Episode: ' + str(episode_true) + ' / Average Score : ' + str(int(mean_score)) + ' / epsilon: ' + str(float(epsilon))) #plot_score(mean_episode_score, episode_true) pickle.dump(mean_episode_score, open('./dqn_Riverraid_mean_scores.pickle', 'wb')) if episode_true % 50 == 1: logger.info('Frame: ' + str(num_frames) + ' / Episode: ' + str(episode_true) + ' / Average Score : ' + str(int(mean_score)) + ' / epsilon: ' + str(float(epsilon)) + ' / last_100episode_score: ' + str(float(np.mean(last_100episode_score)))) if episode % 50 == 0: torch.save(target_model.state_dict(), './dqn_spaceinvaders_target_model_state_dict.pt') torch.save(policy_model.state_dict(), './dqn_spaceinvaders_model_state_dict.pt') pass
def main(): # Initialize environment and : env = atari_wrapper.make_atari('RiverraidNoFrameskip-v4') env = atari_wrapper.wrap_deepmind(env, clip_rewards=True, frame_stack=True, pytorch_img=True) action_space = [a for a in range(env.action_space.n)] # Initialize DQN Model and optimizer: policy_model = DQN_craft() target_model = DQN_craft() print(policy_model) target_model.eval() target_model.load_seq_list(target_model.seq_list()) optimizer = optim.RMSprop(policy_model.parameters(), lr=lr, alpha=alpha) # ------------------------------------------------- # Initialize the Replay Buffer replay_buffer = ReplayBuffer_Init(rep_buf_size, rep_buf_ini, env, action_space) # Use log to record the performance logger = logging.getLogger('dqn_Riverraid') logger.setLevel(logging.INFO) logger_handler = logging.FileHandler('./dqn_Riverraid.log') logger.addHandler(logger_handler) # -------------------------------------------------------------------------------------------------------------- # Training part, Initialization below env.reset() score = 0 episode_scores = [] # A list to record all episode_true score episode_true = 0 # we regard end of life = end of episode, since there are 4 lives in RiverRaid, thus, # one episode_ture = 4 episodes num_frames = 0 episode = 0 average_100_episode = [] # For plot max_100_episode = [] # For plot frame_1000 = [] # For plot last_25episode_score = deque(maxlen=25) # for plot loss_list = [] # for plot loss_running = 0 # for log # End of initialization # -------------------------------------------------------------------------------------------------------------- while episode < max_episodes: observation = env.reset() done = False while not done: t_observation = trace.from_numpy(observation).float() / 255 # t_observation = t_observation.view(1, t_observation.shape[0], # t_observation.shape[1], # t_observation.shape[ # 2]) # t_observation.shape:torch.Size([1, 4, 84, 84]) t_observation = t_observation.unsqueeze(0) epsilon = epsilon_by_frame(num_frames) if random.random() > epsilon: # choose action by epsilon-greedy q_value = policy_model(t_observation) action = q_value.argmax(1).data.numpy().astype(int)[0] else: action = random.sample(range(len(action_space)), 1)[0] # Store experience in the replay buffer next_observation, reward, done, info = env.step( action_space[action]) replay_buffer.push(observation, action, reward, next_observation, done) observation = next_observation num_frames += 1 # update frame score += reward # Update policy if len(replay_buffer ) > batch_size and num_frames % skip_frame == 0: observations, actions, rewards, next_observations, dones = replay_buffer.sample( batch_size) observations = trace.from_numpy(np.array(observations) / 255).float() actions = trace.from_numpy( np.array(actions).astype(int)).float() actions = actions.view(actions.shape[0], 1) # torch.Size([32, 1]) rewards = trace.from_numpy(np.array(rewards)).float() rewards = rewards.view(rewards.shape[0], 1) # torch.Size([32, 1]) next_observations = trace.from_numpy( np.array(next_observations) / 255).float() dones = trace.from_numpy(np.array(dones).astype(int)).float() dones = dones.view(dones.shape[0], 1) # torch.Size([32, 1]) q_values = policy_model(observations) # torch.Size([32, 18]) next_q_values = target_model( next_observations) # torch.Size([32, 18]) q_value = q_values.Gather( actions.squeeze().long()) # torch.Size([32, 1]) next_q_value = next_q_values.max(1)[0].unsqueeze( 1) # torch.Size([32, 1]) expected_q_value = rewards + gamma * next_q_value * (1 - dones) """ if Double_dqn: # Whether use double dqn selected_action = policy_model(next_observations).argmax(dim=1, keepdim=True) next_q_value = next_q_values.gather(1, selected_action) else: next_q_value = next_q_values.max(1)[0].unsqueeze(1) # torch.Size([32, 1]) """ mse = nn.MSE() loss = mse(q_value, expected_q_value) # loss = huber_loss(q_value, expected_q_value) loss_running = loss optimizer.zero_grad() loss.backward() optimizer.step() print(f"\rloss: {loss}", end='') # Soft update or Hard update (common dqn) if Soft_update: for target_param, policy_param in zip( target_model.parameters(), policy_model.parameters()): target_param.data.copy_(TAU * policy_param.data + (1 - TAU) * target_param.data) else: if num_frames % target_update == 0: # target_model.load_state_dict(policy_model.state_dict()) target_model.load_seq_list(target_model.seq_list()) episode += 1 # End of episode_true, reset the score if info['ale.lives'] == 0: episode_scores.append(score) last_25episode_score.append(score) # record in the log logger.info('Frame: ' + str(num_frames) + ' / Episode: ' + str(episode_true) + ' / Average Score : ' + str(int(score)) + ' / epsilon: ' + str(float(epsilon)) + ' / loss : ' + str(float(loss_running))) pickle.dump(episode_scores, open('./dqn_Riverraid_mean_scores.pickle', 'wb')) episode_true += 1 score = 0 print( '\r Episode_true {} \t Average Score(last 100 episodes) {:.2f} ' .format(episode_true, np.mean(last_25episode_score)), end=" ") # Update the log if episode_true % 25 == 1: logger.info('Frame: ' + str(num_frames) + ' / Episode: ' + str(episode_true) + ' / Average Score : ' + ' ' + ' / epsilon: ' + str(float(epsilon)) + ' / last_100episode_score: ' + str(float(np.mean(last_25episode_score)))) print("episode_ture: ", episode_true, "average_100_episode :", np.mean(last_25episode_score)) print("episode:", episode) # This "if " is for plot (to store the data of per iteration ) if episode_true % 25 == 0: average_100_episode.append(np.mean(last_25episode_score)) max_100_episode.append(np.max(last_25episode_score)) loss_list.append(loss_running) frame_1000.append(num_frames / 1000.) # plot the scores and loss and save the picture if episode_true % 500 == 0: plt_result(average_100_episode, max_100_episode, frame_1000) plt_loss(loss_list, frame_1000) # Evaluation Part if Evaluation: if episode_true % evaluate_frequency == 0: test_score = evaluate(policy_model, action_space, episode_true, epsilon=evaluate_epsilon, num_episode=evaluate_episodes) print("test_score : ", test_score, " ", "test episodes: ", evaluate_episodes) if test_score > test_stander: # Save the model if the test score > test_stander trace.save( './dqn_RiverRaid_policy_model_state_dict.pth', policy_model.seq_list()) print("Test score > %d , stop train" % test_stander) break # Save the model if episode % save_frequency == 0: trace.save('./dqn_RiverRaid_policy_model_state_dict.pth', policy_model.seq_list()) plt_result(average_100_episode, max_100_episode, frame_1000) plt_loss(loss_list, frame_1000) pass
from DQL import DQL_agent import timeit import atari_wrapper env_to_use = 'Breakout-ram-v4' # game parameters env = atari_wrapper.make_atari(env_to_use) env = atari_wrapper.wrap_deepmind(env, episode_life=True, clip_rewards=False, frame_stack=False, scale=True) state_space = 128 # Using the ram input action_space = 4 ''' env.step() -> returns array (state,reward,done?,_info) Action State for Time pilot action=1 -> going straight action=2 -> going up no fire action=3 -> going right no fire action=4 -> going left no fire ''' #We initialize our agent agent = DQL_agent(state_space=state_space, action_space=action_space) reward_list = [] eps_length_list = []
def play(env_id, model_path, max_ep, video): device = 'cuda' if torch.cuda.is_available() else 'cpu' if get_env_type(env_id) == 'atari': env = make_atari(env_id) env = wrap_deepmind(env, False, False, False, False) env = wrap_pytorch(env) model_type = 'conv' else: env = gym.make(env_id) model_type = 'linear' obs_shape = env.observation_space.shape num_actions = env.action_space.n agent = DQN(obs_shape, num_actions, device=device, model=model_type) agent.load(model_path) policy = Greedy(agent) ep = 1 episode_reward = 0 obs = env.reset() screen = env.render(mode='rgb_array') if video: writer = skvideo.io.FFmpegWriter(f'videos/{env_id}-ep-{ep}.mp4') for t in count(): action = policy.act(obs, t) next_obs, reward, done, _ = env.step(action) episode_reward += reward screen = env.render(mode='rgb_array') if video: writer.writeFrame(screen) obs = next_obs if done: print(f'ep: {ep:4} reward: {episode_reward}') if ep >= max_ep: break ep += 1 episode_reward = 0 ebs = env.reset() if video: writer.close() writer = skvideo.io.FFmpegWriter( f'videos/{env_id}-ep-{ep}.mp4') if video: writer.close() env.close()
def train(env_id, lr=1e-4, gamma=0.99, memory_size=1000, batch_size=32, train_timesteps=10000, train_start_time=1000, target_update_frequency=1000, init_epsilon=1, final_epsilon=0.1, epsilon_decay=300, model_path=None): device = 'cuda' if torch.cuda.is_available() else 'cpu' LOG_PATH = f'logs/dqn_log_{env_id}.txt' if get_env_type(env_id) == 'atari': env = make_atari(env_id) env = wrap_deepmind(env) env = wrap_pytorch(env) model_type = 'conv' else: env = gym.make(env_id) model_type = 'linear' obs_shape = env.observation_space.shape num_actions = env.action_space.n memory = ReplayBuffer(memory_size) agent = DQN(obs_shape, num_actions, lr, gamma, device, model_type) policy = EpsilonGreedy(agent, num_actions, init_epsilon, final_epsilon, epsilon_decay) # populate replay memory obs = env.reset() for t in range(train_start_time): # uniform random policy action = random.randrange(num_actions) next_obs, reward, done, _ = env.step(action) memory.add(obs, action, reward, next_obs, done) obs = next_obs if done: # start a new episode obs = env.reset() # for monitoring ep_num = 1 ep_start_time = 1 episode_reward = 0 reward_list = [] # train start obs = env.reset() for t in tqdm.tqdm(range(1, train_timesteps + 1)): # choose action action = policy.act(obs, t) next_obs, reward, done, _ = env.step(action) memory.add(obs, action, reward, next_obs, done) obs = next_obs # sample batch transitions from memory transitions = memory.sample(batch_size) # train loss = agent.train(transitions) # record reward episode_reward += reward # update target network at every C timesteps if t % target_update_frequency == 0: agent.update_target() if done: # start a new episode obs = env.reset() # write log with open(LOG_PATH, 'a') as f: f.write(f'{ep_num}\t{episode_reward}\t{ep_start_time}\t{t}\n') if model_path is not None: # save model info = { 'epoch': ep_num, 'timesteps': t, } agent.save(model_path, info) ep_num += 1 ep_start_time = t + 1 reward_list.append(episode_reward) episode_reward = 0
advantage, returns = generalized_advantage(rewards, masks, values) # normalize retuns optimizer.zero_grad() loss_p = (-log_probs * advantage).sum(axis=1).mean() loss_v = F.mse_loss(values, returns, reduction='none').sum(axis=1).mean() loss = loss_p + loss_v loss.backward() optimizer.step() mean_episode_reward = (rewards * masks).sum(axis=1).mean().item() running_reward = (1-alpha)*mean_episode_reward \ + alpha*running_reward if i % 10: print("Episode:{}\t Mean Episode Reward:{} \t\ Running Reward:{}".format(i, mean_episode_reward, running_reward)) if np.abs(running_reward) > 20: print("Model converged in", time.time() - start_time) rewards, _, _, _, _ = \ collect_samples(env, policy, 1, render=True) break return policy if __name__ == '__main__': env = wrap_deepmind(make_atari('PongNoFrameskip-v4'), scale=True) vanilla_pg_with_GAE(env)