def main(_): with tf.Session() as sess: config = get_config(FLAGS) or FLAGS if config.env_type == 'simple': env = SimpleGymEnvironment(config) else: env = GymEnvironment(config) if not tf.test.is_gpu_available() and FLAGS.use_gpu: raise Exception("use_gpu flag is true when no GPUs are available") if not FLAGS.use_gpu: config.cnn_format = 'NHWC' roms = 'roms/Pong2PlayerVS.bin' ale = ALEInterface(roms.encode('utf-8')) width = ale.ale_getScreenWidth() height = ale.ale_getScreenHeight() game_screen = GameScreen() ale.ale_resetGame() (display_width, display_height) = (width * 2, height * 2) pygame.init() screen_ale = pygame.display.set_mode((display_width, display_height)) pygame.display.set_caption("Arcade Learning Environment Random Agent Display") pygame.display.flip() game_surface = pygame.Surface((width, height), depth=8) clock = pygame.time.Clock() # Clear screen screen_ale.fill((0, 0, 0)) agent = Agent(config, env, sess, 'A') agent2 = Agent2(config, env, sess, 'B') if FLAGS.is_train: start_epoch = agent.epoch_op.eval() start_step = agent.step_op.eval() start_time = time.time() # Loop for epochs for agent.epoch in range(start_epoch, agent.max_epoch): agent2.epoch = agent.epoch # Initialize information of gameplay num_game, agent.update_count, agent2.update_count, ep_rewardA, ep_rewardB = 0, 0, 0, 0., 0. total_rewardA, total_rewardB, agent.total_loss, agent2.total_loss, agent.total_q, agent2.total_q = 0., 0., 0., 0., 0., 0. max_avg_ep_rewardA, max_avg_ep_rewardB = 0, 0 ep_rewardsA, ep_rewardsB, actionsA, actionsB = [], [], [], [] # Get first frame of gameplay numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) # Add first frame of gameplay into both agents' replay history for _ in range(agent.history_length): agent.history.add(scaled_pooled_screen) agent2.history.add(scaled_pooled_screen) # Loop for training iterations for agent.step in tqdm(range(start_step, agent.max_step), ncols=70, initial=start_step): agent2.step = agent.step # End of burn in period, start to learn from frames if agent.step == agent.learn_start: num_game, agent.update_count, agent2.update_count, ep_rewardA, ep_rewardB = 0, 0, 0, 0., 0. total_rewardA, total_rewardB, agent.total_loss, agent2.total_loss, agent.total_q, agent2.total_q = 0., 0., 0., 0., 0., 0. max_avg_ep_rewardA, max_avg_ep_rewardB = 0, 0 ep_rewardsA, ep_rewardsB, actionsA, actionsB = [], [], [], [] # 1. predict action1 = agent.predict(agent.history.get()) action2 = agent2.predict(agent2.history.get()) # 2. act ale.ale_act2(action1, action2) terminal = ale.ale_isGameOver() # End of end epoch, finish up training so that game statistics can be collected without training data being messed up if agent.step == agent.max_step - 1: terminal = True rewardA = ale.ale_getRewardA() rewardB = ale.ale_getRewardB() # Fill buffer of game screen with current frame numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) agent.observe(scaled_pooled_screen, rewardA, action1, terminal) agent2.observe(scaled_pooled_screen, rewardB, action2, terminal) # Print frame onto display screen screen_ale.blit(pygame.transform.scale2x(game_surface), (0, 0)) # Update the display screen pygame.display.flip() # Check if current episode ended if terminal: ale.ale_resetGame() terminal = ale.ale_isGameOver() rewardA = ale.ale_getRewardA() rewardB = ale.ale_getRewardB() numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) # End of an episode num_game += 1 ep_rewardsA.append(ep_rewardA) ep_rewardsB.append(ep_rewardB) ep_rewardA = 0. ep_rewardB = 0. else: ep_rewardA += rewardA ep_rewardB += rewardB actionsA.append(action1) actionsB.append(action2) total_rewardA += rewardA total_rewardB += rewardB # Do a test to get statistics so far if agent.step >= agent.learn_start: if agent.step % agent.test_step == agent.test_step - 1: avg_rewardA = total_rewardA / agent.test_step avg_rewardB = total_rewardB / agent2.test_step avg_lossA = agent.total_loss / agent.update_count avg_lossB = agent2.total_loss / agent2.update_count avg_qA = agent.total_q / agent.update_count avg_qB = agent2.total_q / agent2.update_count try: max_ep_rewardA = np.max(ep_rewardsA) min_ep_rewardA = np.min(ep_rewardsA) avg_ep_rewardA = np.mean(ep_rewardsA) max_ep_rewardB = np.max(ep_rewardsB) min_ep_rewardB = np.min(ep_rewardsB) avg_ep_rewardB = np.mean(ep_rewardsB) except: max_ep_rewardA, min_ep_rewardA, avg_ep_rewardA, max_ep_rewardB, min_ep_rewardB, avg_ep_rewardB = 0, 0, 0, 0, 0, 0 print('\nFor Agent A at Epoch %d: avg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \ % (agent.epoch, avg_rewardA, avg_lossA, avg_qA, avg_ep_rewardA, max_ep_rewardA, min_ep_rewardA, num_game)) print('\nFor Agent B at Epoch %d: avg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \ % (agent2.epoch, avg_rewardB, avg_lossB, avg_qB, avg_ep_rewardB, max_ep_rewardB, min_ep_rewardB, num_game)) if max_avg_ep_rewardA * 0.9 <= avg_ep_rewardA: agent.step_assign_op.eval({agent.step_input: agent.step + 1}) agent.save_model(agent.step + 1) max_avg_ep_rewardA = max(max_avg_ep_rewardA, avg_ep_rewardA) if max_avg_ep_rewardB * 0.9 <= avg_ep_rewardB: agent2.step_assign_op.eval({agent2.step_input: agent2.step + 1}) agent2.save_model(agent2.step + 1) max_avg_ep_rewardB = max(max_avg_ep_rewardB, avg_ep_rewardB) if agent.step > 180: agent.inject_summary({ 'average.reward': avg_rewardA, 'average.loss': avg_lossA, 'average.q': avg_qA, 'episode.max reward': max_ep_rewardA, 'episode.min reward': min_ep_rewardA, 'episode.avg reward': avg_ep_rewardA, 'episode.num of game': num_game, 'episode.rewards': ep_rewardsA, 'episode.actions': actionsA, 'training.learning_rate': agent.learning_rate_op.eval({agent.learning_rate_step: agent.step}), }, agent.step) if agent2.step > 180: agent2.inject_summary({ 'average.reward': avg_rewardB, 'average.loss': avg_lossB, 'average.q': avg_qB, 'episode.max reward': max_ep_rewardB, 'episode.min reward': min_ep_rewardB, 'episode.avg reward': avg_ep_rewardB, 'episode.num of game': num_game, 'episode.rewards': ep_rewardsB, 'episode.actions': actionsB, 'training.learning_rate': agent2.learning_rate_op.eval({agent2.learning_rate_step: agent2.step}), }, agent2.step) # Reset statistics num_game = 0 total_rewardA, total_rewardB = 0., 0. agent.total_loss, agent2.total_loss = 0., 0. agent.total_q, agent2.total_q = 0., 0. agent.update_count, agent2.update_count = 0, 0 ep_rewardA, ep_rewardB = 0., 0. ep_rewardsA, ep_rewardsB = [], [] actionsA, actionsB = [], [] # Play 10 games at the end of epoch to get game statistics total_points, paddle_bounce, wall_bounce, serving_time = [], [], [], [] for _ in range(10): cur_total_points, cur_paddle_bounce, cur_wall_bounce, cur_serving_time = 0, 0, 0, 0 # Restart game ale.ale_resetGame() # Get first frame of gameplay numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) # Create history for testing purposes test_history = History(config) # Fill first 4 images with initial screen for _ in range(agent.history_length): test_history.add(scaled_pooled_screen) while not ale.ale_isGameOver(): # 1. predict action1 = agent.predict(agent.history.get()) action2 = agent2.predict(agent2.history.get()) # 2. act ale.ale_act2(action1, action2) terminal = ale.ale_isGameOver() rewardA = ale.ale_getRewardA() rewardB = ale.ale_getRewardB() # Record game statistics of current episode cur_total_points = ale.ale_getPoints() cur_paddle_bounce = ale.ale_getSideBouncing() if ale.ale_getWallBouncing(): cur_wall_bounce += 1 if ale.ale_getServing(): cur_serving_time += 1 # Fill buffer of game screen with current frame numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) agent.observe(scaled_pooled_screen, rewardA, action1, terminal) agent2.observe(scaled_pooled_screen, rewardB, action2, terminal) # Print frame onto display screen screen_ale.blit(pygame.transform.scale2x(game_surface), (0, 0)) # Update the display screen pygame.display.flip() # Append current episode's statistics into list total_points.append(cur_total_points) paddle_bounce.append(cur_paddle_bounce / cur_total_points) if cur_paddle_bounce == 0: wall_bounce.append(cur_wall_bounce / (cur_paddle_bounce + 1)) else: wall_bounce.append(cur_wall_bounce / cur_paddle_bounce) serving_time.append(cur_serving_time / cur_total_points) # Save results of test after current epoch cur_paddle_op = agent.paddle_op.eval() cur_paddle_op[agent.epoch] = sum(paddle_bounce) / len(paddle_bounce) agent.paddle_assign_op.eval({agent.paddle_input: cur_paddle_op}) cur_wall_op = agent.wall_op.eval() cur_wall_op[agent.epoch] = sum(wall_bounce) / len(wall_bounce) agent.wall_assign_op.eval({agent.wall_input: cur_wall_op}) cur_serving_op = agent.serving_op.eval() cur_serving_op[agent.epoch] = sum(serving_time) / len(serving_time) agent.serving_assign_op.eval({agent.serving_input: cur_serving_op}) agent.save_model(agent.step + 1) else: agent.play() agent2.play()
def main(_): with tf.Session() as sess: config = get_config(FLAGS) or FLAGS if config.env_type == 'simple': env = SimpleGymEnvironment(config) else: env = GymEnvironment(config) if not tf.test.is_gpu_available() and FLAGS.use_gpu: raise Exception("use_gpu flag is true when no GPUs are available") if not FLAGS.use_gpu: config.cnn_format = 'NHWC' roms = 'roms/Pong2PlayerVS.bin' ale = ALEInterface(roms.encode('utf-8')) width = ale.ale_getScreenWidth() height = ale.ale_getScreenHeight() game_screen = GameScreen() ale.ale_resetGame() (display_width, display_height) = (width * 2, height * 2) pygame.init() screen_ale = pygame.display.set_mode((display_width, display_height)) pygame.display.set_caption( "Arcade Learning Environment Random Agent Display") pygame.display.flip() game_surface = pygame.Surface((width, height), depth=8) clock = pygame.time.Clock() # Clear screen screen_ale.fill((0, 0, 0)) agent = Agent(config, env, sess) if FLAGS.is_train: start_step = agent.step_op.eval() start_time = time.time() num_game, agent.update_count, ep_reward = 0, 0, 0. total_reward, agent.total_loss, agent.total_q = 0., 0., 0. max_avg_ep_reward = 0 ep_rewards, actions = [], [] numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) for _ in range(agent.history_length): agent.history.add(scaled_pooled_screen) for agent.step in tqdm(range(start_step, agent.max_step), ncols=70, initial=start_step): if agent.step == agent.learn_start: num_game, agent.update_count, ep_reward = 0, 0, 0. total_reward, agent.total_loss, agent.total_q = 0., 0., 0. ep_rewards, actions = [], [] # 1. predict action = agent.predict(agent.history.get()) # 2. act ale.ale_act2(action, np.random.choice([20, 21, 23, 24])) terminal = ale.ale_isGameOver() reward = ale.ale_getRewardA() # screen, reward, terminal = agent.env.act(action, is_training=True) # 3. observe # Both agents perform random actions # Agent A : [NOOP, FIRE, RIGHT, LEFT] # Agent B : [NOOP, FIRE, RIGHT, LEFT] # Fill buffer of game screen with current frame numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) agent.observe(scaled_pooled_screen, reward, action, terminal) # Print frame onto display screen screen_ale.blit(pygame.transform.scale2x(game_surface), (0, 0)) #Update the display screen pygame.display.flip() if terminal: ale.ale_resetGame() terminal = ale.ale_isGameOver() reward = ale.ale_getRewardA() numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) num_game += 1 ep_rewards.append(ep_reward) ep_reward = 0. else: ep_reward += reward actions.append(action) total_reward += reward if agent.step >= agent.learn_start: if agent.step % agent.test_step == agent.test_step - 1: avg_reward = total_reward / agent.test_step avg_loss = agent.total_loss / agent.update_count avg_q = agent.total_q / agent.update_count try: max_ep_reward = np.max(ep_rewards) min_ep_reward = np.min(ep_rewards) avg_ep_reward = np.mean(ep_rewards) except: max_ep_reward, min_ep_reward, avg_ep_reward = 0, 0, 0 print('\navg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \ % (avg_reward, avg_loss, avg_q, avg_ep_reward, max_ep_reward, min_ep_reward, num_game)) if max_avg_ep_reward * 0.9 <= avg_ep_reward: agent.step_assign_op.eval( {agent.step_input: agent.step + 1}) agent.save_model(agent.step + 1) max_avg_ep_reward = max(max_avg_ep_reward, avg_ep_reward) if agent.step > 180: agent.inject_summary( { 'average.reward': avg_reward, 'average.loss': avg_loss, 'average.q': avg_q, 'episode.max reward': max_ep_reward, 'episode.min reward': min_ep_reward, 'episode.avg reward': avg_ep_reward, 'episode.num of game': num_game, 'episode.rewards': ep_rewards, 'episode.actions': actions, 'training.learning_rate': agent.learning_rate_op.eval( {agent.learning_rate_step: agent.step }), }, agent.step) num_game = 0 total_reward = 0. agent.total_loss = 0. agent.total_q = 0. agent.update_count = 0 ep_reward = 0. ep_rewards = [] actions = [] else: while not ale.ale_isGameOver(): # Fill buffer of game screen with current frame numpy_surface = np.frombuffer(game_surface.get_buffer(), dtype=np.uint8) rgb = getRgbFromPalette(ale, game_surface, numpy_surface) del numpy_surface game_screen.paint(rgb) pooled_screen = game_screen.grab() scaled_pooled_screen = scale_image(pooled_screen) ale.ale_act2(agent.predict(pooled_screen), np.random.choice([20, 21, 23, 24])) print(ale.ale_getRewardA()) # Print frame onto display screen screen.blit(pygame.transform.scale2x(game_surface), (0, 0)) # Update the display screen pygame.display.flip() # delay to 60fps clock.tick(60.)