def test_header(self): """Define the arena with one version""" arena = Arena([("Random A", lambda seed: AgentRandom(seed)), ("Random C", lambda seed: AgentRandom(seed)), ("Random B", lambda seed: AgentRandom(seed))], 5) self.assertListEqual(arena.csv_header(), ["opponent", "Random A", "Random B", "Random C"])
def test_init_multiple(self): """Define the arena with several agents""" arena = Arena([("Random A", lambda seed: AgentRandom(seed)), ("Random C", lambda seed: AgentRandom(seed)), ("Random B", lambda seed: AgentRandom(seed))], 5) self.assertListEqual(arena.names(), ["Random A", "Random B", "Random C"])
def test_init_single(self): """Define the arena with one version""" arena = Arena([ ("Random", lambda seed: AgentRandom(seed)) ], 5) results = arena.results() self.assertEqual(len(results), 1) self.assertListEqual(results, [("Random", "Random", 1)])
def test_list(self): """Define the arena with one version""" arena = Arena([("Random A", lambda seed: AgentRandom(seed)), ("Random C", lambda seed: AgentRandom(seed)), ("Random B", lambda seed: AgentRandom(seed))], 5) self.assertEqual(len(arena.csv_results_lists()), 3) self.assertListEqual( arena.csv_results_lists(), [['Random A', 0.2, 0.8, 0.8], ['Random B', 0.2, 0.2, 0.2], ['Random C', 0.2, 0.8, 0.2]])
def test_init_multiple(self): """Define the arena with several agents""" arena = Arena([("Random A", lambda seed: AgentRandom(seed)), ("Random C", lambda seed: AgentRandom(seed)), ("Random B", lambda seed: AgentRandom(seed))], 5) results = arena.results() self.assertEqual(len(results), 6) self.assertListEqual(results, [('Random A', 'Random A', 1), ('Random A', 'Random B', 1), ('Random A', 'Random C', 1), ('Random B', 'Random B', 1), ('Random C', 'Random B', 1), ('Random C', 'Random C', 1)])
default='arena.results.csv', help='Path to write arena results') ARGS = PARSER.parse_args() print('Starting arena') A3C_PATH = os.path.join("models", "stated_2017-05-01T22-59-33.510476_best_0.45875") dtype = torch.cuda.FloatTensor if torch.cuda.is_available( ) else torch.FloatTensor ARENA = Arena( [ # Place agents in this list as created # first in the tuple is the readable name # second is a lambda that ONLY takes a random seed. This can be discarded # if the the Agent does not require a seed ("A3C", lambda seed: AgentA3C(A3C_PATH, dtype, seed)), ("Random", lambda seed: AgentRandom(seed)) ], 500) print('Run the arena for: ', ARENA.csv_header()) with open(ARGS.output, 'w') as f: WRITER = csv.writer(f) WRITER.writerow(ARENA.csv_header()) WRITER.writerows(ARENA.csv_results_lists()) print('Complete')
def test(rank, args, shared_model, dtype): test_ctr = 0 torch.manual_seed(args.seed + rank) # set up logger timestring = str(date.today()) + '_' + \ time.strftime("%Hh-%Mm-%Ss", time.localtime(time.time())) run_name = args.save_name + '_' + timestring configure("logs/run_" + run_name, flush_secs=5) env = LoveLetterEnv(AgentRandom(args.seed + rank), args.seed + rank) env.seed(args.seed + rank) state = env.reset() model = ActorCritic(state.shape[0], env.action_space).type(dtype) model.eval() state = torch.from_numpy(state).type(dtype) reward_sum = 0 max_reward = -99999999 max_winrate = 0 rewards_recent = deque([], 100) done = True start_time = time.time() episode_length = 0 while True: episode_length += 1 # Sync with the shared model if done: model.load_state_dict(shared_model.state_dict()) cx = Variable(torch.zeros(1, 256).type(dtype), volatile=True) hx = Variable(torch.zeros(1, 256).type(dtype), volatile=True) else: cx = Variable(cx.data.type(dtype), volatile=True) hx = Variable(hx.data.type(dtype), volatile=True) value, logit, (hx, cx) = model((Variable(state.unsqueeze(0), volatile=True), (hx, cx))) prob = F.softmax(logit) action = prob.max(1)[1].data.cpu().numpy() state, reward, done, _ = env.step(action[0, 0]) done = done or episode_length >= args.max_episode_length reward_sum += reward if done: rewards_recent.append(reward_sum) rewards_recent_avg = sum(rewards_recent) / len(rewards_recent) print( "{} | Episode Reward {: >4}, Length {: >2} | Avg Reward {:0.2f}" .format( time.strftime("%Hh %Mm %Ss", time.gmtime(time.time() - start_time)), reward_sum, episode_length, rewards_recent_avg)) # if not stuck or args.evaluate: log_value('Reward', reward_sum, test_ctr) log_value('Reward Average', rewards_recent_avg, test_ctr) log_value('Episode length', episode_length, test_ctr) if reward_sum >= max_reward: # pickle.dump(shared_model.state_dict(), open(args.save_name + '_max' + '.p', 'wb')) path_output = args.save_name + '_max' torch.save(shared_model.state_dict(), path_output) path_now = "{}_{}".format(args.save_name, datetime.datetime.now().isoformat()) torch.save(shared_model.state_dict(), path_now) max_reward = reward_sum win_rate_v_random = Arena.compare_agents_float( lambda seed: AgentA3C(path_output, dtype, seed), lambda seed: AgentRandom(seed), 800) msg = " {} | VsRandom: {: >4}%".format( datetime.datetime.now().strftime("%c"), round(win_rate_v_random * 100, 2)) print(msg) log_value('Win Rate vs Random', win_rate_v_random, test_ctr) if win_rate_v_random > max_winrate: print("Found superior model at {}".format( datetime.datetime.now().isoformat())) torch.save( shared_model.state_dict(), "{}_{}_best_{}".format( args.save_name, datetime.datetime.now().isoformat(), win_rate_v_random)) max_winrate = win_rate_v_random reward_sum = 0 episode_length = 0 state = env.reset() test_ctr += 1 if test_ctr % 10 == 0 and not args.evaluate: # pickle.dump(shared_model.state_dict(), open(args.save_name + '.p', 'wb')) torch.save(shared_model.state_dict(), args.save_name) if not args.evaluate: time.sleep(60) elif test_ctr == evaluation_episodes: # Ensure the environment is closed so we can complete the # submission env.close() # gym.upload('monitor/' + run_name, api_key=api_key) state = torch.from_numpy(state).type(dtype)
def test_init(self): """Define the arena init""" arena = Arena([], 5) self.assertListEqual(arena.names(), [])
def test_init_single(self): """Define the arena with one version""" arena = Arena([ ("Random", lambda seed: AgentRandom(seed)) ], 5) self.assertListEqual(arena.names(), ["Random"])