def test_run_game(self, loss_str, game_name): env = rl_environment.Environment(game_name) info_state_size = env.observation_spec()["info_state"][0] num_actions = env.action_spec()["num_actions"] agents = [ policy_gradient.PolicyGradient( # pylint: disable=g-complex-comprehension player_id=player_id, info_state_size=info_state_size, num_actions=num_actions, loss_str=loss_str, hidden_layers_sizes=[8, 8], batch_size=16, entropy_cost=0.001, critic_learning_rate=0.01, pi_learning_rate=0.01, num_critic_before_pi=4) for player_id in [0, 1] ] for _ in range(2): time_step = env.reset() while not time_step.last(): current_player = time_step.observations["current_player"] current_agent = agents[current_player] agent_output = current_agent.step(time_step) time_step = env.step([agent_output.action]) for agent in agents: agent.step(time_step)
def main_loop(unused_arg): """Trains a Policy Gradient agent in the catch environment.""" env = catch.Environment() info_state_size = env.observation_spec()["info_state"][0] num_actions = env.action_spec()["num_actions"] train_episodes = FLAGS.num_episodes agent = policy_gradient.PolicyGradient(player_id=0, info_state_size=info_state_size, num_actions=num_actions, loss_str=FLAGS.algorithm, hidden_layers_sizes=[128, 128], batch_size=128, entropy_cost=0.01, critic_learning_rate=0.1, pi_learning_rate=0.1, num_critic_before_pi=3) # Train agent for ep in range(train_episodes): time_step = env.reset() while not time_step.last(): agent_output = agent.step(time_step) action_list = [agent_output.action] time_step = env.step(action_list) # Episode is over, step agent with final info state. agent.step(time_step) if ep and ep % FLAGS.eval_every == 0: logging.info("-" * 80) logging.info("Episode %s", ep) logging.info("Loss: %s", agent.loss) avg_return = _eval_agent(env, agent, 100) logging.info("Avg return: %s", avg_return)
def test_loss_modes(self): loss_dict = { "qpg": rl_losses.BatchQPGLoss, "rpg": rl_losses.BatchRPGLoss, "rm": rl_losses.BatchRMLoss, "a2c": rl_losses.BatchA2CLoss, } for loss_str, loss_class in loss_dict.items(): agent_by_str = policy_gradient.PolicyGradient(player_id=0, info_state_size=32, num_actions=2, loss_str=loss_str, loss_class=None) agent_by_class = policy_gradient.PolicyGradient( player_id=0, info_state_size=32, num_actions=2, loss_str=None, loss_class=loss_class) self.assertEqual(agent_by_str._loss_class, agent_by_class._loss_class)
def test_run_hanabi(self): # Hanabi is an optional game, so check we have it before running the test. game = "hanabi" if game not in pyspiel.registered_names(): return num_players = 3 env_configs = { "players": num_players, "max_life_tokens": 1, "colors": 2, "ranks": 3, "hand_size": 2, "max_information_tokens": 3, "discount": 0. } env = rl_environment.Environment(game, **env_configs) info_state_size = env.observation_spec()["info_state"][0] num_actions = env.action_spec()["num_actions"] agents = [ policy_gradient.PolicyGradient( # pylint: disable=g-complex-comprehension player_id=player_id, info_state_size=info_state_size, num_actions=num_actions, hidden_layers_sizes=[8, 8], batch_size=16, entropy_cost=0.001, critic_learning_rate=0.01, pi_learning_rate=0.01, num_critic_before_pi=4) for player_id in range(num_players) ] time_step = env.reset() while not time_step.last(): current_player = time_step.observations["current_player"] agent_output = [agent.step(time_step) for agent in agents] time_step = env.step([agent_output[current_player].action]) for agent in agents: agent.step(time_step)