def test_tournament(self): env = rlcard.make('leduc-holdem') env.set_agents( [RandomAgent(env.action_num), RandomAgent(env.action_num)]) payoffs = tournament(env, 1000) self.assertEqual(len(payoffs), 2)
def eval_agents(agent1_name, agent1, agent2_name, agent2): print('\n' + agent1_name + ' vs ' + agent2_name) env = rlcard.make('leduc-holdem') env.set_agents([agent1, agent2]) reward_1, reward_2 = tournament(env, evaluate_num) print('Reward ' + agent1_name + ': ', reward_1) print('Reward ' + agent2_name + ': ', reward_2) return reward_1, reward_2
# The paths for saving the logs and learning curves log_dir = './experiments/leduc_holdem_cfr_result/' # Set a global seed set_global_seed(0) # Initilize CFR Agent agent = CFRAgent(env) agent.load() # If we have saved model, we first load the model # Evaluate CFR against pre-trained NFSP eval_env.set_agents([agent, models.load('leduc-holdem-nfsp').agents[0]]) # Init a Logger to plot the learning curve logger = Logger(log_dir) for episode in range(episode_num): agent.train() print('\rIteration {}'.format(episode), end='') # Evaluate the performance. Play with NFSP agents. if episode % evaluate_every == 0: agent.save() # Save model logger.log_performance(env.timestep, tournament(eval_env, evaluate_num)[0]) # Close files in the logger logger.close_files() # Plot the learning curve logger.plot('CFR')
# Initialize a Logger to plot the learning curve logger = Logger(log_dir) for episode in range(episode_num): # Generate data from the environment trajectories, _ = env.run(is_training=True) # Feed transitions into agent memory, and train the agent for ts in trajectories[0]: agent.feed(ts) # Evaluate the performance. Play with random agents. if episode % evaluate_every == 0: logger.log_performance(env.timestep, tournament(eval_env, evaluate_num)[0]) # Close files in the logger logger.close_files() # Plot the learning curve logger.plot('DQN') # Save model save_dir = 'models/blackjack_dqn' if not os.path.exists(save_dir): os.makedirs(save_dir) saver = tf.train.Saver() saver.save(sess, os.path.join(save_dir, 'model'))
''' Another example of loading a pre-trained NFSP model on Leduc Hold'em Here, we directly load the model from model zoo ''' import rlcard from rlcard.agents.random_agent import RandomAgent from rlcard.utils.utils import set_global_seed, tournament from rlcard import models # Make environment env = rlcard.make('leduc-holdem') # Set a global seed set_global_seed(0) # Here we directly load NFSP models from /models module nfsp_agents = models.load('leduc-holdem-nfsp').agents # Evaluate the performance. Play with random agents. evaluate_num = 10000 random_agent = RandomAgent(env.action_num) env.set_agents([nfsp_agents[0], random_agent]) reward = tournament(env, evaluate_num)[0] print('Average reward against random agent: ', reward)
def train_mahjong(): # Make environment env = rlcard.make('mahjong', config={'seed': 0}) eval_env = rlcard.make('mahjong', config={'seed': 0}) # Set the iterations numbers and how frequently we evaluate the performance evaluate_every = 1000 evaluate_num = 1000 episode_num = 10000 # The intial memory size memory_init_size = 1000 # Train the agent every X steps train_every = 64 # The paths for saving the logs and learning curves log_dir = './experiments/mahjong_nfsp_result/' # Set a global seed set_global_seed(0) with tf.Session() as sess: # Initialize a global step global_step = tf.Variable(0, name='global_step', trainable=False) # Set up the agents agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[512, 512], anticipatory_param=0.5, batch_size=256, rl_learning_rate=0.00005, sl_learning_rate=0.00001, min_buffer_size_to_learn=memory_init_size, q_replay_memory_size=int(1e5), q_replay_memory_init_size=memory_init_size, train_every=train_every, q_train_every=train_every, q_batch_size=256, q_mlp_layers=[512, 512]) agents.append(agent) random_agent = RandomAgent(action_num=eval_env.action_num) env.set_agents(agents) eval_env.set_agents( [agents[0], random_agent, random_agent, random_agent]) # Initialize global variables sess.run(tf.global_variables_initializer()) # Init a Logger to plot the learning curvefrom rlcard.agents.random_agent import RandomAgent logger = Logger(log_dir) for episode in tqdm(range(episode_num)): # First sample a policy for the episode for agent in agents: agent.sample_episode_policy() # Generate data from the environment trajectories, _ = env.run(is_training=True) # Feed transitions into agent memory, and train the agent for i in range(env.player_num): for ts in trajectories[i]: agents[i].feed(ts) # Evaluate the performance. Play with random agents. if episode % evaluate_every == 0: logger.log_performance(env.timestep, tournament(eval_env, evaluate_num)[0]) # Close files in the logger logger.close_files() # Plot the learning curve logger.plot('NFSP') # Save model save_dir = 'models/mahjong_nfsp' if not os.path.exists(save_dir): os.makedirs(save_dir) saver = tf.train.Saver() saver.save(sess, os.path.join(save_dir, 'model'))
with tf.Session() as sess: # Initialize a global step global_step = tf.Variable(0, name='global_step', trainable=False) # Set up the agents agent = DQNAgent(sess, scope='dqn', action_num=env.action_num, replay_memory_size=20000, replay_memory_init_size=memory_init_size, train_every=train_every, state_shape=env.state_shape, mlp_layers=[60, 60, 60, 60, 60], batch_size=512) saver = tf.train.Saver() random_agent = RandomAgent(action_num=eval_env.action_num) env.set_agents([agent, random_agent]) eval_env.set_agents([agent, random_agent]) # Initialize global variables sess.run(tf.global_variables_initializer()) saver.restore(sess, "models/uno_dqn5/model") print(tournament(eval_env, 10000))
# Init a Logger to plot the learning curve logger = Logger(log_dir) for episode in range(episode_num): print('Episode: ' + str(episode)) # Generate data from the environment trajectories, _ = env.run(is_training=True) # Feed transitions into agent memory, and train the agent for ts in trajectories[0]: agent.feed(ts) # Evaluate the performance. Play with random agents. if episode % evaluate_every == 0: a, b = env.timestep, tournament(eval_env, evaluate_num)[0] logger.log_performance(a, b) csvw.writerow([a,b]) f.flush() # Close files in the logger logger.close_files() # Plot the learning curve logger.plot('DQN') saver.save(sess, os.path.join(save_dir, 'model_FINAL'))
state_shape=env.state_shape, batch_size=64, mlp_layers=[64]) env.set_agents([ agent, RandomAgent(action_num=env.action_num), RandomAgent(action_num=env.action_num), RandomAgent(action_num=env.action_num) ]) eval_env.set_agents([ agent, RandomAgent(action_num=env.action_num), RandomAgent(action_num=env.action_num), RandomAgent(action_num=env.action_num) ]) logger = Logger('.') for episode in range(100000): # Generate data from the environment trajectories, _ = env.run(is_training=True) # Feed transitions into agent memory, and train the agent for ts in trajectories[0]: agent.feed(ts) # Evaluate the performance. Play with random agents. if episode % 5000 == 0: logger.log_performance(env.timestep, tournament(eval_env, 10000)[0]) logger.close_files() logger.plot('DQN')