def __init__(self): """ Load pretrained model """ super().__init__() self.graph = tf.Graph() self.sess = tf.Session(graph=self.graph) env = rlcard.make('leduc-holdem') with self.graph.as_default(): self.nfsp_agents = [] for i in range(env.player_num): agent = NFSPAgent(self.sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[128, 128], q_norm_step=1000, q_mlp_layers=[128, 128]) self.nfsp_agents.append(agent) normalize(env, self.nfsp_agents, 1000) self.sess.run(tf.global_variables_initializer()) check_point_path = os.path.join(ROOT_PATH, 'leduc_holdem_nfsp') with self.sess.as_default(): with self.graph.as_default(): saver = tf.train.Saver(tf.model_variables()) saver.restore(self.sess, tf.train.latest_checkpoint(check_point_path))
def load_nfsp_leduc_agent(model_path): # Set a global seed set_global_seed(0) # Load pretrained model graph = tf.Graph() sess = tf.Session(graph=graph) with graph.as_default(): nfsp_agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[128, 128], q_mlp_layers=[128, 128]) nfsp_agents.append(agent) # We have a pretrained model here. Change the path for your model. # check_point_path = os.path.join(rlcard.__path__[0], 'models/pretrained/leduc_holdem_nfsp') check_point_path = model_path with sess.as_default(): with graph.as_default(): saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint(check_point_path)) return nfsp_agents[0]
def __init__(self): ''' Load pretrained model ''' import tensorflow as tf from rlcard.agents.nfsp_agent import NFSPAgent self.graph = tf.Graph() self.sess = tf.Session(graph=self.graph) env = rlcard.make('leduc-holdem') with self.graph.as_default(): self.nfsp_agents = [] for i in range(env.player_num): agent = NFSPAgent(self.sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[128,128], q_mlp_layers=[128,128]) self.nfsp_agents.append(agent) check_point_path = os.path.join(ROOT_PATH, 'leduc_holdem_nfsp') with self.sess.as_default(): with self.graph.as_default(): saver = tf.train.Saver() saver.restore(self.sess, tf.train.latest_checkpoint(check_point_path))
def test_init(self): agent = NFSPAgent(num_actions=10, state_shape=[10], hidden_layers_sizes=[10, 10], q_mlp_layers=[10, 10], device=torch.device('cpu')) self.assertEqual(agent._num_actions, 10)
def test_train(self): memory_init_size = 20 num_steps = 1000 agent = NFSPAgent(num_actions=2, state_shape=[2], hidden_layers_sizes=[10, 10], reservoir_buffer_capacity=50, batch_size=4, min_buffer_size_to_learn=memory_init_size, q_replay_memory_size=50, q_replay_memory_init_size=memory_init_size, q_batch_size=4, q_mlp_layers=[10, 10], device=torch.device('cpu')) predicted_action, _ = agent.eval_step({ 'obs': np.random.random_sample((2, )), 'legal_actions': { 0: None, 1: None }, 'raw_legal_actions': ['call', 'raise'] }) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1) for _ in range(num_steps): agent.sample_episode_policy() predicted_action = agent.step({ 'obs': np.random.random_sample((2, )), 'legal_actions': { 0: None, 1: None } }) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1) ts = [{ 'obs': np.random.random_sample((2, )), 'legal_actions': { 0: None, 1: None } }, np.random.randint(2), 0, { 'obs': np.random.random_sample((2, )), 'legal_actions': { 0: None, 1: None }, 'raw_legal_actions': ['call', 'raise'] }, True] agent.feed(ts)
def test_evaluate_with(self): # Test average policy and value error here sess = tf.compat.v1.InteractiveSession() tf.Variable(0, name='global_step', trainable=False) agent = NFSPAgent(sess=sess, scope='nfsp', action_num=2, state_shape=[2], hidden_layers_sizes=[10, 10], q_mlp_layers=[10, 10], evaluate_with='average_policy') sess.run(tf.compat.v1.global_variables_initializer()) predicted_action, _ = agent.eval_step({ 'obs': np.random.random_sample((2, )), 'legal_actions': [0, 1] }) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1) sess.close() tf.compat.v1.reset_default_graph() sess = tf.compat.v1.InteractiveSession() tf.Variable(0, name='global_step', trainable=False) agent = NFSPAgent(sess=sess, scope='nfsp', action_num=2, state_shape=[2], hidden_layers_sizes=[10, 10], q_mlp_layers=[10, 10], evaluate_with='random') sess.run(tf.compat.v1.global_variables_initializer()) with self.assertRaises(ValueError): predicted_action = agent.eval_step({ 'obs': np.random.random_sample((2, )), 'legal_actions': [0, 1] }) sess.close() tf.compat.v1.reset_default_graph()
def test_train(self): norm_step = 100 memory_init_size = 20 step_num = 1000 sess = tf.InteractiveSession() tf.Variable(0, name='global_step', trainable=False) agent = NFSPAgent(sess=sess, scope='nfsp', action_num=2, state_shape=[2], hidden_layers_sizes=[10, 10], reservoir_buffer_capacity=50, batch_size=4, min_buffer_size_to_learn=memory_init_size, q_replay_memory_size=50, q_replay_memory_init_size=memory_init_size, q_batch_size=4, q_norm_step=norm_step, q_mlp_layers=[10, 10]) sess.run(tf.global_variables_initializer()) predicted_action = agent.eval_step({ 'obs': np.random.random_sample((2, )), 'legal_actions': [0, 1] }) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1) for step in range(step_num): agent.sample_episode_policy() predicted_action = agent.step({ 'obs': np.random.random_sample((2, )), 'legal_actions': [0, 1] }) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1) ts = [{ 'obs': np.random.random_sample((2, )), 'legal_actions': [0, 1] }, np.random.randint(2), 0, { 'obs': np.random.random_sample((2, )), 'legal_actions': [0, 1] }, True] agent.feed(ts) if step > norm_step + memory_init_size: agent.train_rl() agent.train_sl() sess.close() tf.reset_default_graph()
def test_init(self): sess = tf.compat.v1.InteractiveSession() tf.Variable(0, name='global_step', trainable=False) agent = NFSPAgent(sess=sess, scope='nfsp', action_num=10, state_shape=[10], hidden_layers_sizes=[10, 10], q_mlp_layers=[10, 10]) self.assertEqual(agent._action_num, 10) sess.close() tf.compat.v1.reset_default_graph()
csv_path = root_path + 'performance.csv' figure_path = root_path + 'figures/' # Set a global seed set_global_seed(0) with tf.Session() as sess: # Set agents global_step = tf.Variable(0, name='global_step', trainable=False) agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[512, 512], anticipatory_param=0.1, min_buffer_size_to_learn=memory_init_size, q_replay_memory_init_size=memory_init_size, q_norm_step=norm_step, q_mlp_layers=[512, 512]) agents.append(agent) sess.run(tf.global_variables_initializer()) random_agent = RandomAgent(action_num=eval_env.action_num) env.set_agents(agents) eval_env.set_agents([agents[0], random_agent]) # Count the number of steps step_counters = [0 for _ in range(env.player_num)]
def train_mahjong(): # Make environment env = rlcard.make('mahjong', config={'seed': 0}) eval_env = rlcard.make('mahjong', config={'seed': 0}) # Set the iterations numbers and how frequently we evaluate the performance evaluate_every = 1000 evaluate_num = 1000 episode_num = 10000 # The intial memory size memory_init_size = 1000 # Train the agent every X steps train_every = 64 # The paths for saving the logs and learning curves log_dir = './experiments/mahjong_nfsp_result/' # Set a global seed set_global_seed(0) with tf.Session() as sess: # Initialize a global step global_step = tf.Variable(0, name='global_step', trainable=False) # Set up the agents agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[512, 512], anticipatory_param=0.5, batch_size=256, rl_learning_rate=0.00005, sl_learning_rate=0.00001, min_buffer_size_to_learn=memory_init_size, q_replay_memory_size=int(1e5), q_replay_memory_init_size=memory_init_size, train_every=train_every, q_train_every=train_every, q_batch_size=256, q_mlp_layers=[512, 512]) agents.append(agent) random_agent = RandomAgent(action_num=eval_env.action_num) env.set_agents(agents) eval_env.set_agents( [agents[0], random_agent, random_agent, random_agent]) # Initialize global variables sess.run(tf.global_variables_initializer()) # Init a Logger to plot the learning curvefrom rlcard.agents.random_agent import RandomAgent logger = Logger(log_dir) for episode in tqdm(range(episode_num)): # First sample a policy for the episode for agent in agents: agent.sample_episode_policy() # Generate data from the environment trajectories, _ = env.run(is_training=True) # Feed transitions into agent memory, and train the agent for i in range(env.player_num): for ts in trajectories[i]: agents[i].feed(ts) # Evaluate the performance. Play with random agents. if episode % evaluate_every == 0: logger.log_performance(env.timestep, tournament(eval_env, evaluate_num)[0]) # Close files in the logger logger.close_files() # Plot the learning curve logger.plot('NFSP') # Save model save_dir = 'models/mahjong_nfsp' if not os.path.exists(save_dir): os.makedirs(save_dir) saver = tf.train.Saver() saver.save(sess, os.path.join(save_dir, 'model'))
# Set a global seed set_global_seed(0) with tf.Session() as sess: # Initialize a global step global_step = tf.Variable(0, name='global_step', trainable=False) # Set up the agents agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[128, 128], min_buffer_size_to_learn=memory_init_size, q_replay_memory_init_size=memory_init_size, train_every=train_every, q_train_every=train_every, q_mlp_layers=[128, 128]) agents.append(agent) random_agent = RandomAgent(action_num=eval_env.action_num) env.set_agents(agents) eval_env.set_agents([agents[0], random_agent]) # Initialize global variables sess.run(tf.global_variables_initializer()) # Init a Logger to plot the learning curve logger = Logger(log_dir)
# Set a global seed set_global_seed(0) with tf.Session() as sess: # Set agents global_step = tf.Variable(0, name='global_step', trainable=False) agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[512, 1024, 2048, 1024, 512], anticipatory_param=0.5, batch_size=256, rl_learning_rate=0.00005, sl_learning_rate=0.00001, min_buffer_size_to_learn=memory_init_size, q_replay_memory_size=int(1e5), q_replay_memory_init_size=memory_init_size, q_norm_step=norm_step, q_batch_size=256, q_mlp_layers=[512, 1024, 2048, 1024, 512]) agents.append(agent) sess.run(tf.global_variables_initializer()) random_agent = RandomAgent(action_num=eval_env.action_num) env.set_agents(agents) eval_env.set_agents([agents[0], random_agent, random_agent])
# Make environment env = rlcard.make('leduc-holdem') # Set a global seed set_global_seed(0) # Load pretrained model graph = tf.Graph() sess = tf.Session(graph=graph) with graph.as_default(): nfsp_agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[128, 128], q_mlp_layers=[128, 128]) nfsp_agents.append(agent) # We have a pretrained model here. Change the path for your model. check_point_path = os.path.join(rlcard.__path__[0], 'models/pretrained/leduc_holdem_nfsp') with sess.as_default(): with graph.as_default(): saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint(check_point_path)) # Evaluate the performance. Play with random agents. evaluate_num = 10000
# Set a global seed set_global_seed(0) with tf.Session() as sess: # Set agents global_step = tf.Variable(0, name='global_step', trainable=False) agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), anticipatory_param=0.1, action_num=env.action_num, state_shape=env.state_shape, rl_learning_rate=0.1, sl_learning_rate=0.005, hidden_layers_sizes=[128, 128], min_buffer_size_to_learn=memory_init_size, q_replay_memory_init_size=memory_init_size, q_epsilon_start=0.06, q_epsilon_end=0.0, q_norm_step=norm_step, q_mlp_layers=[128, 128]) agents.append(agent) sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() random_agent = RandomAgent(action_num=eval_env.action_num) env.set_agents(agents)
### Step 2: Initialize the NFSP agents. ### import tensorflow.compat.v1 as tf from rlcard.agents.nfsp_agent import NFSPAgent tf.disable_v2_behavior() memory_init_size = 1000 norm_step = 100 with tf.Session() as sess: # Set agents global_step = tf.Variable(0, name='global_step', trainable=False) agents = [] for i in range(env.player_num): agent = NFSPAgent(sess, scope='nfsp' + str(i), action_num=env.action_num, state_shape=env.state_shape, hidden_layers_sizes=[128, 128], min_buffer_size_to_learn=1000, q_replay_memory_init_size=memory_init_size, q_update_target_estimator_every=norm_step, q_mlp_layers=[128, 128]) agents.append(agent) # with sess.as_default(): #uncomment when loading # saver = tf.train.Saver() # saver.restore(sess, tf.train.latest_checkpoint(save_dir)) sess.run(tf.global_variables_initializer()) # comment out when loading env.set_agents(agents) # Setup all nfsp agents into training environments # Setup random agent for evaluation random_agent = RandomAgent(action_num=eval_env.action_num) eval_env.set_agents([agents[0], random_agent])