def test_train(self): norm_step = 1100 memory_init_size = 100 step_num = 1500 sess = tf.InteractiveSession() tf.Variable(0, name='global_step', trainable=False) agent = DQNAgent(sess=sess, scope='dqn', replay_memory_size = 500, replay_memory_init_size=memory_init_size, update_target_estimator_every=100, norm_step=norm_step, state_shape=[2], mlp_layers=[10,10]) sess.run(tf.global_variables_initializer()) predicted_action = agent.eval_step({'obs': np.random.random_sample((2,)), 'legal_actions': [0, 1]}) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1) for step in range(step_num): ts = [{'obs': np.random.random_sample((2,)), 'legal_actions': [0, 1]}, np.random.randint(2), 0, {'obs': np.random.random_sample((2,)), 'legal_actions': [0, 1]}, True] agent.feed(ts) if step > norm_step + memory_init_size: agent.train() predicted_action = agent.step({'obs': np.random.random_sample((2,)), 'legal_actions': [0, 1]}) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1) sess.close() tf.reset_default_graph()
def test_train(self): memory_init_size = 100 num_steps = 500 agent = DQNAgent(replay_memory_size = 200, replay_memory_init_size=memory_init_size, update_target_estimator_every=100, state_shape=[2], mlp_layers=[10,10], device=torch.device('cpu')) predicted_action, _ = agent.eval_step({'obs': np.random.random_sample((2,)), 'legal_actions': {0: None, 1: None}, 'raw_legal_actions': ['call', 'raise']}) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1) for _ in range(num_steps): ts = [{'obs': np.random.random_sample((2,)), 'legal_actions': {0: None, 1: None}}, np.random.randint(2), 0, {'obs': np.random.random_sample((2,)), 'legal_actions': {0: None, 1: None}, 'raw_legal_actions': ['call', 'raise']}, True] agent.feed(ts) predicted_action = agent.step({'obs': np.random.random_sample((2,)), 'legal_actions': {0: None, 1: None}}) self.assertGreaterEqual(predicted_action, 0) self.assertLessEqual(predicted_action, 1)
scope='dqn', action_num=env.action_num, replay_memory_init_size=memory_init_size, train_every=train_every, state_shape=env.state_shape, mlp_layers=[128, 128]) # Initialize global variables sess.run(tf.global_variables_initializer()) # Init a Logger to plot the learning curve logger = Logger(log_dir) state = env.reset() for timestep in range(timesteps): action = agent.step(state) next_state, reward, done = env.step(action) ts = (state, action, reward, next_state, done) agent.feed(ts) if timestep % evaluate_every == 0: rewards = [] state = eval_env.reset() for _ in range(evaluate_num): action, _ = agent.eval_step(state) _, reward, done = env.step(action) if done: rewards.append(reward) logger.log_performance(env.timestep, np.mean(rewards)) # Close files in the logger
class NFSPAgent(object): ''' An approximate clone of rlcard.agents.nfsp_agent that uses pytorch instead of tensorflow. Note that this implementation differs from Henrich and Silver (2016) in that the supervised training minimizes cross-entropy with respect to the stored action probabilities rather than the realized actions. ''' def __init__(self, num_actions=4, state_shape=None, hidden_layers_sizes=None, reservoir_buffer_capacity=20000, anticipatory_param=0.1, batch_size=256, train_every=1, rl_learning_rate=0.1, sl_learning_rate=0.005, min_buffer_size_to_learn=100, q_replay_memory_size=20000, q_replay_memory_init_size=100, q_update_target_estimator_every=1000, q_discount_factor=0.99, q_epsilon_start=0.06, q_epsilon_end=0, q_epsilon_decay_steps=int(1e6), q_batch_size=32, q_train_every=1, q_mlp_layers=None, evaluate_with='average_policy', device=None): ''' Initialize the NFSP agent. Args: num_actions (int): The number of actions. state_shape (list): The shape of the state space. hidden_layers_sizes (list): The hidden layers sizes for the layers of the average policy. reservoir_buffer_capacity (int): The size of the buffer for average policy. anticipatory_param (float): The hyper-parameter that balances rl/avarage policy. batch_size (int): The batch_size for training average policy. train_every (int): Train the SL policy every X steps. rl_learning_rate (float): The learning rate of the RL agent. sl_learning_rate (float): the learning rate of the average policy. min_buffer_size_to_learn (int): The minimum buffer size to learn for average policy. q_replay_memory_size (int): The memory size of inner DQN agent. q_replay_memory_init_size (int): The initial memory size of inner DQN agent. q_update_target_estimator_every (int): The frequency of updating target network for inner DQN agent. q_discount_factor (float): The discount factor of inner DQN agent. q_epsilon_start (float): The starting epsilon of inner DQN agent. q_epsilon_end (float): the end epsilon of inner DQN agent. q_epsilon_decay_steps (int): The decay steps of inner DQN agent. q_batch_size (int): The batch size of inner DQN agent. q_train_step (int): Train the model every X steps. q_mlp_layers (list): The layer sizes of inner DQN agent. device (torch.device): Whether to use the cpu or gpu ''' self.use_raw = False self._num_actions = num_actions self._state_shape = state_shape self._layer_sizes = hidden_layers_sizes + [num_actions] self._batch_size = batch_size self._train_every = train_every self._sl_learning_rate = sl_learning_rate self._anticipatory_param = anticipatory_param self._min_buffer_size_to_learn = min_buffer_size_to_learn self._reservoir_buffer = ReservoirBuffer(reservoir_buffer_capacity) self._prev_timestep = None self._prev_action = None self.evaluate_with = evaluate_with if device is None: self.device = torch.device( 'cuda:0' if torch.cuda.is_available() else 'cpu') else: self.device = device # Total timesteps self.total_t = 0 # Step counter to keep track of learning. self._step_counter = 0 # Build the action-value network self._rl_agent = DQNAgent(q_replay_memory_size, q_replay_memory_init_size, \ q_update_target_estimator_every, q_discount_factor, q_epsilon_start, q_epsilon_end, \ q_epsilon_decay_steps, q_batch_size, num_actions, state_shape, q_train_every, q_mlp_layers, \ rl_learning_rate, device) # Build the average policy supervised model self._build_model() self.sample_episode_policy() def _build_model(self): ''' Build the average policy network ''' # configure the average policy network policy_network = AveragePolicyNetwork(self._num_actions, self._state_shape, self._layer_sizes) policy_network = policy_network.to(self.device) self.policy_network = policy_network self.policy_network.eval() # xavier init for p in self.policy_network.parameters(): if len(p.data.shape) > 1: nn.init.xavier_uniform_(p.data) # configure optimizer self.policy_network_optimizer = torch.optim.Adam( self.policy_network.parameters(), lr=self._sl_learning_rate) def feed(self, ts): ''' Feed data to inner RL agent Args: ts (list): A list of 5 elements that represent the transition. ''' self._rl_agent.feed(ts) self.total_t += 1 if self.total_t > 0 and len( self._reservoir_buffer ) >= self._min_buffer_size_to_learn and self.total_t % self._train_every == 0: sl_loss = self.train_sl() print('\rINFO - Step {}, sl-loss: {}'.format( self.total_t, sl_loss), end='') def step(self, state): ''' Returns the action to be taken. Args: state (dict): The current state Returns: action (int): An action id ''' obs = state['obs'] legal_actions = list(state['legal_actions'].keys()) if self._mode == 'best_response': action = self._rl_agent.step(state) one_hot = np.zeros(self._num_actions) one_hot[action] = 1 self._add_transition(obs, one_hot) elif self._mode == 'average_policy': probs = self._act(obs) probs = remove_illegal(probs, legal_actions) action = np.random.choice(len(probs), p=probs) return action def eval_step(self, state): ''' Use the average policy for evaluation purpose Args: state (dict): The current state. Returns: action (int): An action id. info (dict): A dictionary containing information ''' if self.evaluate_with == 'best_response': action, info = self._rl_agent.eval_step(state) elif self.evaluate_with == 'average_policy': obs = state['obs'] legal_actions = list(state['legal_actions'].keys()) probs = self._act(obs) probs = remove_illegal(probs, legal_actions) action = np.random.choice(len(probs), p=probs) info = {} info['probs'] = { state['raw_legal_actions'][i]: float(probs[list(state['legal_actions'].keys())[i]]) for i in range(len(state['legal_actions'])) } else: raise ValueError( "'evaluate_with' should be either 'average_policy' or 'best_response'." ) return action, info def sample_episode_policy(self): ''' Sample average/best_response policy ''' if np.random.rand() < self._anticipatory_param: self._mode = 'best_response' else: self._mode = 'average_policy' def _act(self, info_state): ''' Predict action probability givin the observation and legal actions Not connected to computation graph Args: info_state (numpy.array): An obervation. Returns: action_probs (numpy.array): The predicted action probability. ''' info_state = np.expand_dims(info_state, axis=0) info_state = torch.from_numpy(info_state).float().to(self.device) with torch.no_grad(): log_action_probs = self.policy_network(info_state).cpu().numpy() action_probs = np.exp(log_action_probs)[0] return action_probs def _add_transition(self, state, probs): ''' Adds the new transition to the reservoir buffer. Transitions are in the form (state, probs). Args: state (numpy.array): The state. probs (numpy.array): The probabilities of each action. ''' transition = Transition(info_state=state, action_probs=probs) self._reservoir_buffer.add(transition) def train_sl(self): ''' Compute the loss on sampled transitions and perform a avg-network update. If there are not enough elements in the buffer, no loss is computed and `None` is returned instead. Returns: loss (float): The average loss obtained on this batch of transitions or `None`. ''' if (len(self._reservoir_buffer) < self._batch_size or len(self._reservoir_buffer) < self._min_buffer_size_to_learn): return None transitions = self._reservoir_buffer.sample(self._batch_size) info_states = [t.info_state for t in transitions] action_probs = [t.action_probs for t in transitions] self.policy_network_optimizer.zero_grad() self.policy_network.train() # (batch, state_size) info_states = torch.from_numpy(np.array(info_states)).float().to( self.device) # (batch, num_actions) eval_action_probs = torch.from_numpy( np.array(action_probs)).float().to(self.device) # (batch, num_actions) log_forecast_action_probs = self.policy_network(info_states) ce_loss = -(eval_action_probs * log_forecast_action_probs).sum(dim=-1).mean() ce_loss.backward() self.policy_network_optimizer.step() ce_loss = ce_loss.item() self.policy_network.eval() return ce_loss def set_device(self, device): self.device = device self._rl_agent.set_device(device)