def test_ppo_model_dc_visual(): tf.reset_default_graph() with tf.Session() as sess: with tf.variable_scope("FakeGraphScope"): model = PPOModel( make_brain_parameters(discrete_action=True, visual_inputs=2)) init = tf.global_variables_initializer() sess.run(init) run_list = [ model.output, model.all_log_probs, model.value, model.entropy, model.learning_rate, ] feed_dict = { model.batch_size: 2, model.sequence_length: 1, model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]), model.visual_in[0]: np.ones([2, 40, 30, 3], dtype=np.float32), model.visual_in[1]: np.ones([2, 40, 30, 3], dtype=np.float32), model.action_masks: np.ones([2, 2], dtype=np.float32), } sess.run(run_list, feed_dict=feed_dict)
def test_ppo_model_cc_vector_rnn(): tf.reset_default_graph() with tf.Session() as sess: with tf.variable_scope("FakeGraphScope"): memory_size = 128 model = PPOModel( make_brain_parameters(discrete_action=False, visual_inputs=0), use_recurrent=True, m_size=memory_size, ) init = tf.global_variables_initializer() sess.run(init) run_list = [ model.output, model.all_log_probs, model.value, model.entropy, model.learning_rate, model.memory_out, ] feed_dict = { model.batch_size: 1, model.sequence_length: 2, model.memory_in: np.zeros((1, memory_size), dtype=np.float32), model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]), model.epsilon: np.array([[0, 1]]), } sess.run(run_list, feed_dict=feed_dict)
def test_ppo_model_cc_vector_rnn(mock_communicator, mock_launcher): tf.reset_default_graph() with tf.Session() as sess: with tf.variable_scope("FakeGraphScope"): mock_communicator.return_value = MockCommunicator( discrete_action=False, visual_inputs=0) env = UnityEnvironment(" ") memory_size = 128 model = PPOModel(env.brains["RealFakeBrain"], use_recurrent=True, m_size=memory_size) init = tf.global_variables_initializer() sess.run(init) run_list = [ model.output, model.all_log_probs, model.value, model.entropy, model.learning_rate, model.memory_out, ] feed_dict = { model.batch_size: 1, model.sequence_length: 2, model.memory_in: np.zeros((1, memory_size)), model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]), model.epsilon: np.array([[0, 1]]), } sess.run(run_list, feed_dict=feed_dict) env.close()
def test_ppo_model_dc_vector(mock_communicator, mock_launcher): tf.reset_default_graph() with tf.Session() as sess: with tf.variable_scope("FakeGraphScope"): mock_communicator.return_value = MockCommunicator( discrete_action=True, visual_inputs=0) env = UnityEnvironment(" ") model = PPOModel(env.brains["RealFakeBrain"]) init = tf.global_variables_initializer() sess.run(init) run_list = [ model.output, model.all_log_probs, model.value, model.entropy, model.learning_rate, ] feed_dict = { model.batch_size: 2, model.sequence_length: 1, model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]), model.action_masks: np.ones([2, 2]), } sess.run(run_list, feed_dict=feed_dict) env.close()
def test_ppo_model_cc_vector_curio(mock_communicator, mock_launcher): tf.reset_default_graph() with tf.Session() as sess: with tf.variable_scope("FakeGraphScope"): mock_communicator.return_value = MockCommunicator( discrete_action=False, visual_inputs=0) env = UnityEnvironment(' ') model = PPOModel(env.brains["RealFakeBrain"], use_curiosity=True) init = tf.global_variables_initializer() sess.run(init) run_list = [ model.output, model.all_log_probs, model.value, model.entropy, model.learning_rate, model.intrinsic_reward ] feed_dict = { model.batch_size: 2, model.sequence_length: 1, model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]), model.next_vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]), model.output: [[0.0, 0.0], [0.0, 0.0]], model.epsilon: np.array([[0, 1], [2, 3]]) } sess.run(run_list, feed_dict=feed_dict) env.close()
def create_model(self, brain, trainer_params, reward_signal_configs, is_training, load, seed): """ Create PPO model :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param reward_signal_configs: Reward signal config :param seed: Random seed. """ with self.graph.as_default(): self.model = PPOModel( brain=brain, lr=float(trainer_params["learning_rate"]), lr_schedule=LearningRateSchedule( trainer_params.get("learning_rate_schedule", "linear")), h_size=int(trainer_params["hidden_units"]), epsilon=float(trainer_params["epsilon"]), beta=float(trainer_params["beta"]), max_step=float(trainer_params["max_steps"]), normalize=trainer_params["normalize"], use_recurrent=trainer_params["use_recurrent"], num_layers=int(trainer_params["num_layers"]), m_size=self.m_size, seed=seed, stream_names=list(reward_signal_configs.keys()), vis_encode_type=EncoderType( trainer_params.get("vis_encode_type", "simple")), ) self.model.create_ppo_optimizer() self.inference_dict.update({ "action": self.model.output, "log_probs": self.model.all_log_probs, "entropy": self.model.entropy, "learning_rate": self.model.learning_rate, }) if self.use_continuous_act: self.inference_dict["pre_action"] = self.model.output_pre if self.use_recurrent: self.inference_dict["memory_out"] = self.model.memory_out self.total_policy_loss = self.model.abs_policy_loss self.update_dict.update({ "value_loss": self.model.value_loss, "policy_loss": self.total_policy_loss, "update_batch": self.model.update_batch, })
def __init__(self, seed, brain, trainer_params, is_training, load): """ Policy for Proximal Policy Optimization Networks. :param seed: Random seed. :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param is_training: Whether the model should be trained. :param load: Whether a pre-trained model will be loaded or a new one created. """ super().__init__(seed, brain, trainer_params) self.has_updated = False self.use_curiosity = bool(trainer_params["use_curiosity"]) with self.graph.as_default(): self.model = PPOModel( brain, lr=float(trainer_params["learning_rate"]), h_size=int(trainer_params["hidden_units"]), epsilon=float(trainer_params["epsilon"]), beta=float(trainer_params["beta"]), max_step=float(trainer_params["max_steps"]), normalize=trainer_params["normalize"], use_recurrent=trainer_params["use_recurrent"], num_layers=int(trainer_params["num_layers"]), m_size=self.m_size, use_curiosity=bool(trainer_params["use_curiosity"]), curiosity_strength=float(trainer_params["curiosity_strength"]), curiosity_enc_size=float(trainer_params["curiosity_enc_size"]), seed=seed, ) if load: self._load_graph() else: self._initialize_graph() self.inference_dict = { "action": self.model.output, "log_probs": self.model.all_log_probs, "value": self.model.value, "entropy": self.model.entropy, "learning_rate": self.model.learning_rate, } if self.use_continuous_act: self.inference_dict["pre_action"] = self.model.output_pre if self.use_recurrent: self.inference_dict["memory_out"] = self.model.memory_out if is_training and self.use_vec_obs and trainer_params["normalize"]: self.inference_dict["update_mean"] = self.model.update_mean self.inference_dict["update_variance"] = self.model.update_variance self.update_dict = { "value_loss": self.model.value_loss, "policy_loss": self.model.policy_loss, "update_batch": self.model.update_batch, } if self.use_curiosity: self.update_dict["forward_loss"] = self.model.forward_loss self.update_dict["inverse_loss"] = self.model.inverse_loss
def __init__(self, seed, brain, trainer_params, is_training, load): """ Policy for Proximal Policy Optimization Networks. :param seed: Random seed. :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param is_training: Whether the model should be trained. :param load: Whether a pre-trained model will be loaded or a new one created. """ super().__init__(seed, brain, trainer_params) self.has_updated = False self.use_curiosity = bool(trainer_params['use_curiosity']) with self.graph.as_default(): self.model = PPOModel( brain, lr=float(trainer_params['learning_rate']), h_size=int(trainer_params['hidden_units']), epsilon=float(trainer_params['epsilon']), beta=float(trainer_params['beta']), max_step=float(trainer_params['max_steps']), normalize=trainer_params['normalize'], use_recurrent=trainer_params['use_recurrent'], num_layers=int(trainer_params['num_layers']), m_size=self.m_size, use_curiosity=bool(trainer_params['use_curiosity']), curiosity_strength=float(trainer_params['curiosity_strength']), curiosity_enc_size=float(trainer_params['curiosity_enc_size']), seed=seed) if load: self._load_graph() else: self._initialize_graph() self.inference_dict = { 'action': self.model.output, 'log_probs': self.model.all_log_probs, 'value': self.model.value, 'entropy': self.model.entropy, 'learning_rate': self.model.learning_rate } if self.use_continuous_act: self.inference_dict['pre_action'] = self.model.output_pre if self.use_recurrent: self.inference_dict['memory_out'] = self.model.memory_out if is_training and self.use_vec_obs and trainer_params['normalize']: self.inference_dict['update_mean'] = self.model.update_mean self.inference_dict['update_variance'] = self.model.update_variance self.update_dict = { 'value_loss': self.model.value_loss, 'policy_loss': self.model.policy_loss, 'update_batch': self.model.update_batch } if self.use_curiosity: self.update_dict['forward_loss'] = self.model.forward_loss self.update_dict['inverse_loss'] = self.model.inverse_loss
def __init__(self, seed, brain, trainer_params, sess, is_training): """ Policy for Proximal Policy Optimization Networks. :param seed: Random seed. :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param sess: TensorFlow session. :param is_training: Whether the model should be trained. """ super().__init__(seed, brain, trainer_params, sess) self.has_updated = False self.use_curiosity = bool(trainer_params['use_curiosity']) self.model = PPOModel(brain, lr=float(trainer_params['learning_rate']), h_size=int(trainer_params['hidden_units']), epsilon=float(trainer_params['epsilon']), beta=float(trainer_params['beta']), max_step=float(trainer_params['max_steps']), normalize=trainer_params['normalize'], use_recurrent=trainer_params['use_recurrent'], num_layers=int(trainer_params['num_layers']), m_size=self.m_size, use_curiosity=bool(trainer_params['use_curiosity']), curiosity_strength=float(trainer_params['curiosity_strength']), curiosity_enc_size=float(trainer_params['curiosity_enc_size']), scope=self.variable_scope, seed=seed, with_heuristics=trainer_params['heuristics']) ############################################################################################################### self.inference_dict = {'action': self.model.output, 'log_probs': self.model.all_log_probs, 'value': self.model.value, 'entropy': self.model.entropy, 'learning_rate': self.model.learning_rate, 'fused_image' :self.model.fused_visual_in, 'input_image_list' :self.model.visual_in, 'predicted_segmentation_list' :self.model.visual_seg} ################################################################################################################ if self.use_continuous_act: self.inference_dict['pre_action'] = self.model.output_pre if self.use_recurrent: self.inference_dict['memory_out'] = self.model.memory_out if is_training and self.use_vec_obs and trainer_params['normalize']: self.inference_dict['update_mean'] = self.model.update_mean self.inference_dict['update_variance'] = self.model.update_variance self.update_dict = {'value_loss': self.model.value_loss, 'policy_loss': self.model.policy_loss, 'update_batch': self.model.update_batch} if self.use_curiosity: self.update_dict['forward_loss'] = self.model.forward_loss self.update_dict['inverse_loss'] = self.model.inverse_loss
def test_ppo_model_cc_vector(): tf.reset_default_graph() with tf.Session() as sess: with tf.variable_scope("FakeGraphScope"): model = PPOModel( make_brain_parameters(discrete_action=False, visual_inputs=0)) init = tf.global_variables_initializer() sess.run(init) run_list = [ model.output, model.log_probs, model.value, model.entropy, model.learning_rate, ] feed_dict = { model.batch_size: 2, model.sequence_length: 1, model.vector_in: np.array([[1, 2, 3, 1, 2, 3], [3, 4, 5, 3, 4, 5]]), model.epsilon: np.array([[0, 1], [2, 3]]), } sess.run(run_list, feed_dict=feed_dict)
class PPOPolicy(TFPolicy): def __init__( self, seed: int, brain: BrainParameters, trainer_params: Dict[str, Any], is_training: bool, load: bool, ): """ Policy for Proximal Policy Optimization Networks. :param seed: Random seed. :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param is_training: Whether the model should be trained. :param load: Whether a pre-trained model will be loaded or a new one created. """ super().__init__(seed, brain, trainer_params) reward_signal_configs = trainer_params["reward_signals"] self.inference_dict: Dict[str, tf.Tensor] = {} self.update_dict: Dict[str, tf.Tensor] = {} self.stats_name_to_update_name = { "Losses/Value Loss": "value_loss", "Losses/Policy Loss": "policy_loss", } self.create_model(brain, trainer_params, reward_signal_configs, is_training, load, seed) self.create_reward_signals(reward_signal_configs) with self.graph.as_default(): self.bc_module: Optional[BCModule] = None # Create pretrainer if needed if "pretraining" in trainer_params: BCModule.check_config(trainer_params["pretraining"]) self.bc_module = BCModule( self, policy_learning_rate=trainer_params["learning_rate"], default_batch_size=trainer_params["batch_size"], default_num_epoch=trainer_params["num_epoch"], **trainer_params["pretraining"], ) if load: self._load_graph() else: self._initialize_graph() def create_model(self, brain, trainer_params, reward_signal_configs, is_training, load, seed): """ Create PPO model :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param reward_signal_configs: Reward signal config :param seed: Random seed. """ with self.graph.as_default(): self.model = PPOModel( brain=brain, lr=float(trainer_params["learning_rate"]), lr_schedule=LearningRateSchedule( trainer_params.get("learning_rate_schedule", "linear")), h_size=int(trainer_params["hidden_units"]), epsilon=float(trainer_params["epsilon"]), beta=float(trainer_params["beta"]), max_step=float(trainer_params["max_steps"]), normalize=trainer_params["normalize"], use_recurrent=trainer_params["use_recurrent"], num_layers=int(trainer_params["num_layers"]), m_size=self.m_size, seed=seed, stream_names=list(reward_signal_configs.keys()), vis_encode_type=EncoderType( trainer_params.get("vis_encode_type", "simple")), ) self.model.create_ppo_optimizer() self.inference_dict.update({ "action": self.model.output, "log_probs": self.model.all_log_probs, "value_heads": self.model.value_heads, "value": self.model.value, "entropy": self.model.entropy, "learning_rate": self.model.learning_rate, }) if self.use_continuous_act: self.inference_dict["pre_action"] = self.model.output_pre if self.use_recurrent: self.inference_dict["memory_out"] = self.model.memory_out self.total_policy_loss = self.model.abs_policy_loss self.update_dict.update({ "value_loss": self.model.value_loss, "policy_loss": self.total_policy_loss, "update_batch": self.model.update_batch, }) def create_reward_signals(self, reward_signal_configs): """ Create reward signals :param reward_signal_configs: Reward signal config. """ self.reward_signals = {} with self.graph.as_default(): # Create reward signals for reward_signal, config in reward_signal_configs.items(): self.reward_signals[reward_signal] = create_reward_signal( self, self.model, reward_signal, config) self.update_dict.update( self.reward_signals[reward_signal].update_dict) @timed def evaluate(self, brain_info): """ Evaluates policy for the agent experiences provided. :param brain_info: BrainInfo object containing inputs. :return: Outputs from network as defined by self.inference_dict. """ feed_dict = { self.model.batch_size: len(brain_info.vector_observations), self.model.sequence_length: 1, } epsilon = None if self.use_recurrent: if not self.use_continuous_act: feed_dict[ self.model. prev_action] = brain_info.previous_vector_actions.reshape( [-1, len(self.model.act_size)]) if brain_info.memories.shape[1] == 0: brain_info.memories = self.make_empty_memory( len(brain_info.agents)) feed_dict[self.model.memory_in] = brain_info.memories if self.use_continuous_act: epsilon = np.random.normal( size=(len(brain_info.vector_observations), self.model.act_size[0])) feed_dict[self.model.epsilon] = epsilon feed_dict = self.fill_eval_dict(feed_dict, brain_info) run_out = self._execute_model(feed_dict, self.inference_dict) if self.use_continuous_act: run_out["random_normal_epsilon"] = epsilon return run_out @timed def update(self, mini_batch, num_sequences): """ Performs update on model. :param mini_batch: Batch of experiences. :param num_sequences: Number of sequences to process. :return: Results of update. """ feed_dict = self.construct_feed_dict(self.model, mini_batch, num_sequences) stats_needed = self.stats_name_to_update_name update_stats = {} # Collect feed dicts for all reward signals. for _, reward_signal in self.reward_signals.items(): feed_dict.update( reward_signal.prepare_update(self.model, mini_batch, num_sequences)) stats_needed.update(reward_signal.stats_name_to_update_name) update_vals = self._execute_model(feed_dict, self.update_dict) for stat_name, update_name in stats_needed.items(): update_stats[stat_name] = update_vals[update_name] return update_stats def construct_feed_dict(self, model, mini_batch, num_sequences): feed_dict = { model.batch_size: num_sequences, model.sequence_length: self.sequence_length, model.mask_input: mini_batch["masks"], model.advantage: mini_batch["advantages"], model.all_old_log_probs: mini_batch["action_probs"], } for name in self.reward_signals: feed_dict[model.returns_holders[name]] = mini_batch[ "{}_returns".format(name)] feed_dict[model.old_values[name]] = mini_batch[ "{}_value_estimates".format(name)] if self.use_continuous_act: feed_dict[model.output_pre] = mini_batch["actions_pre"] feed_dict[model.epsilon] = mini_batch["random_normal_epsilon"] else: feed_dict[model.action_holder] = mini_batch["actions"] if self.use_recurrent: feed_dict[model.prev_action] = mini_batch["prev_action"] feed_dict[model.action_masks] = mini_batch["action_mask"] if self.use_vec_obs: feed_dict[model.vector_in] = mini_batch["vector_obs"] if self.model.vis_obs_size > 0: for i, _ in enumerate(self.model.visual_in): feed_dict[model.visual_in[i]] = mini_batch["visual_obs%d" % i] if self.use_recurrent: mem_in = [ mini_batch["memory"][i] for i in range( 0, len(mini_batch["memory"]), self.sequence_length) ] feed_dict[model.memory_in] = mem_in return feed_dict def get_value_estimates(self, brain_info: BrainInfo, idx: int, done: bool) -> Dict[str, float]: """ Generates value estimates for bootstrapping. :param brain_info: BrainInfo to be used for bootstrapping. :param idx: Index in BrainInfo of agent. :param done: Whether or not this is the last element of the episode, in which case the value estimate will be 0. :return: The value estimate dictionary with key being the name of the reward signal and the value the corresponding value estimate. """ feed_dict: Dict[tf.Tensor, Any] = { self.model.batch_size: 1, self.model.sequence_length: 1, } for i in range(len(brain_info.visual_observations)): feed_dict[self.model.visual_in[i]] = [ brain_info.visual_observations[i][idx] ] if self.use_vec_obs: feed_dict[self.model.vector_in] = [ brain_info.vector_observations[idx] ] if self.use_recurrent: if brain_info.memories.shape[1] == 0: brain_info.memories = self.make_empty_memory( len(brain_info.agents)) feed_dict[self.model.memory_in] = [brain_info.memories[idx]] if not self.use_continuous_act and self.use_recurrent: feed_dict[self.model.prev_action] = [ brain_info.previous_vector_actions[idx] ] value_estimates = self.sess.run(self.model.value_heads, feed_dict) value_estimates = {k: float(v) for k, v in value_estimates.items()} # If we're done, reassign all of the value estimates that need terminal states. if done: for k in value_estimates: if self.reward_signals[k].use_terminal_states: value_estimates[k] = 0.0 return value_estimates
def __init__(self, seed, brain, trainer_params, is_training, load): """ Policy for Proximal Policy Optimization Networks. :param seed: Random seed. :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param is_training: Whether the model should be trained. :param load: Whether a pre-trained model will be loaded or a new one created. """ super().__init__(seed, brain, trainer_params) reward_signal_configs = trainer_params["reward_signals"] self.reward_signals = {} with self.graph.as_default(): self.model = PPOModel( brain, lr=float(trainer_params["learning_rate"]), h_size=int(trainer_params["hidden_units"]), epsilon=float(trainer_params["epsilon"]), beta=float(trainer_params["beta"]), max_step=float(trainer_params["max_steps"]), normalize=trainer_params["normalize"], use_recurrent=trainer_params["use_recurrent"], num_layers=int(trainer_params["num_layers"]), m_size=self.m_size, seed=seed, stream_names=list(reward_signal_configs.keys()), vis_encode_type=EncoderType( trainer_params.get("vis_encode_type", "simple")), ) self.model.create_ppo_optimizer() # Create reward signals for reward_signal, config in reward_signal_configs.items(): self.reward_signals[reward_signal] = create_reward_signal( self, reward_signal, config) # Create pretrainer if needed if "pretraining" in trainer_params: BCModule.check_config(trainer_params["pretraining"]) self.bc_module = BCModule( self, policy_learning_rate=trainer_params["learning_rate"], default_batch_size=trainer_params["batch_size"], default_num_epoch=trainer_params["num_epoch"], **trainer_params["pretraining"], ) else: self.bc_module = None if load: self._load_graph() else: self._initialize_graph() self.inference_dict = { "action": self.model.output, "log_probs": self.model.all_log_probs, "value": self.model.value_heads, "entropy": self.model.entropy, "learning_rate": self.model.learning_rate, } if self.use_continuous_act: self.inference_dict["pre_action"] = self.model.output_pre if self.use_recurrent: self.inference_dict["memory_out"] = self.model.memory_out if (is_training and self.use_vec_obs and trainer_params["normalize"] and not load): self.inference_dict[ "update_mean"] = self.model.update_normalization self.total_policy_loss = self.model.policy_loss self.update_dict = { "value_loss": self.model.value_loss, "policy_loss": self.total_policy_loss, "update_batch": self.model.update_batch, }
class PPOPolicy(TFPolicy): def __init__(self, seed, brain, trainer_params, is_training, load): """ Policy for Proximal Policy Optimization Networks. :param seed: Random seed. :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param is_training: Whether the model should be trained. :param load: Whether a pre-trained model will be loaded or a new one created. """ super().__init__(seed, brain, trainer_params) reward_signal_configs = trainer_params["reward_signals"] self.reward_signals = {} with self.graph.as_default(): self.model = PPOModel( brain, lr=float(trainer_params["learning_rate"]), h_size=int(trainer_params["hidden_units"]), epsilon=float(trainer_params["epsilon"]), beta=float(trainer_params["beta"]), max_step=float(trainer_params["max_steps"]), normalize=trainer_params["normalize"], use_recurrent=trainer_params["use_recurrent"], num_layers=int(trainer_params["num_layers"]), m_size=self.m_size, seed=seed, stream_names=list(reward_signal_configs.keys()), vis_encode_type=EncoderType( trainer_params.get("vis_encode_type", "simple")), ) self.model.create_ppo_optimizer() # Create reward signals for reward_signal, config in reward_signal_configs.items(): self.reward_signals[reward_signal] = create_reward_signal( self, reward_signal, config) # Create pretrainer if needed if "pretraining" in trainer_params: BCModule.check_config(trainer_params["pretraining"]) self.bc_module = BCModule( self, policy_learning_rate=trainer_params["learning_rate"], default_batch_size=trainer_params["batch_size"], default_num_epoch=trainer_params["num_epoch"], **trainer_params["pretraining"], ) else: self.bc_module = None if load: self._load_graph() else: self._initialize_graph() self.inference_dict = { "action": self.model.output, "log_probs": self.model.all_log_probs, "value": self.model.value_heads, "entropy": self.model.entropy, "learning_rate": self.model.learning_rate, } if self.use_continuous_act: self.inference_dict["pre_action"] = self.model.output_pre if self.use_recurrent: self.inference_dict["memory_out"] = self.model.memory_out if (is_training and self.use_vec_obs and trainer_params["normalize"] and not load): self.inference_dict[ "update_mean"] = self.model.update_normalization self.total_policy_loss = self.model.policy_loss self.update_dict = { "value_loss": self.model.value_loss, "policy_loss": self.total_policy_loss, "update_batch": self.model.update_batch, } @timed def evaluate(self, brain_info): """ Evaluates policy for the agent experiences provided. :param brain_info: BrainInfo object containing inputs. :return: Outputs from network as defined by self.inference_dict. """ feed_dict = { self.model.batch_size: len(brain_info.vector_observations), self.model.sequence_length: 1, } epsilon = None if self.use_recurrent: if not self.use_continuous_act: feed_dict[ self.model. prev_action] = brain_info.previous_vector_actions.reshape( [-1, len(self.model.act_size)]) if brain_info.memories.shape[1] == 0: brain_info.memories = self.make_empty_memory( len(brain_info.agents)) feed_dict[self.model.memory_in] = brain_info.memories if self.use_continuous_act: epsilon = np.random.normal( size=(len(brain_info.vector_observations), self.model.act_size[0])) feed_dict[self.model.epsilon] = epsilon feed_dict = self.fill_eval_dict(feed_dict, brain_info) run_out = self._execute_model(feed_dict, self.inference_dict) if self.use_continuous_act: run_out["random_normal_epsilon"] = epsilon return run_out @timed def update(self, mini_batch, num_sequences): """ Updates model using buffer. :param num_sequences: Number of trajectories in batch. :param mini_batch: Experience batch. :return: Output from update process. """ feed_dict = { self.model.batch_size: num_sequences, self.model.sequence_length: self.sequence_length, self.model.mask_input: mini_batch["masks"].flatten(), self.model.advantage: mini_batch["advantages"].reshape([-1, 1]), self.model.all_old_log_probs: mini_batch["action_probs"].reshape([-1, sum(self.model.act_size)]), } for name in self.reward_signals: feed_dict[self.model.returns_holders[name]] = mini_batch[ "{}_returns".format(name)].flatten() feed_dict[self.model.old_values[name]] = mini_batch[ "{}_value_estimates".format(name)].flatten() if self.use_continuous_act: feed_dict[ self.model.output_pre] = mini_batch["actions_pre"].reshape( [-1, self.model.act_size[0]]) feed_dict[self.model. epsilon] = mini_batch["random_normal_epsilon"].reshape( [-1, self.model.act_size[0]]) else: feed_dict[ self.model.action_holder] = mini_batch["actions"].reshape( [-1, len(self.model.act_size)]) if self.use_recurrent: feed_dict[self.model. prev_action] = mini_batch["prev_action"].reshape( [-1, len(self.model.act_size)]) feed_dict[ self.model.action_masks] = mini_batch["action_mask"].reshape( [-1, sum(self.brain.vector_action_space_size)]) if self.use_vec_obs: feed_dict[self.model.vector_in] = mini_batch["vector_obs"].reshape( [-1, self.vec_obs_size]) if self.model.vis_obs_size > 0: for i, _ in enumerate(self.model.visual_in): _obs = mini_batch["visual_obs%d" % i] if self.sequence_length > 1 and self.use_recurrent: (_batch, _seq, _w, _h, _c) = _obs.shape feed_dict[self.model.visual_in[i]] = _obs.reshape( [-1, _w, _h, _c]) else: feed_dict[self.model.visual_in[i]] = _obs if self.use_recurrent: mem_in = mini_batch["memory"][:, 0, :] feed_dict[self.model.memory_in] = mem_in run_out = self._execute_model(feed_dict, self.update_dict) return run_out def get_value_estimates(self, brain_info: BrainInfo, idx: int, done: bool) -> Dict[str, float]: """ Generates value estimates for bootstrapping. :param brain_info: BrainInfo to be used for bootstrapping. :param idx: Index in BrainInfo of agent. :param done: Whether or not this is the last element of the episode, in which case the value estimate will be 0. :return: The value estimate dictionary with key being the name of the reward signal and the value the corresponding value estimate. """ feed_dict: Dict[tf.Tensor, Any] = { self.model.batch_size: 1, self.model.sequence_length: 1, } for i in range(len(brain_info.visual_observations)): feed_dict[self.model.visual_in[i]] = [ brain_info.visual_observations[i][idx] ] if self.use_vec_obs: feed_dict[self.model.vector_in] = [ brain_info.vector_observations[idx] ] if self.use_recurrent: if brain_info.memories.shape[1] == 0: brain_info.memories = self.make_empty_memory( len(brain_info.agents)) feed_dict[self.model.memory_in] = [brain_info.memories[idx]] if not self.use_continuous_act and self.use_recurrent: feed_dict[ self.model. prev_action] = brain_info.previous_vector_actions[idx].reshape( [-1, len(self.model.act_size)]) value_estimates = self.sess.run(self.model.value_heads, feed_dict) value_estimates = {k: float(v) for k, v in value_estimates.items()} # If we're done, reassign all of the value estimates that need terminal states. if done: for k in value_estimates: if self.reward_signals[k].use_terminal_states: value_estimates[k] = 0.0 return value_estimates def get_action(self, brain_info: BrainInfo) -> ActionInfo: """ Decides actions given observations information, and takes them in environment. :param brain_info: A dictionary of brain names and BrainInfo from environment. :return: an ActionInfo containing action, memories, values and an object to be passed to add experiences """ if len(brain_info.agents) == 0: return ActionInfo([], [], [], None, None) run_out = self.evaluate(brain_info) mean_values = np.mean(np.array(list(run_out.get("value").values())), axis=0).flatten() return ActionInfo( action=run_out.get("action"), memory=run_out.get("memory_out"), text=None, value=mean_values, outputs=run_out, )
def create_model(self, brain, trainer_params, reward_signal_configs, is_training, load, seed): """ Create PPO models, one on each device :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param reward_signal_configs: Reward signal config :param seed: Random seed. """ self.devices = get_devices() with self.graph.as_default(): with tf.variable_scope("", reuse=tf.AUTO_REUSE): for device in self.devices: with tf.device(device): self.towers.append( PPOModel( brain=brain, lr=float(trainer_params["learning_rate"]), lr_schedule=LearningRateSchedule( trainer_params.get( "learning_rate_schedule", "linear")), h_size=int(trainer_params["hidden_units"]), epsilon=float(trainer_params["epsilon"]), beta=float(trainer_params["beta"]), max_step=float(trainer_params["max_steps"]), normalize=trainer_params["normalize"], use_recurrent=trainer_params["use_recurrent"], num_layers=int(trainer_params["num_layers"]), m_size=self.m_size, seed=seed, stream_names=list( reward_signal_configs.keys()), vis_encode_type=EncoderType( trainer_params.get("vis_encode_type", "simple")), )) self.towers[-1].create_ppo_optimizer() self.model = self.towers[0] avg_grads = self.average_gradients([t.grads for t in self.towers]) update_batch = self.model.optimizer.apply_gradients(avg_grads) avg_value_loss = tf.reduce_mean( tf.stack([model.value_loss for model in self.towers]), 0) avg_policy_loss = tf.reduce_mean( tf.stack([model.policy_loss for model in self.towers]), 0) self.inference_dict.update({ "action": self.model.output, "log_probs": self.model.all_log_probs, "value_heads": self.model.value_heads, "value": self.model.value, "entropy": self.model.entropy, "learning_rate": self.model.learning_rate, }) if self.use_continuous_act: self.inference_dict["pre_action"] = self.model.output_pre if self.use_recurrent: self.inference_dict["memory_out"] = self.model.memory_out if (is_training and self.use_vec_obs and trainer_params["normalize"] and not load): self.inference_dict[ "update_mean"] = self.model.update_normalization self.total_policy_loss = self.model.abs_policy_loss self.update_dict.update({ "value_loss": avg_value_loss, "policy_loss": avg_policy_loss, "update_batch": update_batch, })
class PPOPolicy(TFPolicy): def __init__( self, seed: int, brain: BrainParameters, trainer_params: Dict[str, Any], is_training: bool, load: bool, ): """ Policy for Proximal Policy Optimization Networks. :param seed: Random seed. :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param is_training: Whether the model should be trained. :param load: Whether a pre-trained model will be loaded or a new one created. """ super().__init__(seed, brain, trainer_params) reward_signal_configs = trainer_params["reward_signals"] self.inference_dict: Dict[str, tf.Tensor] = {} self.update_dict: Dict[str, tf.Tensor] = {} self.stats_name_to_update_name = { "Losses/Value Loss": "value_loss", "Losses/Policy Loss": "policy_loss", } self.create_model( brain, trainer_params, reward_signal_configs, is_training, load, seed ) self.create_reward_signals(reward_signal_configs) with self.graph.as_default(): self.bc_module: Optional[BCModule] = None # Create pretrainer if needed if "behavioral_cloning" in trainer_params: BCModule.check_config(trainer_params["behavioral_cloning"]) self.bc_module = BCModule( self, policy_learning_rate=trainer_params["learning_rate"], default_batch_size=trainer_params["batch_size"], default_num_epoch=3, **trainer_params["behavioral_cloning"], ) if load: self._load_graph() else: self._initialize_graph() def create_model( self, brain, trainer_params, reward_signal_configs, is_training, load, seed ): """ Create PPO model :param brain: Assigned Brain object. :param trainer_params: Defined training parameters. :param reward_signal_configs: Reward signal config :param seed: Random seed. """ with self.graph.as_default(): self.model = PPOModel( brain=brain, lr=float(trainer_params["learning_rate"]), lr_schedule=LearningRateSchedule( trainer_params.get("learning_rate_schedule", "linear") ), h_size=int(trainer_params["hidden_units"]), epsilon=float(trainer_params["epsilon"]), beta=float(trainer_params["beta"]), max_step=float(trainer_params["max_steps"]), normalize=trainer_params["normalize"], use_recurrent=trainer_params["use_recurrent"], num_layers=int(trainer_params["num_layers"]), m_size=self.m_size, seed=seed, stream_names=list(reward_signal_configs.keys()), vis_encode_type=EncoderType( trainer_params.get("vis_encode_type", "simple") ), ) self.model.create_ppo_optimizer() self.inference_dict.update( { "action": self.model.output, "log_probs": self.model.all_log_probs, "entropy": self.model.entropy, "learning_rate": self.model.learning_rate, } ) if self.use_continuous_act: self.inference_dict["pre_action"] = self.model.output_pre if self.use_recurrent: self.inference_dict["memory_out"] = self.model.memory_out self.total_policy_loss = self.model.abs_policy_loss self.update_dict.update( { "value_loss": self.model.value_loss, "policy_loss": self.total_policy_loss, "update_batch": self.model.update_batch, } ) def create_reward_signals(self, reward_signal_configs): """ Create reward signals :param reward_signal_configs: Reward signal config. """ self.reward_signals = {} with self.graph.as_default(): # Create reward signals for reward_signal, config in reward_signal_configs.items(): self.reward_signals[reward_signal] = create_reward_signal( self, self.model, reward_signal, config ) self.update_dict.update(self.reward_signals[reward_signal].update_dict) @timed def evaluate( self, batched_step_result: BatchedStepResult, global_agent_ids: List[str] ) -> Dict[str, Any]: """ Evaluates policy for the agent experiences provided. :param batched_step_result: BatchedStepResult object containing inputs. :param global_agent_ids: The global (with worker ID) agent ids of the data in the batched_step_result. :return: Outputs from network as defined by self.inference_dict. """ feed_dict = { self.model.batch_size: batched_step_result.n_agents(), self.model.sequence_length: 1, } epsilon = None if self.use_recurrent: if not self.use_continuous_act: feed_dict[self.model.prev_action] = self.retrieve_previous_action( global_agent_ids ) feed_dict[self.model.memory_in] = self.retrieve_memories(global_agent_ids) if self.use_continuous_act: epsilon = np.random.normal( size=(batched_step_result.n_agents(), self.model.act_size[0]) ) feed_dict[self.model.epsilon] = epsilon feed_dict = self.fill_eval_dict(feed_dict, batched_step_result) run_out = self._execute_model(feed_dict, self.inference_dict) return run_out @timed def update(self, mini_batch, num_sequences): """ Performs update on model. :param mini_batch: Batch of experiences. :param num_sequences: Number of sequences to process. :return: Results of update. """ feed_dict = self.construct_feed_dict(self.model, mini_batch, num_sequences) stats_needed = self.stats_name_to_update_name update_stats = {} # Collect feed dicts for all reward signals. for _, reward_signal in self.reward_signals.items(): feed_dict.update( reward_signal.prepare_update(self.model, mini_batch, num_sequences) ) stats_needed.update(reward_signal.stats_name_to_update_name) update_vals = self._execute_model(feed_dict, self.update_dict) for stat_name, update_name in stats_needed.items(): update_stats[stat_name] = update_vals[update_name] return update_stats def construct_feed_dict(self, model, mini_batch, num_sequences): feed_dict = { model.batch_size: num_sequences, model.sequence_length: self.sequence_length, model.mask_input: mini_batch["masks"], model.advantage: mini_batch["advantages"], model.all_old_log_probs: mini_batch["action_probs"], } for name in self.reward_signals: feed_dict[model.returns_holders[name]] = mini_batch[ "{}_returns".format(name) ] feed_dict[model.old_values[name]] = mini_batch[ "{}_value_estimates".format(name) ] if self.use_continuous_act: feed_dict[model.output_pre] = mini_batch["actions_pre"] else: feed_dict[model.action_holder] = mini_batch["actions"] if self.use_recurrent: feed_dict[model.prev_action] = mini_batch["prev_action"] feed_dict[model.action_masks] = mini_batch["action_mask"] if self.use_vec_obs: feed_dict[model.vector_in] = mini_batch["vector_obs"] if self.model.vis_obs_size > 0: for i, _ in enumerate(self.model.visual_in): feed_dict[model.visual_in[i]] = mini_batch["visual_obs%d" % i] if self.use_recurrent: mem_in = [ mini_batch["memory"][i] for i in range(0, len(mini_batch["memory"]), self.sequence_length) ] feed_dict[model.memory_in] = mem_in return feed_dict