def build_policy(env, policy_network, estimate_q=True, q_network=None, **network_kwargs): if isinstance(policy_network, str): policy_network = get_network_builder(policy_network)(**network_kwargs) else: assert callable(policy_network) if estimate_q: # The architecture of q_network is the same as policy_network's. if q_network is None: q_network = get_network_builder("mlp")(**network_kwargs) elif isinstance(q_network, str): q_network = get_network_builder(q_network)(**network_kwargs) else: assert callable(q_network) q_network = q_network def policy_fn(obs_ph=None, normalize_observations=False, vf_latent=None, sess=None): # preprocess input ob_space = env.observation_space X = obs_ph if obs_ph is not None else observation_placeholder(ob_space) extra_tensors = {} if normalize_observations and X.dtype == tf.float32: encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: encoded_x = X encoded_x = encode_observation(ob_space, encoded_x) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent = policy_network(encoded_x) policy = PolicyModel(env=env, observations=X, policy_latent=policy_latent, vf_latent=vf_latent, estimate_q=estimate_q, q_network=q_network, sess=sess, **extra_tensors) return policy return policy_fn
def build_policy(observation_space,action_space, policy_network, value_network=None, normalize_observations=False, estimate_q=False, **policy_kwargs): if isinstance(policy_network, str): network_type = policy_network policy_network = get_network_builder(network_type)(**policy_kwargs) def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None): ob_space = observation_space X = observ_placeholder if observ_placeholder is not None else observation_placeholder(ob_space, batch_size=nbatch) extra_tensors = {} if normalize_observations and X.dtype == tf.float32: encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: encoded_x = X encoded_x = encode_observation(ob_space, encoded_x) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent = policy_network(encoded_x) if isinstance(policy_latent, tuple): policy_latent, recurrent_tensors = policy_latent if recurrent_tensors is not None: # recurrent architecture, need a few more steps nenv = nbatch // nsteps assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format(nbatch, nsteps) policy_latent, recurrent_tensors = policy_network(encoded_x, nenv) extra_tensors.update(recurrent_tensors) _v_net = value_network if _v_net is None or _v_net == 'shared': vf_latent = policy_latent else: if _v_net == 'copy': _v_net = policy_network else: assert callable(_v_net) with tf.variable_scope('vf', reuse=tf.AUTO_REUSE): # TODO recurrent architectures are not supported with value_network=copy yet vf_latent = _v_net(encoded_x) policy = PolicyWithValue( observation_space=observation_space, action_space=action_space, observations=X, latent=policy_latent, vf_latent=vf_latent, sess=sess, estimate_q=estimate_q, **extra_tensors ) return policy return policy_fn
def build_value(observation_space, value_network, **network_kwargs): if isinstance(value_network, str): value_network = get_network_builder(value_network)(**network_kwargs) else: assert callable(value_network) def value_fn(obs_ph=None, normalize_observations=False, sess=None): ob_space = observation_space X = obs_ph if obs_ph is not None else observation_placeholder(ob_space) extra_tensors = {} if normalize_observations and X.dtype == tf.float32: encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: encoded_x = X encoded_x = encode_observation(ob_space, encoded_x) with tf.variable_scope('vf', reuse=tf.AUTO_REUSE): vf_latent = value_network(encoded_x) value = ValueModel(observations=X, latent=vf_latent, sess=sess, **extra_tensors) return value return value_fn
def build_q_func(network, hiddens=[256], dueling=True, layer_norm=False, **network_kwargs): if isinstance(network, str): from baselines.common.models import get_network_builder network = get_network_builder(network)(**network_kwargs) def q_func_builder(input_placeholder, num_actions, scope, reuse=False): with tf.variable_scope(scope, reuse=reuse): latent = network(input_placeholder) if isinstance(latent, tuple): if latent[1] is not None: raise NotImplementedError( "DQN is not compatible with recurrent policies yet") latent = latent[0] latent = layers.flatten(latent) with tf.variable_scope("action_value"): action_out = latent for hidden in hiddens: action_out = layers.fully_connected(action_out, num_outputs=hidden, activation_fn=None) if layer_norm: action_out = layers.layer_norm(action_out, center=True, scale=True) action_out = tf.nn.relu(action_out) action_scores = layers.fully_connected(action_out, num_outputs=num_actions, activation_fn=None) if dueling: with tf.variable_scope("state_value"): state_out = latent for hidden in hiddens: state_out = layers.fully_connected(state_out, num_outputs=hidden, activation_fn=None) if layer_norm: state_out = layers.layer_norm(state_out, center=True, scale=True) state_out = tf.nn.relu(state_out) state_score = layers.fully_connected(state_out, num_outputs=1, activation_fn=None) action_scores_mean = tf.reduce_mean(action_scores, 1) action_scores_centered = action_scores - tf.expand_dims( action_scores_mean, 1) q_out = state_score + action_scores_centered else: q_out = action_scores return q_out return q_func_builder
def build_policy(env, policy_network, value_network=None, normalize_observations=False, estimate_q=False, **policy_kwargs): if isinstance(policy_network, str): network_type = policy_network policy_network = get_network_builder(network_type)(**policy_kwargs) def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None): ob_space = env.observation_space X = observ_placeholder if observ_placeholder is not None else observation_placeholder(ob_space, batch_size=nbatch) extra_tensors = {} if normalize_observations and X.dtype == tf.float32: encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: encoded_x = X encoded_x = encode_observation(ob_space, encoded_x) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent = policy_network(encoded_x) if isinstance(policy_latent, tuple): policy_latent, recurrent_tensors = policy_latent if recurrent_tensors is not None: # recurrent architecture, need a few more steps nenv = nbatch // nsteps assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format(nbatch, nsteps) policy_latent, recurrent_tensors = policy_network(encoded_x, nenv) extra_tensors.update(recurrent_tensors) _v_net = value_network if _v_net is None or _v_net == 'shared': vf_latent = policy_latent else: if _v_net == 'copy': _v_net = policy_network else: assert callable(_v_net) with tf.variable_scope('vf', reuse=tf.AUTO_REUSE): # TODO recurrent architectures are not supported with value_network=copy yet vf_latent = _v_net(encoded_x) policy = PolicyWithValue( env=env, observations=X, latent=policy_latent, vf_latent=vf_latent, sess=sess, estimate_q=estimate_q, **extra_tensors ) return policy return policy_fn
def __init__(self, config, section): network_args = {} if config.has_option(section, 'nlstm'): network_args['nlstm'] = config.getint(section, 'nlstm') if config.has_option(section, 'layer_norm'): network_args['layer_norm'] = config.getboolean( section, 'layer_norm') self._network_fn = get_network_builder('lstm')(**network_args)
def __init__(self, nb_actions, ob_shape, name='critic', network='mlp', **network_kwargs): super().__init__(name=name, network=network, **network_kwargs) self.layer_norm = True self.network_builder = get_network_builder(network)(**network_kwargs)((ob_shape[0] + nb_actions,)) self.output_layer = tf.keras.layers.Dense(units=1, kernel_initializer=tf.random_uniform_initializer(minval=-3e-3, maxval=3e-3), name='output') _ = self.output_layer(self.network_builder.outputs[0])
def __init__(self, nb_actions, ob_shape, name='actor', network='mlp', **network_kwargs): super().__init__(name=name, network=network, **network_kwargs) self.nb_actions = nb_actions self.network_builder = get_network_builder(network)(**network_kwargs)(ob_shape) self.output_layer = tf.keras.layers.Dense(units=self.nb_actions, activation=tf.keras.activations.tanh, kernel_initializer=tf.random_uniform_initializer(minval=-3e-3, maxval=3e-3)) _ = self.output_layer(self.network_builder.outputs[0])
def learn_letters(agent, env): if agent == "dqn": model = deepq.learn( env, "mlp", num_layers=4, num_hidden=128, activation=tf.tanh, # tf.nn.relu hiddens=[128], dueling=True, lr=1e-5, total_timesteps=int(1e7), buffer_size=100000, batch_size=32, exploration_fraction=0.1, exploration_final_eps=0.1, #0.01, -> testing... train_freq=1, learning_starts=10000, target_network_update_freq=100, gamma=0.9, print_freq=50) elif "ppo" in agent: mlp_net = get_network_builder("mlp")(num_layers=5, num_hidden=128, activation=tf.tanh) # tf.nn.relu ppo_params = dict( nsteps=128, ent_coef=0.01, vf_coef=0.5, max_grad_norm=0.5, lr=1e-4, gamma= 0.99, # Note that my results over the red/blue doors were computed using gamma=0.9! lam=0.95, log_interval=50, nminibatches=8, noptepochs=1, #save_interval=100, cliprange=0.2) if "lstm" in agent: # Adding a recurrent layer ppo_params["network"] = 'cnn_lstm' ppo_params["nlstm"] = 128 ppo_params["conv_fn"] = mlp_net ppo_params["lr"] = 0.001 else: # Using a standard MLP ppo_params["network"] = mlp_net timesteps = int(1e9) model = ppo2.learn(env=env, total_timesteps=timesteps, **ppo_params) else: assert False, agent + " hasn't been implemented yet" return model
def build_q_func(network, hiddens=[256], dueling=True, layer_norm=False, **network_kwargs): if isinstance(network, str): from baselines.common.models import get_network_builder network = get_network_builder(network)(**network_kwargs) def q_func_builder(input_shape, num_actions): # the sub Functional model which does not include the top layer. model = network(input_shape) # wrapping the sub Functional model with layers that compute action scores into another Functional model. latent = model.outputs if len(latent) > 1: if latent[1] is not None: raise NotImplementedError( "DQN is not compatible with recurrent policies yet") latent = latent[0] latent = tf.keras.layers.Flatten()(latent) with tf.name_scope("action_value"): action_out = latent for hidden in hiddens: action_out = tf.keras.layers.Dense(units=hidden, activation=None)(action_out) if layer_norm: action_out = tf.keras.layers.LayerNormalization( center=True, scale=True)(action_out) action_out = tf.nn.relu(action_out) action_scores = tf.keras.layers.Dense(units=num_actions, activation=None)(action_out) if dueling: with tf.name_scope("state_value"): state_out = latent for hidden in hiddens: state_out = tf.keras.layers.Dense( units=hidden, activation=None)(state_out) if layer_norm: state_out = tf.keras.layers.LayerNormalization( center=True, scale=True)(state_out) state_out = tf.nn.relu(state_out) state_score = tf.keras.layers.Dense(units=1, activation=None)(state_out) action_scores_mean = tf.reduce_mean(action_scores, 1) action_scores_centered = action_scores - tf.expand_dims( action_scores_mean, 1) q_out = state_score + action_scores_centered else: q_out = action_scores return tf.keras.Model(inputs=model.inputs, outputs=[q_out]) return q_func_builder
def __init__(self, env, nbatch, network, **policy_kwargs): ob_space = env.observation_space self.X = observation_placeholder(ob_space, batch_size=nbatch) encoded_x = encode_observation(ob_space, self.X) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): self.net = get_network_builder(network)(**policy_kwargs) self.h1 = self.net(encoded_x) self.h2 = fc(self.h1, 'vf', 1) self.out = self.h2[:, 0]
def build_policy(ob_space, ac_space, policy_network, normalize_observations=False, sess=None, train=True, beta=1.0, l2=0., lr=0.001, init_scale =0.01, init_bias=0.0, trainable_variance=True, state_dependent_variance=True, trainable_bias=True, init_logstd=0., clip=None, **policy_kwargs): if isinstance(policy_network, str): network_type = policy_network policy_network = get_network_builder(network_type)(**policy_kwargs) def policy_fn(scope_name="pi", nbatch=None, nsteps=None, sess=sess, observ_placeholder=None): X = observ_placeholder if observ_placeholder is not None else observation_placeholder(ob_space, batch_size=nbatch) extra_tensors = {} if normalize_observations and X.dtype == tf.float32: encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: encoded_x = X encoded_x = encode_observation(ob_space, encoded_x) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent, recurrent_tensors = policy_network(encoded_x) if recurrent_tensors is not None: # recurrent architecture, need a few more steps nenv = nbatch // nsteps assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format(nbatch, nsteps) policy_latent, recurrent_tensors = policy_network(encoded_x, nenv) extra_tensors.update(recurrent_tensors) policy = Policy( observations=X, action_space=ac_space, latent=policy_latent, sess=sess, train=train, beta=beta, l2=l2, lr=lr, init_scale=init_scale, init_bias=init_bias, trainable_variance=trainable_variance, trainable_bias=trainable_bias, init_logstd=init_logstd, scope_name=scope_name, state_dependent_variance=state_dependent_variance, clip=clip, **extra_tensors ) return policy return policy_fn
def prio_network_builder(env, nbatch, nsteps, nenvs, network_type, **policy_kwargs): network = get_network_builder(network_type)(**policy_kwargs) def prio_net_fn(nbatch=None, nsteps=None, sess=None): ob_space = env.observation_space X = observation_placeholder(ob_space, batch_size=nbatch) return MyNN return prio_net_fn()
def __init__(self, config, section): network_args = {} if config.has_option(section, 'num_layers'): network_args['num_layers'] = config.getint(section, 'num_layers') if config.has_option(section, 'num_hidden'): network_args['num_hidden'] = config.getint(section, 'num_hidden') if config.has_option(section, 'activation'): network_args['activation'] = eval(config.get( section, 'activation')) if config.has_option(section, 'layer_norm'): network_args['layer_norm'] = config.getboolean( section, 'layer_norm') self._network_fn = get_network_builder('mlp')(**network_args)
def build_policy(env, policy_network, arch, value_network=None, normalize_observations=False, estimate_q=False, **policy_kwargs): if isinstance(policy_network, str): network_type = policy_network policy_network = get_network_builder(network_type)(**policy_kwargs) def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None): ob_space = env.observation_space X = observ_placeholder if observ_placeholder is not None else observation_placeholder( ob_space, batch_size=nbatch) extra_tensors = {} if normalize_observations and X.dtype == tf.float32: encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: encoded_x = X encoded_x = encode_observation(ob_space, encoded_x) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent, policy_latent_mean, info_loss = policy_network( encoded_x) if isinstance(policy_latent, tuple): raise NotImplementedError() policy = PolicyWithValue( env=env, observations=X, arch=arch, latent=policy_latent, latent_mean=policy_latent_mean, info_loss=info_loss, # vf_latent=vf_latent, sess=sess, estimate_q=estimate_q, **extra_tensors) return policy return policy_fn
def build_policy(env, policy_network='', value_network=None, normalize_observations=False, estimate_q=False, **policy_kwargs): if isinstance(policy_network, str): network_type = policy_network policy_network = get_network_builder(network_type)(**policy_kwargs) def policy_fn(nbatch=None, nsteps=None, sess=None): ob_space = env.observation_space Xs = tf.stack([observation_placeholder(ob_space, batch_size=nbatch)] * 3) extra_tensors = {} encoded_x_0 = encode_observation(ob_space, Xs[0]) encoded_x_1 = encode_observation(ob_space, Xs[1]) encoded_x_2 = encode_observation(ob_space, Xs[2]) with tf.variable_scope('pi'): _, f_features_0 = policy_network(encoded_x_0) with tf.variable_scope('pi', reuse=True): policy_latent, f_features_1 = policy_network(encoded_x_1) with tf.variable_scope('pi', reuse=True): _, f_features_2 = policy_network(encoded_x_2) _v_net = value_network if _v_net is None or _v_net == 'shared': vf_latent = policy_latent else: raise NotImplementedError policy = ModifiedPolicyWithValue( env=env, observations=Xs, latent=policy_latent, f_features=[f_features_0, f_features_1, f_features_2], vf_latent=vf_latent, sess=sess, estimate_q=estimate_q, **extra_tensors) return policy return policy_fn
def build_qvalue(vf_model, actor_model, qvalue_network, **network_kwargs): if isinstance(qvalue_network, str): qvalue_network = get_network_builder(qvalue_network)(**network_kwargs) else: assert callable(qvalue_network) def qvalue_fn(sess=None): qf_input = tf.concat([vf_model.vf_latent, actor_model.pd_param], axis=-1) with tf.variable_scope('qf', reuse=tf.AUTO_REUSE): qf_latent = qvalue_network(qf_input) qvalue = QValueModel(vf_model=vf_model, actor_model=actor_model, latent=qf_latent, sess=sess) return qvalue return qvalue_fn
def build_q_func(network, hiddens=[256], dueling=True, layer_norm=False, **network_kwargs): if isinstance(network, str): from baselines.common.models import get_network_builder network = get_network_builder(network)(**network_kwargs) def q_func_builder(input_placeholder, num_actions, scope, reuse=False): with tf.variable_scope(scope, reuse=reuse): latent = network(input_placeholder) if isinstance(latent, tuple): if latent[1] is not None: raise NotImplementedError("DQN is not compatible with recurrent policies yet") latent = latent[0] latent = layers.flatten(latent) with tf.variable_scope("action_value"): action_out = latent for hidden in hiddens: action_out = layers.fully_connected(action_out, num_outputs=hidden, activation_fn=None) if layer_norm: action_out = layers.layer_norm(action_out, center=True, scale=True) action_out = tf.nn.relu(action_out) action_scores = layers.fully_connected(action_out, num_outputs=num_actions, activation_fn=None) if dueling: with tf.variable_scope("state_value"): state_out = latent for hidden in hiddens: state_out = layers.fully_connected(state_out, num_outputs=hidden, activation_fn=None) if layer_norm: state_out = layers.layer_norm(state_out, center=True, scale=True) state_out = tf.nn.relu(state_out) state_score = layers.fully_connected(state_out, num_outputs=1, activation_fn=None) action_scores_mean = tf.reduce_mean(action_scores, 1) action_scores_centered = action_scores - tf.expand_dims(action_scores_mean, 1) q_out = state_score + action_scores_centered else: q_out = action_scores return q_out return q_func_builder
def build_hr_func(network, hiddens=[16], layer_norm=False, **network_kwargs): if isinstance(network, str): from baselines.common.models import get_network_builder network = get_network_builder(network)(**network_kwargs) def hr_func_builder(input_placeholder, num_actions, scope="hr_func", reuse=False): with tf.variable_scope(scope, reuse=reuse): latent = network(input_placeholder) if isinstance(latent, tuple): if latent[1] is not None: raise NotImplementedError( "HR is not compatible with recurrent policies yet") latent = latent[0] latent = layers.flatten(latent) with tf.variable_scope("feedback_predictor"): action_out = latent for hidden in hiddens: action_out = layers.fully_connected(action_out, num_outputs=hidden, activation_fn=None) if layer_norm: action_out = layers.layer_norm(action_out, center=True, scale=True) action_out = tf.nn.relu(action_out) predicted_feedback = layers.fully_connected( action_out, num_outputs=num_actions, activation_fn=tf.nn.sigmoid) return predicted_feedback return hr_func_builder
def build_policy(env, policy_network, value_network=None, normalize_observations=False, estimate_q=False, limit_act_range=False, **policy_kwargs): """Daniel: builds the policy and value network. When calling `get_network_builder`, we look at the provided models, but these will give us 'latent' features. We can combine models together, and then if we do that (or if not) the last layer *before* the last dense layer is considered to be the 'latent' one. For example, if we call mlp, by default we get two hidden layers with tanh, so we have: input --> 64 --> tanh --> 64 --> tanh and then there is a 64 dimensional tensor (see `policy_latent`) that we pass as input to the next layer(s). If you're wondering how we get to the action dimension, look at `baselines.common.distributions` and see `pdfromlatent` methods. They do a final 'matching' dense layer, but with no activation by default. To avoid that, we need to form a new layer with the correct output dimensions based on the environment's action space. To debug network construction, use: tf_util.display_var_info(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)) Tricky, returns the inner function here, which then makes the policy via the class above, which exposes `self.action`, `self.pi`, etc., to use. """ if isinstance(policy_network, str): network_type = policy_network policy_network = get_network_builder(network_type)(**policy_kwargs) def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None): ob_space = env.observation_space X = observ_placeholder if observ_placeholder is not None else observation_placeholder( ob_space, batch_size=nbatch) extra_tensors = {} if normalize_observations and X.dtype == tf.float32: encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: encoded_x = X encoded_x = encode_observation(ob_space, encoded_x) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent = policy_network(encoded_x) if isinstance(policy_latent, tuple): policy_latent, recurrent_tensors = policy_latent if recurrent_tensors is not None: # recurrent architecture, need a few more steps nenv = nbatch // nsteps assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format( nbatch, nsteps) policy_latent, recurrent_tensors = policy_network( encoded_x, nenv) extra_tensors.update(recurrent_tensors) _v_net = value_network if _v_net is None or _v_net == 'shared': vf_latent = policy_latent else: if _v_net == 'copy': _v_net = policy_network else: assert callable(_v_net) with tf.variable_scope('vf', reuse=tf.AUTO_REUSE): # TODO recurrent architectures are not supported with value_network=copy yet vf_latent = _v_net(encoded_x) # Daniel: doing this for action range. if limit_act_range: policy_latent = tf.nn.tanh( fc(policy_latent, 'pi', env.action_space.shape[0], init_scale=0.01, init_bias=0.0)) policy = PolicyWithValue(env=env, observations=X, latent=policy_latent, vf_latent=vf_latent, sess=sess, estimate_q=estimate_q, **extra_tensors) return policy return policy_fn
def build_q_func(network, num_experts, hiddens=[256], dueling=True, layer_norm=False, **network_kwargs): assert isinstance(network, str) if isinstance(network, str): from baselines.common.models import get_network_builder # with tf.variable_scope("inp"): inp_network = get_network_builder(network)(**network_kwargs) # with tf.variable_scope("bel"): bel_network = get_network_builder(network)(**network_kwargs) def q_func_builder(input_placeholder, belief_placeholder, expert_q_ph, num_actions, scope, reuse=False): # input_placeholder = tf.Print(input_placeholder, [input_placeholder], '>>>> INP :', summarize=64*48) with tf.variable_scope(scope, reuse=reuse): # input_placeholder = tf.Print(input_placeholder, [input_placeholder], '>>>> INPUT: ', summarize=100) latent_inp = inp_network(input_placeholder) if isinstance(latent_inp, tuple): if latent_inp[1] is not None: raise NotImplementedError("DQN is not compatible with recurrent policies yet") latent_inp = latent_inp[0] latent_inp = layers.flatten(latent_inp) # belief_placeholder = tf.Print(belief_placeholder, [belief_placeholder], '>>>> BEL :', summarize=64*48) with tf.variable_scope(scope, reuse=reuse): with tf.variable_scope("bel", reuse=reuse): # residual network takes both input and bel latent_bel = bel_network(belief_placeholder) if isinstance(latent_bel, tuple): if latent_bel[1] is not None: raise NotImplementedError("DQN is not compatible with recurrent policies yet") latent_bel = latent_bel[0] latent_bel = layers.flatten(latent_bel) stacked = tf.stack([latent_inp, latent_bel], axis=1) latent = layers.flatten(stacked) with tf.variable_scope("action_value"): action_out = latent for hidden in hiddens: action_out = layers.fully_connected(action_out, num_outputs=hidden, activation_fn=None) if layer_norm: action_out = layers.layer_norm(action_out, center=True, scale=True) action_out = tf.nn.relu(action_out) action_scores = layers.fully_connected(action_out, num_outputs=num_actions, activation_fn=None) if dueling: with tf.variable_scope("state_value"): state_out = latent for hidden in hiddens: state_out = layers.fully_connected(state_out, num_outputs=hidden, activation_fn=None) if layer_norm: state_out = layers.layer_norm(state_out, center=True, scale=True) state_out = tf.nn.relu(state_out) state_score = layers.fully_connected(state_out, num_outputs=1, activation_fn=None) action_scores_mean = tf.reduce_mean(action_scores, 1) action_scores_centered = action_scores - tf.expand_dims(action_scores_mean, 1) q_out = state_score + action_scores_centered else: q_out = action_scores log_denominator = tf.log(tf.constant(num_experts, dtype=belief_placeholder.dtype)) entropy = 0.05 * -1 * tf.reduce_sum(belief_placeholder * tf.log(belief_placeholder + 1e-5), axis=1) / log_denominator entropy = tf.tile(tf.reshape(entropy, [tf.shape(entropy)[0], 1]), [1, num_actions]) # q_out = tf.Print(q_out, [q_out], '>>>> QOUT :', summarize=3) # expert_q_ph = tf.Print(expert_q_ph, [expert_q_ph], '>>>> EXP :', summarize=3) q_out = q_out * entropy + (1.0 - entropy) * expert_q_ph # with tf.variable_scope("action_value_residual_final"): # q_out = layers.fully_connected(q_out, num_outputs=num_actions, activation_fn=tf.nn.relu) # q_out = layers.fully_connected(q_out, num_outputs=num_actions, activation_fn=None) # q_out = tf.Print(q_out, [q_out], '>>>> FOUT :', summarize=3) return q_out return q_func_builder
def __init__(self, name, network='mlp', **network_kwargs): self.name = name self.network_builder = get_network_builder(network)(**network_kwargs)
def __init__(self, name, network='mlp', **network_kwargs): self.name = name self.network = network if network != 'cloth_cnn': self.network_builder = get_network_builder(network)(**network_kwargs)
def __init__(self, name, network='mlp', **network_kwargs): self.name = name self.network_builder = get_network_builder(network)(**network_kwargs) self.encoder_builder = get_network_builder("encoder_mlp")( **network_kwargs)
def __init__(self, name, nenvs, network='mlp', **network_kwargs): self.name = name self.network_builder = get_network_builder(network)(**network_kwargs) self.nenvs = nenvs
def build_policy(env, policy_network, value_network=None, normalize_observations=False, estimate_q=False, **policy_kwargs): if isinstance(policy_network, str): network_type = policy_network policy_network = get_network_builder(network_type)(**policy_kwargs) def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None): ob_space = env.observation_space observation_plh = observ_placeholder if observ_placeholder is not None else observation_placeholder( ob_space, batch_size=nbatch) extra_tensors = {} if normalize_observations and observation_plh.dtype == tf.float32: ob_plh_normalize_clip, rms = _normalize_clip_observation( observation_plh) extra_tensors['rms'] = rms else: ob_plh_normalize_clip = observation_plh ob_plh_normalize_clip = encode_observation(ob_space, ob_plh_normalize_clip) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent, recurrent_tensors = policy_network( ob_plh_normalize_clip) if recurrent_tensors is not None: # recurrent architecture, need a few more steps nenv = nbatch // nsteps assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format( nbatch, nsteps) policy_latent, recurrent_tensors = policy_network( ob_plh_normalize_clip, nenv) extra_tensors.update(recurrent_tensors) val_net = value_network if val_net is None or val_net == 'shared': vf_latent = policy_latent else: if val_net == 'copy': val_net = policy_network else: assert callable(val_net) with tf.variable_scope('vf', reuse=tf.AUTO_REUSE): vf_latent, _ = val_net(ob_plh_normalize_clip) policy = PolicyWithValue(env=env, observations=observation_plh, policy_latent=policy_latent, vf_latent=vf_latent, sess=sess, estimate_q=estimate_q, **extra_tensors) return policy # Return policy = PolicyWithValue(...) return policy_fn
def build_policy(env, policy_network, value_network=None, normalize_observations=False, estimate_q=False, **policy_kwargs): if isinstance(policy_network, str): network_type = policy_network policy_network = get_network_builder(network_type)(**policy_kwargs) def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None, randomization=True): ob_space = env.observation_space extra_tensors = {} X = observ_placeholder if observ_placeholder is not None else observation_placeholder( ob_space, batch_size=None) encoded_x = encode_observation(ob_space, X) # Randomization if randomization: encoded_x = tf.layers.conv2d( encoded_x / 255., 3, 3, padding='same', kernel_initializer=tf.initializers.glorot_normal(), trainable=False, name='randcnn') * 255. randcnn_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="ppo2_model/randcnn") extra_tensors['randcnn_param'] = randcnn_param with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent = policy_network(encoded_x) extra_tensors['latent_fts'] = policy_latent if isinstance(policy_latent, tuple): policy_latent, recurrent_tensors = policy_latent if recurrent_tensors is not None: # recurrent architecture, need a few more steps nenv = nbatch // nsteps assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format( nbatch, nsteps) policy_latent, recurrent_tensors = policy_network( encoded_x, nenv) extra_tensors.update(recurrent_tensors) _v_net = value_network if _v_net is None or _v_net == 'shared': vf_latent = policy_latent else: if _v_net == 'copy': _v_net = policy_network else: assert callable(_v_net) with tf.variable_scope('vf', reuse=tf.AUTO_REUSE): # TODO recurrent architectures are not supported with value_network=copy yet vf_latent = _v_net(encoded_x) policy = PolicyWithValue(env=env, observations=X, latent=policy_latent, vf_latent=vf_latent, sess=sess, estimate_q=estimate_q, **extra_tensors) return policy return policy_fn
def learn( *, network, env, eval_env, total_timesteps, timesteps_per_batch=1024, # what to train on max_kl=0.001, cg_iters=10, gamma=0.99, lam=1.0, # advantage estimation seed=None, ent_coef=0.0, cg_damping=1e-2, vf_stepsize=3e-4, vf_iters=3, log_path=None, max_episodes=0, max_iters=0, # time constraint callback=None, load_path=None, **network_kwargs): ''' learn a policy function with TRPO algorithm Parameters: ---------- network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types) or function that takes input placeholder and returns tuple (output, None) for feedforward nets or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class timesteps_per_batch timesteps per gradient estimation batch max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) ) ent_coef coefficient of policy entropy term in the optimization objective cg_iters number of iterations of conjugate gradient algorithm cg_damping conjugate gradient damping vf_stepsize learning rate for adam optimizer used to optimie value function loss vf_iters number of iterations of value function optimization iterations per each policy optimization step total_timesteps max number of timesteps max_episodes max number of episodes max_iters maximum number of policy optimization iterations callback function to be called with (locals(), globals()) each policy optimization step load_path str, path to load the model from (default: None, i.e. no model is loaded) **network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network Returns: ------- learnt model ''' if MPI is not None: nworkers = MPI.COMM_WORLD.Get_size() rank = MPI.COMM_WORLD.Get_rank() else: nworkers = 1 rank = 0 set_global_seeds(seed) np.set_printoptions(precision=3) # Setup losses and stuff # ---------------------------------------- ob_space = env.observation_space ac_space = env.action_space if isinstance(network, str): network = get_network_builder(network)(**network_kwargs) with tf.name_scope("pi"): pi_policy_network = network(ob_space.shape) pi_value_network = network(ob_space.shape) pi = PolicyWithValue(ac_space, pi_policy_network, pi_value_network) with tf.name_scope("oldpi"): old_pi_policy_network = network(ob_space.shape) old_pi_value_network = network(ob_space.shape) oldpi = PolicyWithValue(ac_space, old_pi_policy_network, old_pi_value_network) pi_var_list = pi_policy_network.trainable_variables + list( pi.pdtype.trainable_variables) old_pi_var_list = old_pi_policy_network.trainable_variables + list( oldpi.pdtype.trainable_variables) vf_var_list = pi_value_network.trainable_variables + pi.value_fc.trainable_variables old_vf_var_list = old_pi_value_network.trainable_variables + oldpi.value_fc.trainable_variables if load_path is not None: load_path = osp.expanduser(load_path) ckpt = tf.train.Checkpoint(model=pi) manager = tf.train.CheckpointManager(ckpt, load_path, max_to_keep=None) ckpt.restore(manager.latest_checkpoint) vfadam = MpiAdam(vf_var_list) get_flat = U.GetFlat(pi_var_list) set_from_flat = U.SetFromFlat(pi_var_list) loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"] shapes = [var.get_shape().as_list() for var in pi_var_list] def assign_old_eq_new(): for pi_var, old_pi_var in zip(pi_var_list, old_pi_var_list): old_pi_var.assign(pi_var) for vf_var, old_vf_var in zip(vf_var_list, old_vf_var_list): old_vf_var.assign(vf_var) @tf.function def compute_lossandgrad(ob, ac, atarg): with tf.GradientTape() as tape: old_policy_latent = oldpi.policy_network(ob) old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent) policy_latent = pi.policy_network(ob) pd, _ = pi.pdtype.pdfromlatent(policy_latent) kloldnew = old_pd.kl(pd) ent = pd.entropy() meankl = tf.reduce_mean(kloldnew) meanent = tf.reduce_mean(ent) entbonus = ent_coef * meanent ratio = tf.exp(pd.logp(ac) - old_pd.logp(ac)) surrgain = tf.reduce_mean(ratio * atarg) optimgain = surrgain + entbonus losses = [optimgain, meankl, entbonus, surrgain, meanent] gradients = tape.gradient(optimgain, pi_var_list) return losses + [U.flatgrad(gradients, pi_var_list)] @tf.function def compute_losses(ob, ac, atarg): old_policy_latent = oldpi.policy_network(ob) old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent) policy_latent = pi.policy_network(ob) pd, _ = pi.pdtype.pdfromlatent(policy_latent) kloldnew = old_pd.kl(pd) ent = pd.entropy() meankl = tf.reduce_mean(kloldnew) meanent = tf.reduce_mean(ent) entbonus = ent_coef * meanent ratio = tf.exp(pd.logp(ac) - old_pd.logp(ac)) surrgain = tf.reduce_mean(ratio * atarg) optimgain = surrgain + entbonus losses = [optimgain, meankl, entbonus, surrgain, meanent] return losses #ob shape should be [batch_size, ob_dim], merged nenv #ret shape should be [batch_size] @tf.function def compute_vflossandgrad(ob, ret): with tf.GradientTape() as tape: pi_vf = pi.value(ob) vferr = tf.reduce_mean(tf.square(pi_vf - ret)) return U.flatgrad(tape.gradient(vferr, vf_var_list), vf_var_list) @tf.function def compute_fvp(flat_tangent, ob, ac, atarg): with tf.GradientTape() as outter_tape: with tf.GradientTape() as inner_tape: old_policy_latent = oldpi.policy_network(ob) old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent) policy_latent = pi.policy_network(ob) pd, _ = pi.pdtype.pdfromlatent(policy_latent) kloldnew = old_pd.kl(pd) meankl = tf.reduce_mean(kloldnew) klgrads = inner_tape.gradient(meankl, pi_var_list) start = 0 tangents = [] for shape in shapes: sz = U.intprod(shape) tangents.append( tf.reshape(flat_tangent[start:start + sz], shape)) start += sz gvp = tf.add_n([ tf.reduce_sum(g * tangent) for (g, tangent) in zipsame(klgrads, tangents) ]) hessians_products = outter_tape.gradient(gvp, pi_var_list) fvp = U.flatgrad(hessians_products, pi_var_list) return fvp @contextmanager def timed(msg): if rank == 0: print(colorize(msg, color='magenta')) tstart = time.time() yield print( colorize("done in %.3f seconds" % (time.time() - tstart), color='magenta')) else: yield def allmean(x): assert isinstance(x, np.ndarray) if MPI is not None: out = np.empty_like(x) MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM) out /= nworkers else: out = np.copy(x) return out th_init = get_flat() if MPI is not None: MPI.COMM_WORLD.Bcast(th_init, root=0) set_from_flat(th_init) vfadam.sync() print("Init param sum", th_init.sum(), flush=True) # Prepare for rollouts # ---------------------------------------- seg_gen = traj_segment_generator(pi, env, timesteps_per_batch) episodes_so_far = 0 timesteps_so_far = 0 iters_so_far = 0 tstart = time.time() lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards logdir = log_path + '/evaluator' modeldir = log_path + '/models' if not os.path.exists(logdir): os.makedirs(logdir) if not os.path.exists(modeldir): os.makedirs(modeldir) evaluator = Evaluator(env=eval_env, model=pi, logdir=logdir) max_inner_iter = 500000 if env.spec.id == 'InvertedDoublePendulum-v2' else 3000000 epoch = vf_iters batch_size = timesteps_per_batch mb_size = 256 inner_iter_per_iter = epoch * int(batch_size / mb_size) max_iter = int(max_inner_iter / inner_iter_per_iter) eval_num = 150 eval_interval = save_interval = int( int(max_inner_iter / eval_num) / inner_iter_per_iter) if sum([max_iters > 0, total_timesteps > 0, max_episodes > 0]) == 0: # noththing to be done return pi assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \ 'out of max_iters, total_timesteps, and max_episodes only one should be specified' for update in range(1, max_iter + 1): if callback: callback(locals(), globals()) # if total_timesteps and timesteps_so_far >= total_timesteps: # break # elif max_episodes and episodes_so_far >= max_episodes: # break # elif max_iters and iters_so_far >= max_iters: # break logger.log("********** Iteration %i ************" % iters_so_far) if (update - 1) % eval_interval == 0: evaluator.run_evaluation(update - 1) if (update - 1) % save_interval == 0: ckpt = tf.train.Checkpoint(model=pi) ckpt.save(modeldir + '/ckpt_ite' + str((update - 1))) with timed("sampling"): seg = seg_gen.__next__() add_vtarg_and_adv(seg, gamma, lam) # ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets)) ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[ "tdlamret"] ob = sf01(ob) vpredbefore = seg["vpred"] # predicted value function before udpate atarg = (atarg - atarg.mean() ) / atarg.std() # standardized advantage function estimate if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret) if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy args = ob, ac, atarg fvpargs = [arr[::5] for arr in args] def fisher_vector_product(p): return allmean(compute_fvp(p, *fvpargs).numpy()) + cg_damping * p assign_old_eq_new() # set old parameter values to new parameter values with timed("computegrad"): *lossbefore, g = compute_lossandgrad(*args) lossbefore = allmean(np.array(lossbefore)) g = g.numpy() g = allmean(g) if np.allclose(g, 0): logger.log("Got zero gradient. not updating") else: with timed("cg"): stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank == 0) assert np.isfinite(stepdir).all() shs = .5 * stepdir.dot(fisher_vector_product(stepdir)) lm = np.sqrt(shs / max_kl) # logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g)) fullstep = stepdir / lm expectedimprove = g.dot(fullstep) surrbefore = lossbefore[0] stepsize = 1.0 thbefore = get_flat() for _ in range(10): thnew = thbefore + fullstep * stepsize set_from_flat(thnew) meanlosses = surr, kl, *_ = allmean( np.array(compute_losses(*args))) improve = surr - surrbefore logger.log("Expected: %.3f Actual: %.3f" % (expectedimprove, improve)) if not np.isfinite(meanlosses).all(): logger.log("Got non-finite value of losses -- bad!") elif kl > max_kl * 1.5: logger.log("violated KL constraint. shrinking step.") elif improve < 0: logger.log("surrogate didn't improve. shrinking step.") else: logger.log("Stepsize OK!") break stepsize *= .5 else: logger.log("couldn't compute a good step") set_from_flat(thbefore) if nworkers > 1 and iters_so_far % 20 == 0: paramsums = MPI.COMM_WORLD.allgather( (thnew.sum(), vfadam.getflat().sum())) # list of tuples assert all( np.allclose(ps, paramsums[0]) for ps in paramsums[1:]) for (lossname, lossval) in zip(loss_names, meanlosses): logger.record_tabular(lossname, lossval) with timed("vf"): for _ in range(vf_iters): for (mbob, mbret) in dataset.iterbatches( (seg["ob"], seg["tdlamret"]), include_final_partial_batch=False, batch_size=mb_size): mbob = sf01(mbob) g = allmean(compute_vflossandgrad(mbob, mbret).numpy()) vfadam.update(g, vf_stepsize) logger.record_tabular("ev_tdlam_before", explained_variance(vpredbefore, tdlamret)) lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values if MPI is not None: listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples else: listoflrpairs = [lrlocal] lens, rews = map(flatten_lists, zip(*listoflrpairs)) lenbuffer.extend(lens) rewbuffer.extend(rews) logger.record_tabular("EpLenMean", np.mean(lenbuffer)) logger.record_tabular("EpRewMean", np.mean(rewbuffer)) logger.record_tabular("EpThisIter", len(lens)) episodes_so_far += len(lens) timesteps_so_far += sum(lens) iters_so_far += 1 logger.record_tabular("EpisodesSoFar", episodes_so_far) logger.record_tabular("TimestepsSoFar", timesteps_so_far) logger.record_tabular("TimeElapsed", time.time() - tstart) if rank == 0: logger.dump_tabular() return pi
def build_policy(env, policy_network, value_network=None, normalize_observations=False, estimate_q=False, **policy_kwargs): todropoutpi = policy_kwargs['dropoutpi'] < 1.0 todropoutvf = policy_kwargs['dropoutvf'] < 1.0 if isinstance(policy_network, str): network_type = policy_network else: network_type = 'mlp' if todropoutpi or todropoutvf: print("Dropout: policy = {}, value = {}".format( policy_kwargs['dropoutpi'], policy_kwargs['dropoutvf'])) policy_network, dropoutpi_keep_prob, dropoutvf_keep_prob = get_network_builder( network_type)(**policy_kwargs) else: policy_network = get_network_builder(network_type)(**policy_kwargs) batchnormpi = policy_kwargs["batchnormpi"] batchnormvf = policy_kwargs["batchnormvf"] if batchnormpi and batchnormvf: policy_network, isbnpitrainmode, isbnvftrainmode = policy_network elif batchnormpi and not batchnormvf: policy_network, isbnpitrainmode = policy_network elif batchnormvf and not batchnormpi: policy_network, isbnvftrainmode = policy_network if batchnormpi: print("Batchnorm: Policy network") if batchnormvf: print("Batchnorm: Value network") def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None): ob_space = env.observation_space X = observ_placeholder if observ_placeholder is not None else observation_placeholder( ob_space, batch_size=nbatch) extra_tensors = {} if normalize_observations and X.dtype == tf.float32: encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: encoded_x = X encoded_x = encode_observation(ob_space, encoded_x) with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): policy_latent = policy_network(encoded_x, mode="pi") if isinstance(policy_latent, tuple): policy_latent, recurrent_tensors = policy_latent if recurrent_tensors is not None: # recurrent architecture, need a few more steps nenv = nbatch // nsteps assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format( nbatch, nsteps) policy_latent, recurrent_tensors = policy_network( encoded_x, nenv) extra_tensors.update(recurrent_tensors) _v_net = value_network if _v_net is None or _v_net == 'shared': vf_latent = policy_latent else: if _v_net == 'copy': _v_net = policy_network else: assert callable(_v_net) with tf.variable_scope('vf', reuse=tf.AUTO_REUSE): # TODO recurrent architectures are not supported with value_network=copy yet vf_latent = _v_net(encoded_x, mode="vf") if todropoutpi: extra_tensors.update({"dropoutpi_keep_prob": dropoutpi_keep_prob}) if todropoutvf: extra_tensors.update({"dropoutvf_keep_prob": dropoutvf_keep_prob}) if batchnormpi: extra_tensors.update({"isbnpitrainmode": isbnpitrainmode}) if batchnormvf: extra_tensors.update({"isbnvftrainmode": isbnvftrainmode}) policy = PolicyWithValue(env=env, observations=X, latent=policy_latent, vf_latent=vf_latent, sess=sess, estimate_q=estimate_q, **extra_tensors) return policy ret = policy_fn if batchnormpi or batchnormvf: ret = (ret, ) if batchnormpi: ret = ret + (isbnpitrainmode, ) if batchnormvf: ret = ret + (isbnvftrainmode, ) if todropoutpi or todropoutvf: return ret, dropoutpi_keep_prob, dropoutvf_keep_prob else: return ret
def learn(*, network, env, total_timesteps, eval_env=None, seed=None, nsteps=2048, ent_coef=0.0, lr=3e-4, vf_coef=0.5, max_grad_norm=0.5, gamma=0.99, lam=0.95, log_interval=10, nminibatches=4, noptepochs=4, cliprange=0.2, save_interval=0, load_path=None, model_fn=None, **network_kwargs): ''' Learn policy using PPO algorithm (https://arxiv.org/abs/1707.06347) Parameters: ---------- network: policy network architecture. Either string (mlp, lstm, lnlstm, cnn_lstm, cnn, cnn_small, conv_only - see baselines.common/models.py for full list) specifying the standard network architecture, or a function that takes tensorflow tensor as input and returns tuple (output_tensor, extra_feed) where output tensor is the last network layer output, extra_feed is None for feed-forward neural nets, and extra_feed is a dictionary describing how to feed state into the network for recurrent neural nets. See common/models.py/lstm for more details on using recurrent nets in policies env: baselines.common.vec_env.VecEnv environment. Needs to be vectorized for parallel environment simulation. The environments produced by gym.make can be wrapped using baselines.common.vec_env.DummyVecEnv class. nsteps: int number of steps of the vectorized environment per update (i.e. batch size is nsteps * nenv where nenv is number of environment copies simulated in parallel) total_timesteps: int number of timesteps (i.e. number of actions taken in the environment) ent_coef: float policy entropy coefficient in the optimization objective lr: float or function learning rate, constant or a schedule function [0,1] -> R+ where 1 is beginning of the training and 0 is the end of the training. vf_coef: float value function loss coefficient in the optimization objective max_grad_norm: float or None gradient norm clipping coefficient gamma: float discounting factor lam: float advantage estimation discounting factor (lambda in the paper) log_interval: int number of timesteps between logging events nminibatches: int number of training minibatches per update. For recurrent policies, should be smaller or equal than number of environments run in parallel. noptepochs: int number of training epochs per update cliprange: float or function clipping range, constant or schedule function [0,1] -> R+ where 1 is beginning of the training and 0 is the end of the training save_interval: int number of timesteps between saving events load_path: str path to load the model from **network_kwargs: keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network For instance, 'mlp' network architecture has arguments num_hidden and num_layers. ''' set_global_seeds(seed) total_timesteps = int(total_timesteps) # Get the nb of env nenvs = env.num_envs # Get state_space and action_space ob_space = env.observation_space ac_space = env.action_space if isinstance(network, str): network_type = network policy_network_fn = get_network_builder(network_type)(**network_kwargs) policy_network = policy_network_fn(ob_space.shape) # Calculate the batch_size nbatch = nenvs * nsteps nbatch_train = nbatch // nminibatches # Instantiate the model object (that creates act_model and train_model) if model_fn is None: from baselines.ppo2.model import Model model_fn = Model model = model_fn(ac_space=ac_space, policy_network=policy_network, ent_coef=ent_coef, vf_coef=vf_coef, max_grad_norm=max_grad_norm, lr=lr) if load_path is not None: load_path = osp.expanduser(load_path) ckpt = tf.train.Checkpoint(model=model) manager = tf.train.CheckpointManager(ckpt, load_path, max_to_keep=None) ckpt.restore(manager.latest_checkpoint) print("Restoring from {}".format(manager.latest_checkpoint)) print('after restore, all trainable weights {}'.format( model.train_model.policy_network.trainable_weights)) #model.load_weights(load_path) # Instantiate the runner object runner = Runner(env=env, model=model, nsteps=nsteps, gamma=gamma, lam=lam) if eval_env is not None: eval_runner = Runner(env=eval_env, model=model, nsteps=nsteps, gamma=gamma, lam=lam) epinfobuf = deque(maxlen=100) if eval_env is not None: eval_epinfobuf = deque(maxlen=100) # Start total timer tfirststart = time.perf_counter() nupdates = total_timesteps // nbatch for update in range(1, nupdates + 1): assert nbatch % nminibatches == 0 # Start timer tstart = time.perf_counter() frac = 1.0 - (update - 1.0) / nupdates # Calculate the learning rate #lrnow = lr(frac) lrnow = lr # Get minibatch obs, returns, masks, actions, values, neglogpacs, states, epinfos = runner.run( ) #pylint: disable=E0632 if eval_env is not None: eval_obs, eval_returns, eval_masks, eval_actions, eval_values, eval_neglogpacs, eval_states, eval_epinfos = eval_runner.run( ) #pylint: disable=E0632 epinfobuf.extend(epinfos) if eval_env is not None: eval_epinfobuf.extend(eval_epinfos) # Here what we're going to do is for each minibatch calculate the loss and append it. mblossvals = [] if states is None: # nonrecurrent version # Index of each element of batch_size # Create the indices array inds = np.arange(nbatch) for _ in range(noptepochs): # Randomize the indexes np.random.shuffle(inds) # 0 to batch_size with batch_train_size step for start in range(0, nbatch, nbatch_train): end = start + nbatch_train mbinds = inds[start:end] slices = (tf.constant(arr[mbinds]) for arr in (obs, returns, masks, actions, values, neglogpacs)) # slice_obs, slice_returns, slice_masks, slice_actions, slice_values, slice_neglogpacs = (arr[mbinds] for arr in (obs, returns, masks, actions, values, neglogpacs)) # slice_advs = slice_returns - slice_values # slice_advs = (slice_advs - slice_advs.mean()) / (slice_advs.std() + 1e-8) # slices = (tf.constant(slice_obs), tf.constant(slice_returns), tf.constant(slice_masks), # tf.constant(slice_advs), tf.constant(slice_actions), tf.constant(slice_values), tf.constant(slice_neglogpacs)) # print('slice actions {}'.format(slice_actions.dtype)) # print('-------------------------------------------') # print('inds {}'.format(inds)) # print('slice obs {}'.format(slice_obs)) # print('slice returns {}'.format(slice_returns)) # print('slice masks {}'.format(slice_masks)) # print('slice actions {}'.format(slice_actions)) # print('slice values {}'.format(slice_values)) # print('slice neglogpacs {}'.format(slice_neglogpacs)) # print('slice advs {}'.format(slice_advs)) pg_loss, vf_loss, entropy, approxkl, clipfrac, vpred, vpredclipped = model.train( lrnow, cliprange, *slices) # pg_loss, vf_loss, entropy, approxkl, clipfrac, vpred, vpredclipped = model.train( # cliprange, obs=slice_obs, returns=slice_returns, masks=slice_masks, advs=slice_advs, # actions=slice_actions, values=slice_values, neglogpac_old=slice_neglogpacs) # print('pg_loss {}'.format(pg_loss)) # print('vf_loss {}'.format(vf_loss)) # print('entropy {}'.format(entropy)) # print('approxkl {}'.format(approxkl)) # print('clipfrac {}'.format(clipfrac)) # print('vpred {}'.format(vpred)) # print('vpredclipped {}'.format(vpredclipped)) # print('pg_loss1 {}'.format(pg_loss1)) # print('pg_loss2 {}'.format(pg_loss2)) # train_model = model.train_model # params = train_model.policy_network.trainable_weights + train_model.value_fc.trainable_weights + train_model.pdtype.matching_fc.trainable_weights # for param in params: # print('param {} is {}'.format(param.name, param.numpy())) # print('-------------------------------------------') mblossvals.append([ pg_loss.numpy(), vf_loss.numpy(), entropy.numpy(), approxkl.numpy(), clipfrac.numpy() ]) # mblossvals.append([output for output.numpy() in model.train(cliprange, *slices)]) else: # recurrent version raise ValueError('Not Support Yet') # Feedforward --> get losses --> update lossvals = np.mean(mblossvals, axis=0) # End timer tnow = time.perf_counter() # Calculate the fps (frame per second) fps = int(nbatch / (tnow - tstart)) if update % log_interval == 0 or update == 1: # Calculates if value function is a good predicator of the returns (ev > 1) # or if it's just worse than predicting nothing (ev =< 0) ev = explained_variance(values, returns) logger.logkv("serial_timesteps", update * nsteps) logger.logkv("nupdates", update) logger.logkv("total_timesteps", update * nbatch) logger.logkv("fps", fps) logger.logkv("explained_variance", float(ev)) logger.logkv('eprewmean', safemean([epinfo['r'] for epinfo in epinfobuf])) logger.logkv('eplenmean', safemean([epinfo['l'] for epinfo in epinfobuf])) if eval_env is not None: logger.logkv( 'eval_eprewmean', safemean([epinfo['r'] for epinfo in eval_epinfobuf])) logger.logkv( 'eval_eplenmean', safemean([epinfo['l'] for epinfo in eval_epinfobuf])) logger.logkv('time_elapsed', tnow - tfirststart) for (lossval, lossname) in zip(lossvals, model.loss_names): logger.logkv(lossname, lossval) if MPI is None or MPI.COMM_WORLD.Get_rank() == 0: logger.dumpkvs() return model
def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None, encoded_x=None): ob_space = env.observation_space extra_tensors = {} if observ_placeholder is None: X = observation_placeholder(ob_space, batch_size=nbatch) if normalize_observations and X.dtype == tf.float32: new_encoded_x, rms = _normalize_clip_observation(X) extra_tensors['rms'] = rms else: new_encoded_x = X new_encoded_x = encode_observation(ob_space, new_encoded_x) new_encoded_x = get_network_builder("cnn")( **policy_kwargs)(new_encoded_x) else: X = observ_placeholder new_encoded_x = encoded_x with tf.variable_scope('pi' + str(head), reuse=tf.AUTO_REUSE): policy_latent = policy_network(new_encoded_x) if isinstance(policy_latent, tuple): policy_latent, recurrent_tensors = policy_latent if recurrent_tensors is not None: # recurrent architecture, need a few more steps nenv = nbatch // nsteps assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format( nbatch, nsteps) policy_latent, recurrent_tensors = policy_network( new_encoded_x, nenv) extra_tensors.update(recurrent_tensors) _v_net = value_network if _v_net is None or _v_net == 'shared': vf_latent = policy_latent else: if _v_net == 'copy': _v_net = policy_network else: assert callable(_v_net) with tf.variable_scope('vf' + str(head), reuse=tf.AUTO_REUSE): vf_latent, _ = _v_net(new_encoded_x) policy = PolicyWithValue( env=env, observations=X, latent=policy_latent, head=head, vf_latent=vf_latent, #this is the same as policy_latent... sess=sess, estimate_q=estimate_q, **extra_tensors) #print(policy.vf) return policy, X, new_encoded_x
def build_q_func(network, num_experts, hiddens=[24], dueling=False, layer_norm=False, **network_kwargs): assert isinstance(network, str) if isinstance(network, str): from baselines.common.models import get_network_builder # with tf.variable_scope("inp"): inp_network = get_network_builder(network)(**network_kwargs) # with tf.variable_scope("bel"): bel_network = get_network_builder(network)(**network_kwargs) # def q_func_builder(input_placeholder, belief_placeholder, num_actions, scope, reuse=False): # with tf.variable_scope(scope, reuse=reuse): # # input_placeholder = tf.Print(input_placeholder, [input_placeholder], '>>>> INPUT: ', summarize=100) # latent_inp = inp_network(input_placeholder) # if isinstance(latent_inp, tuple): # if latent_inp[1] is not None: # raise NotImplementedError("DQN is not compatible with recurrent policies yet") # latent_inp = latent_inp[0] # latent_inp = layers.flatten(latent_inp) # # Experts do not share # out_list =[] # #warms = [[[0.85, -10, 1]], [[0.85, 1, -10]]] # with tf.variable_scope(scope + "_experts", reuse=reuse): # for i in range(num_experts): # scope_net = "action_value_expert_" + str(i) # with tf.variable_scope(scope_net): # latent_inp_exp = inp_network(input_placeholder) # if isinstance(latent_inp_exp, tuple): # if latent_inp_exp[1] is not None: # raise NotImplementedError("DQN is not compatible with recurrent policies yet") # latent_inp_exp = latent_inp_exp[0] # action_out = layers.flatten(latent_inp_exp) # # for hidden in hiddens: # # action_out = layers.fully_connected(action_out, num_outputs=hidden, activation_fn=None) # # if layer_norm: # # action_out = layers.layer_norm(action_out, center=True, scale=True) # # action_out = tf.nn.relu(action_out) # action_scores = layers.fully_connected(action_out, num_outputs=num_actions, activation_fn=None) # if dueling: # with tf.variable_scope("state_value"): # state_out = latent_inp_exp # # for hidden in hiddens: # # state_out = layers.fully_connected(state_out, num_outputs=hidden, activation_fn=None) # # if layer_norm: # # state_out = layers.layer_norm(state_out, center=True, scale=True) # # state_out = tf.nn.relu(state_out) # state_score = layers.fully_connected(state_out, num_outputs=1, activation_fn=None) # action_scores_mean = tf.reduce_mean(action_scores, 1) # action_scores_centered = action_scores - tf.expand_dims(action_scores_mean, 1) # q_out = state_score + action_scores_centered # else: # q_out = action_scores # #action_scores = tf.tile(tf.Variable(warms[i], name='var'), [tf.shape(belief_placeholder)[0], 1]) # out_list.append(action_scores) # # import IPython; IPython.embed(); import sys; sys.exit() # # This is for tiger, fixed # # out_list = [tf.tile([[-0.05,1,-10]], [tf.shape(belief_placeholder)[0], 1]), # # tf.tile([[-0.05,-10,1]], [tf.shape(belief_placeholder)[0], 1])] # # belief_placeholder = tf.Print(belief_placeholder, [belief_placeholder], '>>>> BELIEF: ', summarize=100) # stacked_q_values = tf.stack(out_list, axis=0) # qmean = [] # for i in range(num_actions): # qmean += [tf.transpose(belief_placeholder) * stacked_q_values[:,:,i]] # qmean = tf.math.reduce_sum(tf.stack(qmean, axis=2), axis=0) # # qmean = tf.Print(qmean, [qmean], '>>>> QMEAN: ', summarize=3) # with tf.variable_scope(scope, reuse=reuse): # with tf.variable_scope("bel", reuse=reuse): # # residual network takes both input and bel # latent_bel = bel_network(belief_placeholder) # if isinstance(latent_bel, tuple): # if latent_bel[1] is not None: # raise NotImplementedError("DQN is not compatible with recurrent policies yet") # latent_bel = latent_bel[0] # latent_bel = layers.flatten(latent_bel) # # latent_inp = tf.Print(latent_inp, [latent_inp], '>>>> LATENT_INP: ', summarize=64 * 3) # # latent_bel = tf.Print(latent_bel, [latent_bel], '>>>> LATENT_BEL: ', summarize=64 * 3) # stacked = tf.stack([latent_inp, latent_bel], axis=1) # latent = layers.flatten(stacked) # with tf.variable_scope("action_value"): # action_out = latent # # for hidden in hiddens: # # action_out = layers.fully_connected(action_out, num_outputs=hidden, activation_fn=tf.nn.relu) # # if layer_norm: # # action_out = layers.layer_norm(action_out, center=True, scale=True) # # action_out = tf.nn.relu(action_out) # action_scores = layers.fully_connected(action_out, num_outputs=num_actions, activation_fn=None) # # with tf.variable_scope("alpha"): # # alpha = latent # # for hidden in hiddens: # # alpha = layers.fully_connected(alpha, num_outputs=hidden, activation_fn=tf.nn.relu) # # # if layer_norm: # # alpha = layers.layer_norm(alpha, center=True, scale=True) # # # alpha = tf.nn.relu(alpha) # # alpha = layers.fully_connected(alpha, num_outputs=1, activation_fn=tf.nn.sigmoid) # #alpha = tf.Print(alpha, [alpha], '>>>> alpha :', summarize=3) # # action_scores = tf.Print(action_scores, [action_scores], '>>>> action_scores 1 :', summarize=3) # # action_scores = tf.Print(action_scores, [action_scores], '>>>> action_scores 2 :', summarize=3) # if dueling: # with tf.variable_scope("state_value"): # state_out = latent # for hidden in hiddens: # state_out = layers.fully_connected(state_out, num_outputs=hidden, activation_fn=None) # if layer_norm: # state_out = layers.layer_norm(state_out, center=True, scale=True) # state_out = tf.nn.relu(state_out) # state_score = layers.fully_connected(state_out, num_outputs=1, activation_fn=None) # action_scores_mean = tf.reduce_mean(action_scores, 1) # action_scores_centered = action_scores - tf.expand_dims(action_scores_mean, 1) # q_out = state_score + action_scores_centered # else: # q_out = action_scores # entropy = 0.0 * -1 * tf.reduce_sum(belief_placeholder * tf.log(belief_placeholder + 1e-5), axis=1) # entropy = tf.tile(tf.reshape(entropy, [tf.shape(entropy)[0], 1]), [1, tf.shape(qmean)[1]]) # q_out = q_out * entropy + (1.0 - entropy) * qmean # #q_out = tf.Print(q_out, [q_out], '>>>> QOUT :', summarize=3) # return qmean, stacked_q_values # should be q_out, stacked_q_values def q_func_builder(input_placeholder, belief_placeholder, num_actions, scope, reuse=False): with tf.variable_scope(scope, reuse=reuse): print("Scope", scope, reuse, input_placeholder) latent = inp_network(input_placeholder) if isinstance(latent, tuple): if latent[1] is not None: raise NotImplementedError( "DQN is not compatible with recurrent policies yet") latent = latent[0] latent = layers.flatten(latent) with tf.variable_scope("action_value"): action_out = latent # for hidden in hiddens: # action_out = layers.fully_connected(action_out, num_outputs=hidden, activation_fn=None) # if layer_norm: # action_out = layers.layer_norm(action_out, center=True, scale=True) # action_out = tf.nn.relu(action_out) action_scores = layers.fully_connected(action_out, num_outputs=num_actions, activation_fn=None) if dueling: with tf.variable_scope("state_value"): state_out = latent # for hidden in hiddens: # state_out = layers.fully_connected(state_out, num_outputs=hidden, activation_fn=None) # if layer_norm: # state_out = layers.layer_norm(state_out, center=True, scale=True) # state_out = tf.nn.relu(state_out) state_score = layers.fully_connected(state_out, num_outputs=1, activation_fn=None) action_scores_mean = tf.reduce_mean(action_scores, 1) action_scores_centered = action_scores - tf.expand_dims( action_scores_mean, 1) q_out = state_score + action_scores_centered else: q_out = action_scores # import IPython; IPython.embed(); import sys; sys.exit(0) return q_out, tf.expand_dims(q_out, -1) return q_func_builder
def __init__(self, agent, network, nsteps, rho, max_kl, ent_coef, vf_stepsize, vf_iters, cg_damping, cg_iters, seed, load_path, **network_kwargs): super(AgentModel, self).__init__(name='MATRPOModel') self.agent = agent self.nsteps = nsteps self.rho = rho self.max_kl = max_kl self.ent_coef = ent_coef self.cg_damping = cg_damping self.cg_iters = cg_iters self.vf_stepsize = vf_stepsize self.vf_iters = vf_iters set_global_seeds(seed) np.set_printoptions(precision=3) if MPI is not None: self.nworkers = MPI.COMM_WORLD.Get_size() self.rank = MPI.COMM_WORLD.Get_rank() else: self.nworkers = 1 self.rank = 0 # Setup losses and stuff # ---------------------------------------- ob_space = agent.observation_space ac_space = agent.action_space if isinstance(network, str): network = get_network_builder(network)(**network_kwargs) with tf.name_scope(agent.name): with tf.name_scope("pi"): pi_policy_network = network(ob_space.shape) pi_value_network = network(ob_space.shape) self.pi = pi = PolicyWithValue(ac_space, pi_policy_network, pi_value_network) with tf.name_scope("oldpi"): old_pi_policy_network = network(ob_space.shape) old_pi_value_network = network(ob_space.shape) self.oldpi = oldpi = PolicyWithValue(ac_space, old_pi_policy_network, old_pi_value_network) self.comm_matrix = agent.comm_matrix.copy() self.estimates = np.ones([agent.nmates, nsteps], dtype=np.float32) self.multipliers = np.zeros([self.agent.nmates, self.nsteps]).astype(np.float32) for i, comm_i in enumerate(self.comm_matrix): self.estimates[i] = comm_i[self.agent.id] * self.estimates[i] pi_var_list = pi_policy_network.trainable_variables + list( pi.pdtype.trainable_variables) old_pi_var_list = old_pi_policy_network.trainable_variables + list( oldpi.pdtype.trainable_variables) vf_var_list = pi_value_network.trainable_variables + pi.value_fc.trainable_variables old_vf_var_list = old_pi_value_network.trainable_variables + oldpi.value_fc.trainable_variables self.pi_var_list = pi_var_list self.old_pi_var_list = old_pi_var_list self.vf_var_list = vf_var_list self.old_vf_var_list = old_vf_var_list if load_path is not None: load_path = osp.expanduser(load_path) ckpt = tf.train.Checkpoint(model=pi) manager = tf.train.CheckpointManager(ckpt, load_path, max_to_keep=None) ckpt.restore(manager.latest_checkpoint) self.vfadam = MpiAdam(vf_var_list) self.get_flat = U.GetFlat(pi_var_list) self.set_from_flat = U.SetFromFlat(pi_var_list) self.loss_names = [ "Lagrange", "surrgain", "sync", "meankl", "entloss", "entropy" ] self.shapes = [var.get_shape().as_list() for var in pi_var_list]