def __init__(self, env, ph_observations, params):
self.ph_observations = ph_observations
num_actions = env.action_space.n
obs_space = main_observation_space(env)
# Goal observation
self.ph_goal_obs = None
self.is_goal_env = is_goal_based_env(env)
if self.is_goal_env:
# goal obs has the same shape as main obs
self.ph_goal_obs = placeholder_from_space(main_observation_space(env))
make_encoder_func = make_encoder_with_goal if self.is_goal_env else make_encoder
regularizer = None # don't use L2 regularization
actor_enc_params = get_enc_params(params, 'actor')
# actor computation graph
# use actor encoder as main observation encoder (including landmarks, etc.)
if self.is_goal_env:
actor_encoder = make_encoder_func(
self.ph_observations, self.ph_goal_obs, obs_space, regularizer, actor_enc_params, name='act_enc',
)
else:
actor_encoder = make_encoder_func(
self.ph_observations, obs_space, regularizer, actor_enc_params, name='act_enc',
)
actor_model = make_model(actor_encoder.encoded_input, regularizer, params, 'act_mdl')
actions_fc = dense(actor_model.latent, params.model_fc_size // 2, regularizer)
action_logits = tf.contrib.layers.fully_connected(actions_fc, num_actions, activation_fn=None)
self.best_action_deterministic = tf.argmax(action_logits, axis=1)
self.actions_distribution = CategoricalProbabilityDistribution(action_logits)
self.act = self.actions_distribution.sample()
self.action_prob = self.actions_distribution.probability(self.act)
critic_enc_params = get_enc_params(params, 'critic')
# critic computation graph
if self.is_goal_env:
value_encoder = make_encoder_func(
self.ph_observations, self.ph_goal_obs, obs_space, regularizer, critic_enc_params, 'val_enc',
)
else:
value_encoder = make_encoder_func(
self.ph_observations, obs_space, regularizer, critic_enc_params, 'val_enc',
)
value_model = make_model(value_encoder.encoded_input, regularizer, params, 'val_mdl')
value_fc = dense(value_model.latent, params.model_fc_size // 2, regularizer)
self.value = tf.squeeze(tf.contrib.layers.fully_connected(value_fc, 1, activation_fn=None), axis=[1])
log.info('Total parameters in the model: %d', count_total_parameters())
def test_normalize(self):
env = make_doom_env(doom_env_by_name(TEST_ENV_NAME))
obs_space = main_observation_space(env)
env.reset()
obs = [env.step(0)[0] for _ in range(10)]
self.assertTrue(np.all(obs_space.low == 0))
self.assertTrue(np.all(obs_space.high == 255))
self.assertEqual(obs_space.dtype, np.uint8)
self.assertFalse(is_normalized(obs_space))
tf.reset_default_graph()
ph_obs = placeholder_from_space(obs_space)
obs_tensor = tf_normalize(ph_obs, obs_space)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
normalized_obs = sess.run(obs_tensor, feed_dict={ph_obs: obs})
self.assertEqual(normalized_obs.dtype, np.float32)
self.assertLessEqual(normalized_obs.max(), 1.0)
self.assertGreaterEqual(normalized_obs.min(), -1.0)
tf.reset_default_graph()
gc.collect()
def __init__(self, make_env_func, params):
super(AgentCuriousPPO, self).__init__(make_env_func, params)
env = self.make_env_func(
) # we need it to query observation shape, number of actions, etc.
self.ph_next_observations = placeholder_from_space(
main_observation_space(env))
self.num_actions = env.action_space.n
env.close()
if self.params.curiosity_type == 'icm':
# create graph for curiosity module (ICM)
self.curiosity = IntrinsicCuriosityModule(
env,
self.ph_observations,
self.ph_next_observations,
self.ph_actions,
params.forward_fc,
params,
)
elif self.params.curiosity_type == 'ecr':
self.curiosity = ECRModule(env, params)
elif self.params.curiosity_type == 'ecr_map':
self.curiosity = ECRMapModule(env, params)
elif self.params.curiosity_type == 'rnd':
self.curiosity = RandomNetworkDistillation(env,
self.ph_observations,
params)
else:
raise Exception(
f'Curiosity type {self.params.curiosity_type} not supported')
self.previous_actions = np.random.randint(0, self.num_actions,
self.params.num_envs)
def __init__(self, env, ph_obs, params=None):
"""
:param env
:param ph_obs - placeholder for observations
"""
with tf.variable_scope('rnd'):
self.params = params
self.ph_obs = ph_obs
reg = None # don't use regularization
obs_space = main_observation_space(env)
target_enc_params = get_enc_params(params, 'rnd_target')
encoder_obs = make_encoder(ph_obs, obs_space, reg, target_enc_params, name='target_encoder')
self.predicted_features = encoder_obs.encoded_input
predictor_enc_params = get_enc_params(params, 'rnd_predictor')
target_features = make_encoder(ph_obs, obs_space, reg, predictor_enc_params, name='predictor_encoder')
self.tgt_features = tf.stop_gradient(target_features.encoded_input)
self.feature_vector_size = self.predicted_features.get_shape().as_list()[-1]
log.info('Feature vector size in RND module: %d', self.feature_vector_size)
self.objectives = self._objectives()
self._add_summaries()
self.summaries = merge_summaries(collections=['rnd'])
self.step = tf.Variable(0, trainable=False, dtype=tf.int64, name='rnd_step')
opt = tf.train.AdamOptimizer(learning_rate=self.params.learning_rate, name='rnd_opt')
self.train_rnd = opt.minimize(self.objectives.loss, global_step=self.step)
def __init__(self, make_env_func, params):
"""Initialize PPO computation graph and some auxiliary tensors."""
super(AgentPPO, self).__init__(params)
self.actor_step = tf.Variable(0, trainable=False, dtype=tf.int64, name='actor_step')
self.critic_step = tf.Variable(0, trainable=False, dtype=tf.int64, name='critic_step')
self.make_env_func = make_env_func
env = make_env_func() # we need the env to query observation shape, number of actions, etc.
self.obs_shape = [-1] + list(main_observation_space(env).shape)
self.ph_observations = placeholder_from_space(main_observation_space(env))
self.ph_actions = placeholder_from_space(env.action_space) # actions sampled from the policy
self.ph_advantages, self.ph_returns, self.ph_old_action_probs = placeholders(None, None, None)
self.actor_critic = ActorCritic(env, self.ph_observations, self.params)
env.close()
self.objectives = self.add_ppo_objectives(
self.actor_critic,
self.ph_actions, self.ph_old_action_probs, self.ph_advantages, self.ph_returns,
self.params,
self.actor_step,
)
# optimizers
actor_opt = tf.train.AdamOptimizer(learning_rate=self.params.learning_rate, name='actor_opt')
self.train_actor = actor_opt.minimize(self.objectives.actor_loss, global_step=self.actor_step)
critic_opt = tf.train.AdamOptimizer(learning_rate=self.params.learning_rate, name='critic_opt')
self.train_critic = critic_opt.minimize(self.objectives.critic_loss, global_step=self.critic_step)
self.add_ppo_summaries()
summary_dir = summaries_dir(self.params.experiment_dir())
self.summary_writer = tf.summary.FileWriter(summary_dir)
self.actor_summaries = merge_summaries(collections=['actor'])
self.critic_summaries = merge_summaries(collections=['critic'])
if self.params.use_env_map:
self.map_img, self.coord_limits = generate_env_map(make_env_func)
def __init__(self, env, params):
obs_space = main_observation_space(env)
self.ph_obs_first, self.ph_obs_second = placeholders_from_spaces(
obs_space, obs_space)
self.ph_labels = tf.placeholder(dtype=tf.int32, shape=(None, ))
self.ph_is_training = tf.placeholder(dtype=tf.bool, shape=[])
with tf.variable_scope('distance') as scope:
log.info('Distance network graph...')
self.step = tf.Variable(0,
trainable=False,
dtype=tf.int64,
name='dist_step')
reg = tf.contrib.layers.l2_regularizer(scale=1e-5)
summary_collections = ['dist']
enc_params = EncoderParams()
enc_params.enc_name = params.distance_encoder
enc_params.batch_norm = params.distance_use_batch_norm
enc_params.ph_is_training = self.ph_is_training
enc_params.summary_collections = summary_collections
encoder = tf.make_template(
'siamese_enc',
make_encoder,
create_scope_now_=True,
obs_space=obs_space,
regularizer=reg,
enc_params=enc_params,
)
obs_first_enc = encoder(self.ph_obs_first)
obs_second_enc = encoder(self.ph_obs_second)
self.first_encoded = obs_first_enc.encoded_input
self.second_encoded = obs_second_enc.encoded_input
encoder_reg_loss = 0.0
if hasattr(obs_first_enc, 'reg_loss'):
encoder_reg_loss = obs_first_enc.reg_loss
observations_encoded = tf.concat(
[self.first_encoded, self.second_encoded], axis=1)
fc_layers = [params.distance_fc_size] * params.distance_fc_num
x = observations_encoded
for fc_layer_size in fc_layers:
x = dense(
x,
fc_layer_size,
reg,
batch_norm=params.distance_use_batch_norm,
is_training=self.ph_is_training,
)
logits = tf.contrib.layers.fully_connected(x,
2,
activation_fn=None)
self.probabilities = tf.nn.softmax(logits)
self.correct = tf.reduce_mean(
tf.to_float(
tf.equal(self.ph_labels,
tf.cast(tf.argmax(logits, axis=1), tf.int32))), )
self.dist_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=self.ph_labels)
self.dist_loss = tf.reduce_mean(self.dist_loss)
reg_losses = tf.losses.get_regularization_losses(scope=scope.name)
self.reg_loss = tf.reduce_sum(reg_losses) + encoder_reg_loss
self.loss = self.dist_loss + self.reg_loss
# helpers to encode observations (saves time)
# does not matter if we use first vs second here
self.ph_obs = self.ph_obs_first
self.encoded_observation = self.first_encoded
self._add_summaries(summary_collections)
self.summaries = merge_summaries(collections=summary_collections)
opt = tf.train.AdamOptimizer(learning_rate=params.learning_rate,
name='dist_opt')
with tf.control_dependencies(
tf.get_collection(tf.GraphKeys.UPDATE_OPS,
scope='distance')):
self.train_op = opt.minimize(self.loss, global_step=self.step)
# other stuff not related to computation graph
self.obs_encoder = ObservationEncoder(
encode_func=self.encode_observation)
def __init__(self, env, params):
obs_space = main_observation_space(env)
self.ph_obs_prev, self.ph_obs_curr, self.ph_obs_goal = placeholders_from_spaces(
obs_space, obs_space, obs_space)
self.ph_actions = placeholder_from_space(env.action_space)
self.ph_is_training = tf.placeholder(dtype=tf.bool, shape=[])
with tf.variable_scope('loco') as scope:
log.info('Locomotion network graph...')
self.step = tf.Variable(0,
trainable=False,
dtype=tf.int64,
name='loco_step')
reg = tf.contrib.layers.l2_regularizer(scale=1e-5)
enc_params = EncoderParams()
enc_params.enc_name = params.locomotion_encoder
enc_params.batch_norm = params.locomotion_use_batch_norm
enc_params.ph_is_training = self.ph_is_training
enc_params.summary_collections = ['loco']
encoder = tf.make_template(
'enc_loco',
make_encoder,
create_scope_now_=True,
obs_space=obs_space,
regularizer=reg,
enc_params=enc_params,
)
if params.locomotion_siamese:
obs_curr_encoder = encoder(self.ph_obs_curr)
obs_curr_encoded = obs_curr_encoder.encoded_input
obs_goal_encoder = encoder(self.ph_obs_goal)
obs_goal_encoded = obs_goal_encoder.encoded_input
obs_encoder = obs_curr_encoder # any of the two
obs_encoded = tf.concat([obs_curr_encoded, obs_goal_encoded],
axis=1)
else:
if params.locomotion_use_prev:
obs_concat = tf.concat(
[self.ph_obs_prev, self.ph_obs_curr, self.ph_obs_goal],
axis=3)
else:
obs_concat = tf.concat(
[self.ph_obs_curr, self.ph_obs_goal], axis=3)
obs_encoder = encoder(obs_concat)
obs_encoded = obs_encoder.encoded_input
encoder_reg_loss = 0.0
if hasattr(obs_encoder, 'reg_loss'):
encoder_reg_loss = obs_encoder.reg_loss
fc_layers = [params.locomotion_fc_size] * params.locomotion_fc_num
x = obs_encoded
for fc_layer_size in fc_layers:
x = dense(
x,
fc_layer_size,
regularizer=reg,
batch_norm=params.locomotion_use_batch_norm,
is_training=self.ph_is_training,
)
action_logits = tf.layers.dense(x,
env.action_space.n,
activation=None)
self.actions_distribution = CategoricalProbabilityDistribution(
action_logits)
self.best_action_deterministic = tf.argmax(action_logits, axis=1)
self.act = self.actions_distribution.sample()
self.correct = tf.reduce_mean(
tf.to_float(
tf.equal(
self.ph_actions,
tf.cast(tf.argmax(action_logits, axis=1),
tf.int32))), )
actions_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.ph_actions, logits=action_logits)
self.actions_loss = tf.reduce_mean(actions_loss)
reg_losses = tf.losses.get_regularization_losses(scope=scope.name)
self.reg_loss = tf.reduce_sum(reg_losses) + encoder_reg_loss
self.loss = self.actions_loss + self.reg_loss
loco_opt = tf.train.AdamOptimizer(
learning_rate=params.learning_rate, name='loco_opt')
with tf.control_dependencies(
tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope='loco')):
self.train_loco = loco_opt.minimize(self.loss,
global_step=self.step)
def __init__(self,
env,
ph_obs,
ph_next_obs,
ph_actions,
forward_fc=256,
params=None):
"""
:param ph_obs - placeholder for observations
:param ph_actions - placeholder for selected actions
"""
with tf.variable_scope('icm'):
self.params = params
self.ph_obs = ph_obs
self.ph_next_obs = ph_next_obs
self.ph_actions = ph_actions
reg = None # don't use regularization
obs_space = main_observation_space(env)
num_actions = env.action_space.n
enc_params = get_enc_params(params, 'icm_enc')
encoder_template = tf.make_template(
'obs_encoder',
make_encoder,
create_scope_now_=True,
obs_space=obs_space,
regularizer=reg,
enc_params=enc_params,
)
encoder_obs = encoder_template(ph_obs)
encoder_next_obs = encoder_template(ph_next_obs)
encoded_obs = encoder_obs.encoded_input
self.encoded_next_obs = encoder_next_obs.encoded_input
self.feature_vector_size = encoded_obs.get_shape().as_list()[-1]
log.info('Feature vector size in ICM/RND module: %d',
self.feature_vector_size)
actions_one_hot = tf.one_hot(ph_actions, num_actions)
# forward model
forward_model_input = tf.concat([encoded_obs, actions_one_hot],
axis=1)
forward_model_hidden = dense(forward_model_input, forward_fc, reg)
forward_model_hidden = dense(forward_model_hidden, forward_fc, reg)
forward_model_output = tf.contrib.layers.fully_connected(
forward_model_hidden,
self.feature_vector_size,
activation_fn=None,
)
self.predicted_features = forward_model_output
# inverse model
inverse_model_input = tf.concat(
[encoded_obs, self.encoded_next_obs], axis=1)
inverse_model_hidden = dense(inverse_model_input, 256, reg)
inverse_model_output = tf.contrib.layers.fully_connected(
inverse_model_hidden,
num_actions,
activation_fn=None,
)
self.predicted_actions = inverse_model_output
self.objectives = self._objectives()
self._add_summaries()
self.summaries = merge_summaries(collections=['icm'])
self.step = tf.Variable(0,
trainable=False,
dtype=tf.int64,
name='icm_step')
opt = tf.train.AdamOptimizer(
learning_rate=self.params.learning_rate, name='icm_opt')
self.train_icm = opt.minimize(self.objectives.loss,
global_step=self.step)