def setup_model(self):
with SetVerbosity(self.verbose):
assert issubclass(self.policy, ActorCriticPolicy), "Error: the input policy for the A2C model must be an " \
"instance of common.policies.ActorCriticPolicy."
self.graph = tf.Graph()
with self.graph.as_default():
self.set_random_seed(self.seed)
self.sess = tf_util.make_session(num_cpu=self.n_cpu_tf_sess,
graph=self.graph)
self.n_batch = self.n_envs * self.n_steps
n_batch_step = None
n_batch_train = None
if issubclass(self.policy, RecurrentActorCriticPolicy):
n_batch_step = self.n_envs
n_batch_train = self.n_envs * self.n_steps
step_model = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
1,
n_batch_step,
reuse=False,
**self.policy_kwargs)
with tf.compat.v1.variable_scope(
"train_model",
reuse=True,
custom_getter=tf_util.outer_scope_getter(
"train_model")):
train_model = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
self.n_steps,
n_batch_train,
reuse=True,
**self.policy_kwargs)
with tf.compat.v1.variable_scope("loss", reuse=False):
self.actions_ph = train_model.pdtype.sample_placeholder(
[None], name="action_ph")
self.advs_ph = tf.compat.v1.placeholder(tf.float32, [None],
name="advs_ph")
self.rewards_ph = tf.compat.v1.placeholder(
tf.float32, [None], name="rewards_ph")
self.learning_rate_ph = tf.compat.v1.placeholder(
tf.float32, [], name="learning_rate_ph")
neglogpac = train_model.proba_distribution.neglogp(
self.actions_ph)
self.entropy = tf.reduce_mean(
input_tensor=train_model.proba_distribution.entropy())
self.pg_loss = tf.reduce_mean(input_tensor=self.advs_ph *
neglogpac)
self.vf_loss = mse(tf.squeeze(train_model.value_flat),
self.rewards_ph)
# https://arxiv.org/pdf/1708.04782.pdf#page=9, https://arxiv.org/pdf/1602.01783.pdf#page=4
# and https://github.com/dennybritz/reinforcement-learning/issues/34
# suggest to add an entropy component in order to improve exploration.
loss = self.pg_loss - self.entropy * self.ent_coef + self.vf_loss * self.vf_coef
tf.compat.v1.summary.scalar('entropy_loss', self.entropy)
tf.compat.v1.summary.scalar('policy_gradient_loss',
self.pg_loss)
tf.compat.v1.summary.scalar('value_function_loss',
self.vf_loss)
tf.compat.v1.summary.scalar('loss', loss)
self.params = tf_util.get_trainable_vars("model")
grads = tf.gradients(ys=loss, xs=self.params)
if self.max_grad_norm is not None:
grads, _ = tf.clip_by_global_norm(
grads, self.max_grad_norm)
grads = list(zip(grads, self.params))
with tf.compat.v1.variable_scope("input_info", reuse=False):
tf.compat.v1.summary.scalar(
'discounted_rewards',
tf.reduce_mean(input_tensor=self.rewards_ph))
tf.compat.v1.summary.scalar(
'learning_rate',
tf.reduce_mean(input_tensor=self.learning_rate_ph))
tf.compat.v1.summary.scalar(
'advantage', tf.reduce_mean(input_tensor=self.advs_ph))
if self.full_tensorboard_log:
tf.compat.v1.summary.histogram('discounted_rewards',
self.rewards_ph)
tf.compat.v1.summary.histogram('learning_rate',
self.learning_rate_ph)
tf.compat.v1.summary.histogram('advantage',
self.advs_ph)
if tf_util.is_image(self.observation_space):
tf.compat.v1.summary.image('observation',
train_model.obs_ph)
else:
tf.compat.v1.summary.histogram(
'observation', train_model.obs_ph)
trainer = tf.compat.v1.train.RMSPropOptimizer(
learning_rate=self.learning_rate_ph,
decay=self.alpha,
epsilon=self.epsilon,
momentum=self.momentum)
self.apply_backprop = trainer.apply_gradients(grads)
self.train_model = train_model
self.step_model = step_model
self.step = step_model.step
self.proba_step = step_model.proba_step
self.value = step_model.value
self.initial_state = step_model.initial_state
tf.compat.v1.global_variables_initializer().run(
session=self.sess)
self.summary = tf.compat.v1.summary.merge_all()
def learn(self,
total_timesteps,
callback=None,
log_interval=1,
tb_log_name="ACKTR",
reset_num_timesteps=True):
new_tb_log = self._init_num_timesteps(reset_num_timesteps)
callback = self._init_callback(callback)
with SetVerbosity(self.verbose), TensorboardWriter(
self.graph, self.tensorboard_log, tb_log_name,
new_tb_log) as writer:
self._setup_learn()
self.n_batch = self.n_envs * self.n_steps
self.learning_rate_schedule = Scheduler(
initial_value=self.learning_rate,
n_values=total_timesteps,
schedule=self.lr_schedule)
# FIFO queue of the q_runner thread is closed at the end of the learn function.
# As a result, it needs to be redefinied at every call
with self.graph.as_default():
with tf.compat.v1.variable_scope(
"kfac_apply",
reuse=self.trained,
custom_getter=tf_util.outer_scope_getter(
"kfac_apply")):
# Some of the variables are not in a scope when they are create
# so we make a note of any previously uninitialized variables
tf_vars = tf.compat.v1.global_variables()
is_uninitialized = self.sess.run([
tf.compat.v1.is_variable_initialized(var)
for var in tf_vars
])
old_uninitialized_vars = [
v for (v, f) in zip(tf_vars, is_uninitialized) if not f
]
self.train_op, self.q_runner = self.optim.apply_gradients(
list(zip(self.grads_check, self.params)))
# then we check for new uninitialized variables and initialize them
tf_vars = tf.compat.v1.global_variables()
is_uninitialized = self.sess.run([
tf.compat.v1.is_variable_initialized(var)
for var in tf_vars
])
new_uninitialized_vars = [
v for (v, f) in zip(tf_vars, is_uninitialized)
if not f and v not in old_uninitialized_vars
]
if len(new_uninitialized_vars) != 0:
self.sess.run(
tf.compat.v1.variables_initializer(
new_uninitialized_vars))
self.trained = True
t_start = time.time()
coord = tf.train.Coordinator()
if self.q_runner is not None:
enqueue_threads = self.q_runner.create_threads(self.sess,
coord=coord,
start=True)
else:
enqueue_threads = []
callback.on_training_start(locals(), globals())
for update in range(1, total_timesteps // self.n_batch + 1):
callback.on_rollout_start()
# pytype:disable=bad-unpacking
# true_reward is the reward without discount
if isinstance(self.runner, PPO2Runner):
# We are using GAE
rollout = self.runner.run(callback)
obs, returns, masks, actions, values, _, states, ep_infos, true_reward = rollout
else:
rollout = self.runner.run(callback)
obs, states, returns, masks, actions, values, ep_infos, true_reward = rollout
# pytype:enable=bad-unpacking
callback.update_locals(locals())
callback.on_rollout_end()
# Early stopping due to the callback
if not self.runner.continue_training:
break
self.ep_info_buf.extend(ep_infos)
policy_loss, value_loss, policy_entropy = self._train_step(
obs, states, returns, masks, actions, values,
self.num_timesteps // (self.n_batch + 1), writer)
n_seconds = time.time() - t_start
fps = int((update * self.n_batch) / n_seconds)
if writer is not None:
total_episode_reward_logger(
self.episode_reward,
true_reward.reshape((self.n_envs, self.n_steps)),
masks.reshape((self.n_envs, self.n_steps)), writer,
self.num_timesteps)
if self.verbose >= 1 and (update % log_interval == 0
or update == 1):
explained_var = explained_variance(values, returns)
logger.record_tabular("nupdates", update)
logger.record_tabular("total_timesteps",
self.num_timesteps)
logger.record_tabular("fps", fps)
logger.record_tabular("policy_entropy",
float(policy_entropy))
logger.record_tabular("policy_loss", float(policy_loss))
logger.record_tabular("value_loss", float(value_loss))
logger.record_tabular("explained_variance",
float(explained_var))
if len(self.ep_info_buf) > 0 and len(
self.ep_info_buf[0]) > 0:
logger.logkv(
'ep_reward_mean',
safe_mean([
ep_info['r'] for ep_info in self.ep_info_buf
]))
logger.logkv(
'ep_normal_mean',
safe_mean([
ep_info['n'] for ep_info in self.ep_info_buf
]))
logger.logkv(
'ep_attack_mean',
safe_mean([
ep_info['a'] for ep_info in self.ep_info_buf
]))
logger.logkv(
'ep_precision_mean',
safe_mean([
ep_info['p'] for ep_info in self.ep_info_buf
]))
logger.dump_tabular()
coord.request_stop()
coord.join(enqueue_threads)
callback.on_training_end()
return self
def setup_model(self):
with SetVerbosity(self.verbose):
assert issubclass(self.policy, ActorCriticPolicy), "Error: the input policy for the PPO2 model must be " \
"an instance of common.policies.ActorCriticPolicy."
self.n_batch = self.n_envs * self.n_runs * self.n_steps
self.graph = tf.Graph()
with self.graph.as_default():
self.set_random_seed(self.seed)
self.sess = tf_util.make_session(num_cpu=self.n_cpu_tf_sess, graph=self.graph)
n_batch_step = None
n_batch_train = None
if issubclass(self.policy, RecurrentActorCriticPolicy):
assert self.n_envs % self.nminibatches == 0, "For recurrent policies, "\
"the number of environments run in parallel should be a multiple of nminibatches."
n_batch_step = self.n_envs
n_batch_train = self.n_batch // self.nminibatches
act_model = self.policy(self.sess, self.observation_space, self.action_space, self.n_envs, 1, n_batch_step, reuse=False, **self.policy_kwargs)
with tf.compat.v1.variable_scope("train_model", reuse=True, custom_getter=tf_util.outer_scope_getter("train_model")):
train_model = self.policy(self.sess, self.observation_space, self.action_space, self.n_envs * self.n_runs // self.nminibatches, self.n_steps, n_batch_train, reuse=True, **self.policy_kwargs)
self.observation_ph = tf.compat.v1.placeholder(shape=(None,) + self.observation_space.shape, dtype=self.observation_space.dtype, name='obs')
self.processed_obs = tf.cast(self.observation_ph, tf.float32)
self.observation_next_ph = tf.compat.v1.placeholder(shape=(None,) + self.observation_space.shape, dtype=self.observation_space.dtype, name='obs_next')
self.processed_obs_next = tf.cast(self.observation_next_ph, tf.float32)
with tf.compat.v1.variable_scope("obs_encoded", reuse=tf.compat.v1.AUTO_REUSE):
self.obs_encoded = obs_autoencoder(self.processed_obs, self.observation_space)
self.obs_next_encoded = obs_autoencoder(self.processed_obs_next, self.observation_space)
#self.obs_encoded = self.processed_obs
#self.obs_next_encoded = self.processed_obs_next
self.act_hat = inverse_model(self.obs_encoded, self.obs_next_encoded, self.action_space)
with tf.compat.v1.variable_scope("loss", reuse=False):
self.action_ph = train_model.pdtype.sample_placeholder([None], name="action_ph")
self.processed_act = tf.cast(tf.one_hot(self.action_ph, self.action_space.n), tf.float32)
self.obs_next_hat = forward_model(self.obs_encoded, self.processed_act, self.observation_space)
self.advs_ph = tf.compat.v1.placeholder(tf.float32, [None], name="advs_ph")
self.rewards_ph = tf.compat.v1.placeholder(tf.float32, [None], name="rewards_ph")
self.true_rewards_ph = tf.compat.v1.placeholder(tf.float32, [None], name="true_rewards_ph")
self.old_neglog_pac_ph = tf.compat.v1.placeholder(tf.float32, [None], name="old_neglog_pac_ph")
self.old_vpred_ph = tf.compat.v1.placeholder(tf.float32, [None], name="old_vpred_ph")
self.learning_rate_ph = tf.compat.v1.placeholder(tf.float32, [], name="learning_rate_ph")
self.clip_range_ph = tf.compat.v1.placeholder(tf.float32, [], name="clip_range_ph")
neglogpac = train_model.proba_distribution.neglogp(self.action_ph)
self.entropy = tf.reduce_mean(input_tensor=train_model.proba_distribution.entropy())
vpred = train_model.value_flat
# Value function clipping: not present in the original PPO
if self.cliprange_vf is None:
# Default behavior (legacy from OpenAI baselines):
# use the same clipping as for the policy
self.clip_range_vf_ph = self.clip_range_ph
self.cliprange_vf = self.cliprange
elif isinstance(self.cliprange_vf, (float, int)) and self.cliprange_vf < 0:
# Original PPO implementation: no value function clipping
self.clip_range_vf_ph = None
else:
# Last possible behavior: clipping range
# specific to the value function
self.clip_range_vf_ph = tf.compat.v1.placeholder(tf.float32, [], name="clip_range_vf_ph")
if self.clip_range_vf_ph is None:
# No clipping
vpred_clipped = train_model.value_flat
else:
# Clip the different between old and new value
# NOTE: this depends on the reward scaling
vpred_clipped = self.old_vpred_ph + tf.clip_by_value(train_model.value_flat - self.old_vpred_ph, - self.clip_range_vf_ph, self.clip_range_vf_ph)
vf_losses1 = tf.square(vpred - self.rewards_ph)
vf_losses2 = tf.square(vpred_clipped - self.rewards_ph)
self.vf_loss = .5 * tf.reduce_mean(input_tensor=tf.maximum(vf_losses1, vf_losses2))
ratio = tf.exp(self.old_neglog_pac_ph - neglogpac)
pg_losses = -self.advs_ph * ratio
pg_losses2 = -self.advs_ph * tf.clip_by_value(ratio, 1.0 - self.clip_range_ph, 1.0 + self.clip_range_ph)
self.pg_loss = tf.reduce_mean(input_tensor=tf.maximum(pg_losses, pg_losses2))
self.approxkl = .5 * tf.reduce_mean(input_tensor=tf.square(neglogpac - self.old_neglog_pac_ph))
self.clipfrac = tf.reduce_mean(input_tensor=tf.cast(tf.greater(tf.abs(ratio - 1.0), self.clip_range_ph), tf.float32))
self.params = tf.compat.v1.trainable_variables()
weight_params = [v for v in self.params if '/b' not in v.name]
l2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in weight_params])
self.frw_loss = 0.5 * tf.reduce_sum(tf.math.square(self.obs_next_encoded - self.obs_next_hat))
#self.inv_loss = - tf.reduce_sum(self.processed_act * tf.math.log(self.act_hat + tf.keras.backend.epsilon()))
self.inv_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.act_hat, labels=tf.cast(self.action_ph, tf.int64)))
self.int_loss = self.beta * self.frw_loss + (1.0 - self.beta) * self.inv_loss
loss = self.lmd * (self.pg_loss - self.entropy * self.ent_coef + self.vf_loss * self.vf_coef) + self.int_loss
self.int_reward = self.eta * self.frw_loss
tf.compat.v1.summary.scalar('entropy_loss', self.entropy)
tf.compat.v1.summary.scalar('policy_gradient_loss', self.pg_loss)
tf.compat.v1.summary.scalar('value_function_loss', self.vf_loss)
tf.compat.v1.summary.scalar('intrinsic_loss', self.int_loss)
tf.compat.v1.summary.scalar('approximate_kullback-leibler', self.approxkl)
tf.compat.v1.summary.scalar('clip_factor', self.clipfrac)
tf.compat.v1.summary.scalar('loss', loss)
for var in self.params:
print(var.name, var)
with tf.compat.v1.variable_scope('model'):
if self.full_tensorboard_log:
for var in self.params:
tf.compat.v1.summary.histogram(var.name, var)
grads = tf.gradients(ys=loss, xs=self.params)
if self.max_grad_norm is not None:
grads, _grad_norm = tf.clip_by_global_norm(grads, self.max_grad_norm)
grads = list(zip(grads, self.params))
for gr in grads:
print(gr)
trainer = tf.compat.v1.train.AdamOptimizer(learning_rate=self.learning_rate_ph, epsilon=1e-5)
self._train = trainer.apply_gradients(grads)
self.loss_names = ['policy_loss', 'value_loss', 'int_loss', 'policy_entropy', 'approxkl', 'clipfrac']
with tf.compat.v1.variable_scope("input_info", reuse=False):
tf.compat.v1.summary.scalar('true_rewards', tf.reduce_mean(input_tensor=self.true_rewards_ph))
tf.compat.v1.summary.scalar('discounted_rewards', tf.reduce_mean(input_tensor=self.rewards_ph))
tf.compat.v1.summary.scalar('learning_rate', tf.reduce_mean(input_tensor=self.learning_rate_ph))
tf.compat.v1.summary.scalar('advantage', tf.reduce_mean(input_tensor=self.advs_ph))
tf.compat.v1.summary.scalar('clip_range', tf.reduce_mean(input_tensor=self.clip_range_ph))
if self.clip_range_vf_ph is not None:
tf.compat.v1.summary.scalar('clip_range_vf', tf.reduce_mean(input_tensor=self.clip_range_vf_ph))
tf.compat.v1.summary.scalar('old_neglog_action_probability', tf.reduce_mean(input_tensor=self.old_neglog_pac_ph))
tf.compat.v1.summary.scalar('old_value_pred', tf.reduce_mean(input_tensor=self.old_vpred_ph))
if self.full_tensorboard_log:
tf.compat.v1.summary.histogram('discounted_rewards', self.rewards_ph)
tf.compat.v1.summary.histogram('learning_rate', self.learning_rate_ph)
tf.compat.v1.summary.histogram('advantage', self.advs_ph)
tf.compat.v1.summary.histogram('clip_range', self.clip_range_ph)
tf.compat.v1.summary.histogram('old_neglog_action_probability', self.old_neglog_pac_ph)
tf.compat.v1.summary.histogram('old_value_pred', self.old_vpred_ph)
if tf_util.is_image(self.observation_space):
tf.compat.v1.summary.image('observation', train_model.obs_ph)
else:
tf.compat.v1.summary.histogram('observation', train_model.obs_ph)
self.train_model = train_model
self.act_model = act_model
self.step = act_model.step
self.proba_step = act_model.proba_step
self.value = act_model.value
self.initial_state = act_model.initial_state
tf.compat.v1.global_variables_initializer().run(session=self.sess) # pylint: disable=E1101
self.summary = tf.compat.v1.summary.merge_all()
def setup_model(self):
with SetVerbosity(self.verbose):
assert issubclass(self.policy, ActorCriticPolicy), "Error: the input policy for the ACKTR model must be " \
"an instance of common.policies.ActorCriticPolicy."
# Enable continuous actions tricks (normalized advantage)
self.continuous_actions = isinstance(self.action_space, Box)
self.graph = tf.Graph()
with self.graph.as_default():
self.set_random_seed(self.seed)
self.sess = tf_util.make_session(num_cpu=self.n_cpu_tf_sess,
graph=self.graph)
n_batch_step = None
n_batch_train = None
if issubclass(self.policy, RecurrentActorCriticPolicy):
n_batch_step = self.n_envs
n_batch_train = self.n_envs * self.n_steps
step_model = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
1,
n_batch_step,
reuse=False,
**self.policy_kwargs)
self.params = params = tf_util.get_trainable_vars("model")
with tf.compat.v1.variable_scope(
"train_model",
reuse=True,
custom_getter=tf_util.outer_scope_getter(
"train_model")):
train_model = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
self.n_steps,
n_batch_train,
reuse=True,
**self.policy_kwargs)
with tf.compat.v1.variable_scope(
"loss",
reuse=False,
custom_getter=tf_util.outer_scope_getter("loss")):
self.advs_ph = advs_ph = tf.compat.v1.placeholder(
tf.float32, [None])
self.rewards_ph = rewards_ph = tf.compat.v1.placeholder(
tf.float32, [None])
self.learning_rate_ph = learning_rate_ph = tf.compat.v1.placeholder(
tf.float32, [])
self.actions_ph = train_model.pdtype.sample_placeholder(
[None])
neg_log_prob = train_model.proba_distribution.neglogp(
self.actions_ph)
# training loss
pg_loss = tf.reduce_mean(input_tensor=advs_ph *
neg_log_prob)
self.entropy = entropy = tf.reduce_mean(
input_tensor=train_model.proba_distribution.entropy())
self.pg_loss = pg_loss = pg_loss - self.ent_coef * entropy
self.vf_loss = vf_loss = mse(
tf.squeeze(train_model.value_fn), rewards_ph)
train_loss = pg_loss + self.vf_coef * vf_loss
# Fisher loss construction
self.pg_fisher = pg_fisher_loss = -tf.reduce_mean(
input_tensor=neg_log_prob)
sample_net = train_model.value_fn + tf.random.normal(
tf.shape(input=train_model.value_fn))
self.vf_fisher = vf_fisher_loss = -self.vf_fisher_coef * tf.reduce_mean(
input_tensor=tf.pow(
train_model.value_fn -
tf.stop_gradient(sample_net), 2))
self.joint_fisher = pg_fisher_loss + vf_fisher_loss
tf.compat.v1.summary.scalar('entropy_loss', self.entropy)
tf.compat.v1.summary.scalar('policy_gradient_loss',
pg_loss)
tf.compat.v1.summary.scalar('policy_gradient_fisher_loss',
pg_fisher_loss)
tf.compat.v1.summary.scalar('value_function_loss',
self.vf_loss)
tf.compat.v1.summary.scalar('value_function_fisher_loss',
vf_fisher_loss)
tf.compat.v1.summary.scalar('loss', train_loss)
self.grads_check = tf.gradients(ys=train_loss, xs=params)
with tf.compat.v1.variable_scope("input_info", reuse=False):
tf.compat.v1.summary.scalar(
'discounted_rewards',
tf.reduce_mean(input_tensor=self.rewards_ph))
tf.compat.v1.summary.scalar(
'learning_rate',
tf.reduce_mean(input_tensor=self.learning_rate_ph))
tf.compat.v1.summary.scalar(
'advantage', tf.reduce_mean(input_tensor=self.advs_ph))
if self.full_tensorboard_log:
tf.compat.v1.summary.histogram('discounted_rewards',
self.rewards_ph)
tf.compat.v1.summary.histogram('learning_rate',
self.learning_rate_ph)
tf.compat.v1.summary.histogram('advantage',
self.advs_ph)
if tf_util.is_image(self.observation_space):
tf.compat.v1.summary.image('observation',
train_model.obs_ph)
else:
tf.compat.v1.summary.histogram(
'observation', train_model.obs_ph)
with tf.compat.v1.variable_scope(
"kfac",
reuse=False,
custom_getter=tf_util.outer_scope_getter("kfac")):
with tf.device('/gpu:0'):
self.optim = optim = kfac.KfacOptimizer(
learning_rate=learning_rate_ph,
clip_kl=self.kfac_clip,
momentum=0.9,
kfac_update=self.kfac_update,
epsilon=0.01,
stats_decay=0.99,
async_eigen_decomp=self.async_eigen_decomp,
cold_iter=10,
max_grad_norm=self.max_grad_norm,
verbose=self.verbose)
optim.compute_and_apply_stats(self.joint_fisher,
var_list=params)
self.train_model = train_model
self.step_model = step_model
self.step = step_model.step
self.proba_step = step_model.proba_step
self.value = step_model.value
self.initial_state = step_model.initial_state
tf.compat.v1.global_variables_initializer().run(
session=self.sess)
self.summary = tf.compat.v1.summary.merge_all()