def _helper_runningmeanstd():
comm = MPI.COMM_WORLD
np.random.seed(0)
for (triple, axis) in [
((np.random.randn(3), np.random.randn(4), np.random.randn(5)), 0),
((np.random.randn(3, 2), np.random.randn(4,
2), np.random.randn(5,
2)), 0),
((np.random.randn(2, 3), np.random.randn(2,
4), np.random.randn(2,
4)), 1),
]:
x = np.concatenate(triple, axis=axis)
ms1 = [x.mean(axis=axis), x.std(axis=axis), x.shape[axis]]
ms2 = mpi_moments(triple[comm.Get_rank()], axis=axis)
for (a1, a2) in zipsame(ms1, ms2):
print(a1, a2)
assert np.allclose(a1, a2)
print("ok!")
def __init__(self, *, ob, ob_length, ent_coef, vf_coef, max_grad_norm):
sess = get_session()
# sequential state
# sequential action
decoder_input = tf.placeholder(tf.int32, [None, None])
action = tf.placeholder(tf.int32, [None, None])
action_length = tf.placeholder(tf.int32, [None])
# sequential adv
adv = tf.placeholder(tf.float32, [None, None])
# sequential reward
r = tf.placeholder(tf.float32, [None, None])
# keep track of old actor(sequential descision)
oldneglogpac = tf.placeholder(tf.float32, [None, None])
oldvpred = tf.placeholder(tf.float32, [None, None])
lr = tf.placeholder(tf.float32, [])
# Cliprange
cliprange = tf.placeholder(tf.float32, [])
train_model = Seq2seqPolicy("pi", hparams, reuse=True, encoder_inputs=ob,
encoder_lengths=ob_length,
decoder_inputs=decoder_input,
decoder_full_length=action_length,
decoder_targets=action)
act_model = Seq2seqPolicy("oldpi", hparams, reuse=False, encoder_inputs=ob,
encoder_lengths=ob_length,
decoder_inputs=decoder_input,
decoder_full_length=action_length,
decoder_targets=action)
assign_old_eq_new = U.function([], [], updates=[tf.assign(oldv, newv)
for (oldv, newv) in
zipsame(act_model.get_variables(), train_model.get_variables())])
# neglogpac = train_model.neglogp()
# Calculate the entropy
# Entropy is used to improve exploration by limiting the premature convergence to suboptimal policy.
entropy = tf.reduce_mean(train_model.entropy())
# Calculate the loss
# Total loss = Policy gradient loss - entropy * entropy coefficient + Value coefficient * value loss
# Clip the value to reduce variability during Critic training.
# Get the predicted value
vpred = train_model.vf
vpredclipped = oldvpred + tf.clip_by_value(train_model.vf - oldvpred, -cliprange, cliprange)
# Unclipped value
vf_losses1 = tf.square(vpred - r)
# Clipped value
vf_losses2 = tf.square(vpredclipped - r)
vf_loss = .5 * tf.reduce_mean(tf.maximum(vf_losses1, vf_losses2))
# Calculate ratio (pi current policy / pi old policy)
#ratio = tf.exp(oldneglogpac - neglogpac)
ratio = tf.exp(train_model.logp() - act_model.logp())
# define the loss = -J is equivalent to max J
pg_losses = -adv * ratio
pg_losses2 = -adv * tf.clip_by_value(ratio, 1.0 - cliprange, 1.0 + cliprange)
# Final pg loss
pg_loss = tf.reduce_mean(tf.maximum(pg_losses, pg_losses2))
#approxkl = .5 * tf.reduce_mean(tf.square(neglogpac - oldneglogpac))
kloldnew = act_model.kl(train_model)
approxkl = tf.reduce_mean(kloldnew)
clipfrac = tf.reduce_mean(tf.to_float(tf.greater(tf.abs(ratio - 1.0), cliprange)))
# total loss
loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef
# Update the parameters using loss
# 1. get the model parameters
params = tf.trainable_variables('pi')
# 2. Build our trainer
trainer = MpiAdamOptimizer(MPI.COMM_WORLD, learning_rate=lr, epsilon=1e-5)
# 3. Calculate the gradients
grads_and_var = trainer.compute_gradients(loss, params)
grads, var = zip(*grads_and_var)
if max_grad_norm is not None:
# Clip the gradients (normalize)
grads, _grad_norm = tf.clip_by_global_norm(grads, max_grad_norm)
grads_and_var = list(zip(grads, var))
# zip aggregate each gradient with parameters associated
# For instance zip(ABCD, xyza) => Ax, By, Cz, Da
_train = trainer.apply_gradients(grads_and_var)
# decoder_input action_length is speciallized for the training model
def train(learning_reate, clipingrange, obs, obs_length,
returns, advs, decoder_inputs, actions, decoder_full_length,
values, neglogpacs, states=None):
# the advantage function is calculated as A(s,a) = R + yV(s') - V(s)
# the return = R + yV(s')
# Sequential Normalize the advantages
advs = (advs - np.mean(advs, axis=0)) / (np.std(advs, axis=0) + 1e-8)
td_map = {train_model.encoder_inputs: obs, train_model.encoder_lengths:obs_length,
decoder_input:decoder_inputs, action: actions, action_length: decoder_full_length, adv:advs,
r: returns, lr:learning_reate, cliprange:clipingrange, oldneglogpac:neglogpacs, oldvpred: values}
return sess.run([pg_loss, vf_loss, entropy, approxkl, clipfrac, _train], td_map)[:-1]
self.loss_names = ['policy_loss', 'value_loss', 'policy_entropy', 'approxkl', 'clipfrac']
self.train = train
self.train_model = train_model
self.time_major = self.train_model.time_major
self.act_model = act_model
self.step = act_model.step
self.greedy_predict = act_model.greedy_predict
self.save = functools.partial(save_variables, sess=sess)
self.load = functools.partial(load_variables, sess=sess)
self.assign_old_eq_new = assign_old_eq_new
if MPI.COMM_WORLD.Get_rank() == 0:
initialize()
global_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="")
sync_from_root(sess, global_variables)