def setup_model(self):
# prevent import loops
from stable_baselines.gail.adversary import TransitionClassifier
with SetVerbosity(self.verbose):
assert issubclass(self.policy, ActorCriticPolicy), "Error: the input policy for the TRPO model must be " \
"an instance of common.policies.ActorCriticPolicy."
self.nworkers = MPI.COMM_WORLD.Get_size()
self.rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
self.graph = tf.Graph()
with self.graph.as_default():
self.sess = tf_util.single_threaded_session(graph=self.graph)
if self.using_gail:
self.reward_giver = TransitionClassifier(
self.observation_space,
self.action_space,
self.hidden_size_adversary,
entcoeff=self.adversary_entcoeff)
# Construct network for new policy
self.policy_pi = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
1,
None,
reuse=False,
**self.policy_kwargs)
# Network for old policy
with tf.variable_scope("oldpi", reuse=False):
old_policy = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
1,
None,
reuse=False,
**self.policy_kwargs)
with tf.variable_scope("loss", reuse=False):
atarg = tf.placeholder(dtype=tf.float32, shape=[
None
]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32,
shape=[None]) # Empirical return
observation = self.policy_pi.obs_ph
action = self.policy_pi.pdtype.sample_placeholder([None])
kloldnew = old_policy.proba_distribution.kl(
self.policy_pi.proba_distribution)
ent = self.policy_pi.proba_distribution.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = self.entcoeff * meanent
vferr = tf.reduce_mean(
tf.square(self.policy_pi.value_fn[:, 0] - ret))
# advantage * pnew / pold
ratio = tf.exp(
self.policy_pi.proba_distribution.logp(action) -
old_policy.proba_distribution.logp(action))
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
self.loss_names = [
"optimgain", "meankl", "entloss", "surrgain", "entropy"
]
dist = meankl
all_var_list = tf_util.get_trainable_vars("model")
var_list = [
v for v in all_var_list
if "/vf" not in v.name and "/q/" not in v.name
]
vf_var_list = [
v for v in all_var_list
if "/pi" not in v.name and "/logstd" not in v.name
]
self.get_flat = tf_util.GetFlat(var_list, sess=self.sess)
self.set_from_flat = tf_util.SetFromFlat(var_list,
sess=self.sess)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
var_size = tf_util.intprod(shape)
tangents.append(
tf.reshape(flat_tangent[start:start + var_size],
shape))
start += var_size
gvp = tf.add_n([
tf.reduce_sum(grad * tangent)
for (grad, tangent) in zipsame(klgrads, tangents)
]) # pylint: disable=E1111
fvp = tf_util.flatgrad(gvp, var_list)
tf.summary.scalar('entropy_loss', meanent)
tf.summary.scalar('policy_gradient_loss', optimgain)
tf.summary.scalar('value_function_loss', surrgain)
tf.summary.scalar('approximate_kullback-leiber', meankl)
tf.summary.scalar(
'loss',
optimgain + meankl + entbonus + surrgain + meanent)
self.assign_old_eq_new = \
tf_util.function([], [], updates=[tf.assign(oldv, newv) for (oldv, newv) in
zipsame(tf_util.get_globals_vars("oldpi"),
tf_util.get_globals_vars("model"))])
self.compute_losses = tf_util.function(
[observation, old_policy.obs_ph, action, atarg],
losses)
self.compute_fvp = tf_util.function([
flat_tangent, observation, old_policy.obs_ph, action,
atarg
], fvp)
self.compute_vflossandgrad = tf_util.function(
[observation, old_policy.obs_ph, ret],
tf_util.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if self.rank == 0 and self.verbose >= 1:
print(colorize(msg, color='magenta'))
start_time = time.time()
yield
print(
colorize("done in {:.3f} seconds".format(
(time.time() - start_time)),
color='magenta'))
else:
yield
def allmean(arr):
assert isinstance(arr, np.ndarray)
out = np.empty_like(arr)
MPI.COMM_WORLD.Allreduce(arr, out, op=MPI.SUM)
out /= self.nworkers
return out
tf_util.initialize(sess=self.sess)
th_init = self.get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
self.set_from_flat(th_init)
with tf.variable_scope("Adam_mpi", reuse=False):
self.vfadam = MpiAdam(vf_var_list, sess=self.sess)
if self.using_gail:
self.d_adam = MpiAdam(
self.reward_giver.get_trainable_variables(),
sess=self.sess)
self.d_adam.sync()
self.vfadam.sync()
with tf.variable_scope("input_info", reuse=False):
tf.summary.scalar('discounted_rewards',
tf.reduce_mean(ret))
tf.summary.scalar('learning_rate',
tf.reduce_mean(self.vf_stepsize))
tf.summary.scalar('advantage', tf.reduce_mean(atarg))
tf.summary.scalar('kl_clip_range',
tf.reduce_mean(self.max_kl))
if self.full_tensorboard_log:
tf.summary.histogram('discounted_rewards', ret)
tf.summary.histogram('learning_rate', self.vf_stepsize)
tf.summary.histogram('advantage', atarg)
tf.summary.histogram('kl_clip_range', self.max_kl)
if tf_util.is_image(self.observation_space):
tf.summary.image('observation', observation)
else:
tf.summary.histogram('observation', observation)
self.timed = timed
self.allmean = allmean
self.step = self.policy_pi.step
self.proba_step = self.policy_pi.proba_step
self.initial_state = self.policy_pi.initial_state
self.params = find_trainable_variables("model")
if self.using_gail:
self.params.extend(
self.reward_giver.get_trainable_variables())
self.summary = tf.summary.merge_all()
self.compute_lossandgrad = \
tf_util.function([observation, old_policy.obs_ph, action, atarg, ret],
[self.summary, tf_util.flatgrad(optimgain, var_list)] + losses)
def setup_model(self):
# prevent import loops
from stable_baselines.gail.adversary import TransitionClassifier
with SetVerbosity(self.verbose):
assert issubclass(self.policy, ActorCriticPolicy), "Error: the input policy for the TRPO model must be " \
"an instance of common.policies.ActorCriticPolicy."
self.nworkers = MPI.COMM_WORLD.Get_size()
self.rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
self.graph = tf.Graph()
with self.graph.as_default():
self.set_random_seed(self.seed)
self.sess = tf_util.make_session(num_cpu=self.n_cpu_tf_sess, graph=self.graph)
if self.using_gail:
self.reward_giver = TransitionClassifier(self.observation_space, self.action_space,
self.hidden_size_adversary,
entcoeff=self.adversary_entcoeff)
# Penalty related variable
with tf.variable_scope('penalty'):
cur_cost_ph = tf.placeholder(dtype=tf.float32, shape=[None]) # episodic cost
param_init = np.log(max(np.exp(self.penalty_init) - 1, 1e-8))
penalty_param = tf.get_variable('penalty_param',
initializer=float(param_init),
trainable=True,
dtype=tf.float32)
penalty = tf.nn.softplus(penalty_param)
penalty_loss = tf.reduce_mean(-penalty_param * (cur_cost_ph - self.cost_lim))
# Construct network for new policy
self.policy_pi = self.policy(self.sess, self.observation_space, self.action_space, self.n_envs, 1,
None, reuse=False, **self.policy_kwargs)
# Network for old policy
with tf.variable_scope("oldpi", reuse=False):
old_policy = self.policy(self.sess, self.observation_space, self.action_space, self.n_envs, 1,
None, reuse=False, **self.policy_kwargs)
# # Network for safety value function
# with tf.variable_Scope("vc",reuse=False):
# self.cost_value = MLPValue(self.sess, self.observation_spacem, self.n_envs, 1, None)
with tf.variable_scope("loss", reuse=False):
atarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
catarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target cost advantage function
cret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical cost
observation = self.policy_pi.obs_ph
action = self.policy_pi.pdtype.sample_placeholder([None])
kloldnew = old_policy.proba_distribution.kl(self.policy_pi.proba_distribution)
ent = self.policy_pi.proba_distribution.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = self.entcoeff * meanent
vferr = tf.reduce_mean(tf.square(self.policy_pi.value_flat - ret))
vcerr = tf.reduce_mean(tf.square(self.policy_pi.vcf_flat - cret))
# advantage * pnew / pold
ratio = tf.exp(self.policy_pi.proba_distribution.logp(action) -
old_policy.proba_distribution.logp(action))
surrgain = tf.reduce_mean(ratio * atarg)
# Surrogate for cost function
surrcost = tf.reduce_mean(ratio * catarg)
optimgain = surrgain + entbonus
# Include surr_cost in pi_objective
optimgain -= penalty * surrcost
optimgain /= (1 + penalty)
# # Loss function for pi is negative of pi_objective
# optimgain = -optimgain # Should we??
losses = [optimgain, meankl, entbonus, surrgain, meanent, surrcost]
self.loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy", "surrcost"]
dist = meankl
all_var_list = tf_util.get_trainable_vars("model")
var_list = [v for v in all_var_list if "/vf" not in v.name and "/q/" not in v.name and "/vcf" not in v.name] # policy parameters
vf_var_list = [v for v in all_var_list if "/pi" not in v.name and "/logstd" not in v.name and "/vcf" not in v.name] # value parameters
vcf_var_list = [v for v in all_var_list if "/pi" not in v.name and "/logstd" not in v.name and "/vf" not in v.name] # cost value parameters
self.get_flat = tf_util.GetFlat(var_list, sess=self.sess)
self.set_from_flat = tf_util.SetFromFlat(var_list, sess=self.sess)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
var_size = tf_util.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start: start + var_size], shape))
start += var_size
gvp = tf.add_n([tf.reduce_sum(grad * tangent)
for (grad, tangent) in zipsame(klgrads, tangents)]) # pylint: disable=E1111
# Fisher vector products
fvp = tf_util.flatgrad(gvp, var_list)
tf.summary.scalar('penalty_loss', penalty_loss)
tf.summary.scalar('entropy_loss', meanent)
tf.summary.scalar('policy_gradient_loss', optimgain)
tf.summary.scalar('value_function_loss', surrgain)
tf.summary.scalar('constraint_cost_function_loss', surrcost)
tf.summary.scalar('approximate_kullback-leibler', meankl)
tf.summary.scalar('loss', optimgain + meankl + entbonus + surrgain + meanent + surrcost + penalty_loss)
self.assign_old_eq_new = \
tf_util.function([], [], updates=[tf.assign(oldv, newv) for (oldv, newv) in
zipsame(tf_util.get_globals_vars("oldpi"),
tf_util.get_globals_vars("model"))])
self.compute_losses = tf_util.function([observation, old_policy.obs_ph, action, atarg, catarg], losses)
self.compute_fvp = tf_util.function([flat_tangent, observation, old_policy.obs_ph, action, atarg, catarg],
fvp) # Why need all inputs? Might for implementation easiness
# self.compute_vflossandgrad = tf_util.function([observation, old_policy.obs_ph, ret],
# tf_util.flatgrad(vferr, vf_var_list)) # Why need old_policy.obs_ph? Doesn't seem to be used
# self.compute_vcflossandgrad = tf_util.function([observation, old_policy.obs_ph, cret],
# tf_util.flatgrad(vcerr, vcf_var_list))
self.compute_vflossandgrad = tf_util.function([observation, old_policy.obs_ph, ret, cret],
[tf_util.flatgrad(vferr, vf_var_list), tf_util.flatgrad(vcerr, vcf_var_list)])
self.compute_lagrangiangrad = tf_util.function([cur_cost_ph],
tf_util.flatgrad(penalty_loss, [penalty_param]))
@contextmanager
def timed(msg):
if self.rank == 0 and self.verbose >= 1:
print(colorize(msg, color='magenta'))
start_time = time.time()
yield
print(colorize("done in {:.3f} seconds".format((time.time() - start_time)),
color='magenta'))
else:
yield
def allmean(arr):
assert isinstance(arr, np.ndarray)
out = np.empty_like(arr)
MPI.COMM_WORLD.Allreduce(arr, out, op=MPI.SUM)
out /= self.nworkers
return out
tf_util.initialize(sess=self.sess)
th_init = self.get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
self.set_from_flat(th_init)
with tf.variable_scope("Adam_mpi", reuse=False):
self.vfadam = MpiAdam(vf_var_list, sess=self.sess)
if self.using_gail:
self.d_adam = MpiAdam(self.reward_giver.get_trainable_variables(), sess=self.sess)
self.d_adam.sync()
self.vfadam.sync()
# optimizer for constraint costs value function
self.vcadam = MpiAdam(vcf_var_list, sess=self.sess)
self.vcadam.sync()
# optimizer for lagragian value of safe RL
self.penaltyadam = MpiAdam([penalty_param], sess=self.sess)
self.penaltyadam.sync()
with tf.variable_scope("input_info", reuse=False):
tf.summary.scalar('discounted_rewards', tf.reduce_mean(ret))
tf.summary.scalar('discounted_costs', tf.reduce_mean(cret))
tf.summary.scalar('learning_rate', tf.reduce_mean(self.vf_stepsize))
tf.summary.scalar('advantage', tf.reduce_mean(atarg))
tf.summary.scalar('cost_advantage', tf.reduce_mean(catarg))
tf.summary.scalar('kl_clip_range', tf.reduce_mean(self.max_kl))
if self.full_tensorboard_log:
tf.summary.histogram('discounted_rewards', ret)
tf.summary.histogram('discounted_rewards', cret)
tf.summary.histogram('learning_rate', self.vf_stepsize)
tf.summary.histogram('penalty_learning_rate', self.penalty_lr)
tf.summary.histogram('advantage', atarg)
tf.summary.histogram('cost_advantage', catarg)
tf.summary.histogram('kl_clip_range', self.max_kl)
if tf_util.is_image(self.observation_space):
tf.summary.image('observation', observation)
else:
tf.summary.histogram('observation', observation)
self.timed = timed
self.allmean = allmean
self.step = self.policy_pi.step
self.proba_step = self.policy_pi.proba_step
self.initial_state = self.policy_pi.initial_state
self.params = tf_util.get_trainable_vars("model") + tf_util.get_trainable_vars("oldpi")
if self.using_gail:
self.params.extend(self.reward_giver.get_trainable_variables())
self.summary = tf.summary.merge_all()
self.compute_lossandgrad = \
tf_util.function([observation, old_policy.obs_ph, action, atarg, catarg, ret, cret, cur_cost_ph],
[self.summary, tf_util.flatgrad(optimgain, var_list)] + losses)
def setup_model(self):
# prevent import loops
with SetVerbosity(self.verbose):
assert issubclass(self.policy, ActorCriticPolicy), "Error: the input policy for the TRPO model must be " \
"an instance of common.policies.ActorCriticPolicy."
self.nworkers = MPI.COMM_WORLD.Get_size()
print("number of workers are", self.nworkers)
self.rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
self.graph = tf.Graph()
with self.graph.as_default():
self.sess = tf_util.single_threaded_session(graph=self.graph)
self._setup_learn(self.seed)
# Construct network for new policy
self.policy_pi = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
1,
None,
reuse=False,
**self.policy_kwargs)
# Network for old policy
with tf.variable_scope("oldpi", reuse=False):
old_policy = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
1,
None,
reuse=False,
**self.policy_kwargs)
# Network for phi
with tf.variable_scope("phi", reuse=False):
self.policy_phi = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
1,
None,
reuse=False,
**self.policy_kwargs)
# Network for phi old
with tf.variable_scope("oldphi", reuse=False):
self.policy_phi_old = self.policy(self.sess,
self.observation_space,
self.action_space,
self.n_envs,
1,
None,
reuse=False,
**self.policy_kwargs)
with tf.variable_scope("loss", reuse=False):
atarg = tf.placeholder(dtype=tf.float32, shape=[
None
]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32,
shape=[None]) # Empirical return
observation = self.policy_pi.obs_ph
action = self.policy_pi.pdtype.sample_placeholder([None])
kloldnew = old_policy.proba_distribution.kl(
self.policy_pi.proba_distribution)
#kloldnew = self.policy_pi.proba_distribution.kl(old_policy.proba_distribution)
ent = self.policy_pi.proba_distribution.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = self.entcoeff * meanent
vferr = tf.reduce_mean(
tf.square(self.policy_pi.value_flat - ret))
vf_phi_err = tf.reduce_mean(
tf.square(self.policy_phi.value_flat - ret))
vf_phi_old_err = tf.reduce_mean(
tf.square(self.policy_phi_old.value_flat))
# advantage * pnew / pold
ratio = tf.exp(
self.policy_pi.proba_distribution.logp(action) -
old_policy.proba_distribution.logp(action))
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
self.loss_names = [
"optimgain", "meankl", "entloss", "surrgain", "entropy"
]
dist = meankl
all_var_list = tf_util.get_trainable_vars("model")
var_list = [
v for v in all_var_list
if "/vf" not in v.name and "/q/" not in v.name
]
vf_var_list = [
v for v in all_var_list
if "/pi" not in v.name and "/logstd" not in v.name
]
all_var_oldpi_list = tf_util.get_trainable_vars("oldpi")
var_oldpi_list = [
v for v in all_var_oldpi_list
if "/vf" not in v.name and "/q/" not in v.name
]
all_var_phi_list = tf_util.get_trainable_vars("phi")
vf_phi_var_list = [
v for v in all_var_phi_list if "/pi" not in v.name
and "/logstd" not in v.name and "/q" not in v.name
]
all_var_phi_old_list = tf_util.get_trainable_vars("oldphi")
vf_phi_old_var_list = [
v for v in all_var_phi_old_list if "/pi" not in v.name
and "/logstd" not in v.name and "/q" not in v.name
]
#print("vars", vf_var_list)
self.policy_vars = all_var_list
self.oldpolicy_vars = all_var_oldpi_list
print("all var list", all_var_list)
print("phi vars", vf_phi_var_list)
print("phi old vars", vf_phi_old_var_list)
self.get_flat = tf_util.GetFlat(var_list, sess=self.sess)
self.set_from_flat = tf_util.SetFromFlat(var_list,
sess=self.sess)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
var_size = tf_util.intprod(shape)
tangents.append(
tf.reshape(flat_tangent[start:start + var_size],
shape))
start += var_size
gvp = tf.add_n([
tf.reduce_sum(grad * tangent)
for (grad, tangent) in zipsame(klgrads, tangents)
]) # pylint: disable=E1111
fvp = tf_util.flatgrad(gvp, var_list)
tf.summary.scalar('entropy_loss', meanent)
tf.summary.scalar('policy_gradient_loss', optimgain)
tf.summary.scalar('value_function_loss', surrgain)
tf.summary.scalar('approximate_kullback-leibler', meankl)
tf.summary.scalar(
'loss',
optimgain + meankl + entbonus + surrgain + meanent)
self.assign_old_eq_new = \
tf_util.function([], [], updates=[tf.assign(oldv, newv) for (oldv, newv) in
zipsame(tf_util.get_globals_vars("oldpi"),
tf_util.get_globals_vars("model"))])
self.compute_losses = tf_util.function(
[observation, old_policy.obs_ph, action, atarg],
losses)
self.compute_fvp = tf_util.function([
flat_tangent, observation, old_policy.obs_ph, action,
atarg
], fvp)
self.compute_vflossandgrad = tf_util.function(
[observation, old_policy.obs_ph, ret],
tf_util.flatgrad(vferr, vf_var_list))
self.compute_vf_phi_lossandgrad = tf_util.function(
[observation, self.policy_phi.obs_ph, ret],
tf_util.flatgrad(vf_phi_err, vf_phi_var_list))
self.compute_vf_loss = tf_util.function(
[observation, old_policy.obs_ph, ret], vferr)
self.compute_vf_phi_loss = tf_util.function(
[observation, self.policy_phi.obs_ph, ret], vf_phi_err)
#self.compute_vf_phi_old_loss = tf_util.function([self.policy_phi_old.obs_ph], vf_phi_old_err)
#self.phi_old_obs = np.array([-0.012815 , -0.02076313, 0.07524705, 0.09407324, 0.0901745 , -0.09339058, 0.03544853, -0.03297224])
#self.phi_old_obs = self.phi_old_obs.reshape((1, 8))
update_phi_old_expr = []
for var, var_target in zip(
sorted(vf_phi_var_list, key=lambda v: v.name),
sorted(vf_phi_old_var_list, key=lambda v: v.name)):
update_phi_old_expr.append(var_target.assign(var))
update_phi_old_expr = tf.group(*update_phi_old_expr)
self.update_phi_old = tf_util.function(
[], [], updates=[update_phi_old_expr])
@contextmanager
def timed(msg):
if self.rank == 0 and self.verbose >= 1:
print(colorize(msg, color='magenta'))
start_time = time.time()
yield
print(
colorize("done in {:.3f} seconds".format(
(time.time() - start_time)),
color='magenta'))
else:
yield
@contextmanager
def temp_seed(seed):
state = np.random.get_state()
np.random.seed(seed)
try:
yield
finally:
np.random.set_state(state)
def allmean(arr):
assert isinstance(arr, np.ndarray)
out = np.empty_like(arr)
MPI.COMM_WORLD.Allreduce(arr, out, op=MPI.SUM)
out /= self.nworkers
return out
tf_util.initialize(sess=self.sess)
th_init = self.get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
self.set_from_flat(th_init)
with tf.variable_scope("Adam_mpi", reuse=False):
self.vfadam = MpiAdam(vf_var_list, sess=self.sess)
self.vf_phi_adam = MpiAdam(vf_phi_var_list, sess=self.sess)
self.vfadam.sync()
self.vf_phi_adam.sync()
with tf.variable_scope("input_info", reuse=False):
tf.summary.scalar('discounted_rewards',
tf.reduce_mean(ret))
tf.summary.scalar('learning_rate',
tf.reduce_mean(self.vf_stepsize))
tf.summary.scalar('advantage', tf.reduce_mean(atarg))
tf.summary.scalar('kl_clip_range',
tf.reduce_mean(self.max_kl))
self.timed = timed
self.allmean = allmean
self.temp_seed = temp_seed
self.step = self.policy_pi.step
self.proba_step = self.policy_pi.proba_step
self.initial_state = self.policy_pi.initial_state
self.params = tf_util.get_trainable_vars(
"model") + tf_util.get_trainable_vars("oldpi")
self.summary = tf.summary.merge_all()
self.compute_lossandgrad = \
tf_util.function([observation, old_policy.obs_ph, action, atarg, ret],
[self.summary, tf_util.flatgrad(optimgain, var_list)] + losses)
def setup_model(self):
# prevent import loops
from stable_baselines.gail.adversary import TransitionClassifier
from stable_baselines.mdal.adversary import TabularAdversaryTF, NeuralAdversaryTRPO
with SetVerbosity(self.verbose):
assert issubclass(self.policy, ActorCriticPolicy), "Error: the input policy for the MDPO model must be " \
"an instance of common.policies.ActorCriticPolicy."
self.nworkers = MPI.COMM_WORLD.Get_size()
self.rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
self.graph = tf.Graph()
with self.graph.as_default():
self.sess = tf_util.single_threaded_session(graph=self.graph)
# self._setup_learn(self.seed)
self._setup_learn()
if self.using_gail:
self.reward_giver = TransitionClassifier(self.observation_space, self.action_space,
self.hidden_size_adversary,
entcoeff=self.adversary_entcoeff)
elif self.using_mdal:
if self.neural:
self.reward_giver = NeuralAdversaryTRPO(self.sess, self.observation_space, self.action_space,
self.hidden_size_adversary,
entcoeff=self.adversary_entcoeff)
else:
self.reward_giver = TabularAdversaryTF(self.sess, self.observation_space, self.action_space,
self.hidden_size_adversary,
entcoeff=self.adversary_entcoeff,
expert_features=self.expert_dataset.successor_features,
exploration_bonus=self.exploration_bonus,
bonus_coef=self.bonus_coef,
t_c=self.t_c,
is_action_features=self.is_action_features)
# Construct network for new policy
self.policy_pi = self.policy(self.sess, self.observation_space, self.action_space, self.n_envs, 1,
None, reuse=False, **self.policy_kwargs)
# Network for old policy
with tf.variable_scope("oldpi", reuse=False):
self.old_policy = self.policy(self.sess, self.observation_space, self.action_space, self.n_envs, 1,
None, reuse=False, **self.policy_kwargs)
# Network for fitting closed form
with tf.variable_scope("closedpi", reuse=False):
self.closed_policy = self.policy(self.sess, self.observation_space, self.action_space, self.n_envs, 1,
None, reuse=False, **self.policy_kwargs)
with tf.variable_scope("loss", reuse=False):
self.atarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)
self.vtarg = tf.placeholder(dtype=tf.float32, shape=[None])
self.ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
self.learning_rate_ph = tf.placeholder(dtype=tf.float32, shape=[], name="learning_rate_ph")
self.outer_learning_rate_ph = tf.placeholder(dtype=tf.float32, shape=[], name="outer_learning_rate_ph")
self.old_vpred_ph = tf.placeholder(dtype=tf.float32, shape=[None], name="old_vpred_ph")
self.clip_range_vf_ph = tf.placeholder(dtype=tf.float32, shape=[], name="clip_range_ph")
observation = self.policy_pi.obs_ph
self.action = self.policy_pi.pdtype.sample_placeholder([None])
if self.tsallis_q == 1.0:
kloldnew = self.policy_pi.proba_distribution.kl(self.old_policy.proba_distribution)
ent = self.policy_pi.proba_distribution.entropy()
meankl = tf.reduce_mean(kloldnew)
else:
logp_pi = self.policy_pi.proba_distribution.logp(self.action)
logp_pi_old = self.old_policy.proba_distribution.logp(self.action)
ent = self.policy_pi.proba_distribution.entropy()
#kloldnew = self.policy_pi.proba_distribution.kl_tsallis(self.old_policy.proba_distribution, self.tsallis_q)
tsallis_q = 2.0 - self.tsallis_q
meankl = tf.reduce_mean(tf_log_q(tf.exp(logp_pi), tsallis_q) - tf_log_q(tf.exp(logp_pi_old), tsallis_q)) #tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = self.entcoeff * meanent
if self.cliprange_vf is None:
vpred_clipped = self.policy_pi.value_flat
else:
vpred_clipped = self.old_vpred_ph + \
tf.clip_by_value(self.policy_pi.value_flat - self.old_vpred_ph,
- self.clip_range_vf_ph, self.clip_range_vf_ph)
vf_losses1 = tf.square(self.policy_pi.value_flat - self.ret)
vf_losses2 = tf.square(vpred_clipped - self.ret)
vferr = tf.reduce_mean(tf.maximum(vf_losses1, vf_losses2))
# advantage * pnew / pold
ratio = tf.exp(self.policy_pi.proba_distribution.logp(self.action) -
self.old_policy.proba_distribution.logp(self.action))
if self.method == "multistep-SGD":
surrgain = tf.reduce_mean(ratio * self.atarg) - meankl / self.learning_rate_ph
elif self.method == "closedreverse-KL":
surrgain = tf.reduce_mean(tf.exp(self.atarg) * self.policy_pi.proba_distribution.logp(self.action))
else:
policygain = tf.reduce_mean(tf.exp(self.atarg) * tf.log(self.closed_policy.proba_distribution.mean))
surrgain = tf.reduce_mean(ratio * self.atarg) - tf.reduce_mean(self.learning_rate_ph * ratio * self.policy_pi.proba_distribution.logp(self.action))
optimgain = surrgain #+ entbonus - self.learning_rate_ph * meankl
losses = [optimgain, meankl, entbonus, surrgain, meanent]
self.loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = tf_util.get_trainable_vars("model")
var_list = [v for v in all_var_list if "/vf" not in v.name and "/q/" not in v.name]
vf_var_list = [v for v in all_var_list if "/pi" not in v.name and "/logstd" not in v.name]
print("policy vars", var_list)
all_closed_var_list = tf_util.get_trainable_vars("closedpi")
closed_var_list = [v for v in all_closed_var_list if "/vf" not in v.name and "/q" not in v.name]
self.get_flat = tf_util.GetFlat(var_list, sess=self.sess)
self.set_from_flat = tf_util.SetFromFlat(var_list, sess=self.sess)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
var_size = tf_util.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start: start + var_size], shape))
start += var_size
gvp = tf.add_n([tf.reduce_sum(grad * tangent)
for (grad, tangent) in zipsame(klgrads, tangents)]) # pylint: disable=E1111
fvp = tf_util.flatgrad(gvp, var_list)
# tf.summary.scalar('entropy_loss', meanent)
# tf.summary.scalar('policy_gradient_loss', optimgain)
# tf.summary.scalar('value_function_loss', surrgain)
# tf.summary.scalar('approximate_kullback-leibler', meankl)
# tf.summary.scalar('loss', optimgain + meankl + entbonus + surrgain + meanent)
self.assign_old_eq_new = \
tf_util.function([], [], updates=[tf.assign(oldv, newv) for (oldv, newv) in
zipsame(tf_util.get_globals_vars("oldpi"),
tf_util.get_globals_vars("model"))])
self.compute_losses = tf_util.function([observation, self.old_policy.obs_ph, self.action, self.atarg, self.learning_rate_ph, self.vtarg], losses)
self.compute_fvp = tf_util.function([flat_tangent, observation, self.old_policy.obs_ph, self.action, self.atarg],
fvp)
self.compute_vflossandgrad = tf_util.function([observation, self.old_policy.obs_ph, self.ret, self.old_vpred_ph, self.clip_range_vf_ph],
tf_util.flatgrad(vferr, vf_var_list))
grads = tf.gradients(-optimgain, var_list)
grads, _grad_norm = tf.clip_by_global_norm(grads, 0.5)
trainer = tf.train.AdamOptimizer(learning_rate=self.outer_learning_rate_ph, epsilon=1e-5)
# trainer = tf.train.AdamOptimizer(learning_rate=3e-4, epsilon=1e-5)
grads = list(zip(grads, var_list))
self._train = trainer.apply_gradients(grads)
@contextmanager
def timed(msg):
if self.rank == 0 and self.verbose >= 1:
# print(colorize(msg, color='magenta'))
# start_time = time.time()
yield
# print(colorize("done in {:.3f} seconds".format((time.time() - start_time)),
# color='magenta'))
else:
yield
def allmean(arr):
assert isinstance(arr, np.ndarray)
out = np.empty_like(arr)
MPI.COMM_WORLD.Allreduce(arr, out, op=MPI.SUM)
out /= self.nworkers
return out
tf_util.initialize(sess=self.sess)
th_init = self.get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
self.set_from_flat(th_init)
with tf.variable_scope("Adam_mpi", reuse=False):
self.vfadam = MpiAdam(vf_var_list, sess=self.sess)
if self.using_gail or self.using_mdal:
self.d_adam = MpiAdam(self.reward_giver.get_trainable_variables(), sess=self.sess)
self.d_adam.sync()
self.vfadam.sync()
with tf.variable_scope("input_info", reuse=False):
tf.summary.scalar('discounted_rewards', tf.reduce_mean(self.ret))
tf.summary.scalar('learning_rate', tf.reduce_mean(self.vf_stepsize))
tf.summary.scalar('advantage', tf.reduce_mean(self.atarg))
tf.summary.scalar('kl_clip_range', tf.reduce_mean(self.max_kl))
if self.full_tensorboard_log:
tf.summary.histogram('discounted_rewards', self.ret)
tf.summary.histogram('learning_rate', self.vf_stepsize)
tf.summary.histogram('advantage', self.atarg)
tf.summary.histogram('kl_clip_range', self.max_kl)
if tf_util.is_image(self.observation_space):
tf.summary.image('observation', observation)
else:
tf.summary.histogram('observation', observation)
self.timed = timed
self.allmean = allmean
self.step = self.policy_pi.step
self.proba_step = self.policy_pi.proba_step
self.initial_state = self.policy_pi.initial_state
self.params = tf_util.get_trainable_vars("model") + tf_util.get_trainable_vars("oldpi")
if self.using_gail:
self.params.extend(self.reward_giver.get_trainable_variables())
self.summary = tf.summary.merge_all()
self.compute_lossandgrad = \
tf_util.function([observation, self.old_policy.obs_ph, self.action, self.atarg, self.ret, self.learning_rate_ph, self.vtarg, self.closed_policy.obs_ph],
[self.summary, tf_util.flatgrad(optimgain, var_list)] + losses)
