def learn(
*,
network,
env,
eval_env,
make_eval_env,
env_id,
total_timesteps,
timesteps_per_batch,
sil_update,
sil_loss, # what to train on
max_kl=0.001,
cg_iters=10,
gamma=0.99,
lam=1.0, # advantage estimation
seed=None,
ent_coef=0.0,
lr=3e-4,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters=5,
sil_value=0.01,
sil_alpha=0.6,
sil_beta=0.1,
max_episodes=0,
max_iters=0, # time constraint
callback=None,
save_interval=0,
load_path=None,
# MBL
# For train mbl
mbl_train_freq=5,
# For eval
num_eval_episodes=5,
eval_freq=5,
vis_eval=False,
#eval_targs=('mbmf',),
eval_targs=('mf', ),
quant=2,
# For mbl.step
#num_samples=(1500,),
num_samples=(1, ),
horizon=(2, ),
#horizon=(2,1),
#num_elites=(10,),
num_elites=(1, ),
mbl_lamb=(1.0, ),
mbl_gamma=0.99,
#mbl_sh=1, # Number of step for stochastic sampling
mbl_sh=10000,
#vf_lookahead=-1,
#use_max_vf=False,
reset_per_step=(0, ),
# For get_model
num_fc=2,
num_fwd_hidden=500,
use_layer_norm=False,
# For MBL
num_warm_start=int(1e4),
init_epochs=10,
update_epochs=5,
batch_size=512,
update_with_validation=False,
use_mean_elites=1,
use_ent_adjust=0,
adj_std_scale=0.5,
# For data loading
validation_set_path=None,
# For data collect
collect_val_data=False,
# For traj collect
traj_collect='mf',
# For profile
measure_time=True,
eval_val_err=False,
measure_rew=True,
model_fn=None,
update_fn=None,
init_fn=None,
mpi_rank_weight=1,
comm=None,
vf_coef=0.5,
max_grad_norm=0.5,
log_interval=1,
nminibatches=4,
noptepochs=4,
cliprange=0.2,
**network_kwargs):
'''
learn a policy function with TRPO algorithm
Parameters:
----------
network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
or function that takes input placeholder and returns tuple (output, None) for feedforward nets
or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets
env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class
timesteps_per_batch timesteps per gradient estimation batch
max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )
ent_coef coefficient of policy entropy term in the optimization objective
cg_iters number of iterations of conjugate gradient algorithm
cg_damping conjugate gradient damping
vf_stepsize learning rate for adam optimizer used to optimie value function loss
vf_iters number of iterations of value function optimization iterations per each policy optimization step
total_timesteps max number of timesteps
max_episodes max number of episodes
max_iters maximum number of policy optimization iterations
callback function to be called with (locals(), globals()) each policy optimization step
load_path str, path to load the model from (default: None, i.e. no model is loaded)
**network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
Returns:
-------
learnt model
'''
if not isinstance(num_samples, tuple): num_samples = (num_samples, )
if not isinstance(horizon, tuple): horizon = (horizon, )
if not isinstance(num_elites, tuple): num_elites = (num_elites, )
if not isinstance(mbl_lamb, tuple): mbl_lamb = (mbl_lamb, )
if not isinstance(reset_per_step, tuple):
reset_per_step = (reset_per_step, )
if validation_set_path is None:
if collect_val_data:
validation_set_path = os.path.join(logger.get_dir(), 'val.pkl')
else:
validation_set_path = os.path.join('dataset',
'{}-val.pkl'.format(env_id))
if eval_val_err:
eval_val_err_path = os.path.join('dataset',
'{}-combine-val.pkl'.format(env_id))
logger.log(locals())
logger.log('MBL_SH', mbl_sh)
logger.log('Traj_collect', traj_collect)
if MPI is not None:
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
else:
nworkers = 1
rank = 0
cpus_per_worker = 1
U.get_session(
config=tf.ConfigProto(allow_soft_placement=True,
inter_op_parallelism_threads=cpus_per_worker,
intra_op_parallelism_threads=cpus_per_worker))
set_global_seeds(seed)
if isinstance(lr, float): lr = constfn(lr)
else: assert callable(lr)
if isinstance(cliprange, float): cliprange = constfn(cliprange)
else: assert callable(cliprange)
policy = build_policy(env, network, value_network='copy', **network_kwargs)
nenvs = env.num_envs
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
nbatch = nenvs * timesteps_per_batch
nbatch_train = nbatch // nminibatches
is_mpi_root = (MPI is None or MPI.COMM_WORLD.Get_rank() == 0)
ob = observation_placeholder(ob_space)
with tf.variable_scope("pi"):
pi = policy(observ_placeholder=ob)
make_model = lambda: Model(
policy=policy,
ob_space=ob_space,
ac_space=ac_space,
nbatch_act=nenvs,
nbatch_train=nbatch_train,
nsteps=timesteps_per_batch,
ent_coef=ent_coef,
vf_coef=vf_coef,
max_grad_norm=max_grad_norm,
sil_update=sil_update,
sil_value=sil_value,
sil_alpha=sil_alpha,
sil_beta=sil_beta,
sil_loss=sil_loss,
# fn_reward=env.process_reward,
fn_reward=None,
# fn_obs=env.process_obs,
fn_obs=None,
ppo=False,
prev_pi='pi',
silm=pi)
model = make_model()
with tf.variable_scope("oldpi"):
oldpi = policy(observ_placeholder=ob)
make_old_model = lambda: Model(
policy=policy,
ob_space=ob_space,
ac_space=ac_space,
nbatch_act=nenvs,
nbatch_train=nbatch_train,
nsteps=timesteps_per_batch,
ent_coef=ent_coef,
vf_coef=vf_coef,
max_grad_norm=max_grad_norm,
sil_update=sil_update,
sil_value=sil_value,
sil_alpha=sil_alpha,
sil_beta=sil_beta,
sil_loss=sil_loss,
# fn_reward=env.process_reward,
fn_reward=None,
# fn_obs=env.process_obs,
fn_obs=None,
ppo=False,
prev_pi='oldpi',
silm=oldpi)
old_model = make_old_model()
# MBL
# ---------------------------------------
#viz = Visdom(env=env_id)
win = None
eval_targs = list(eval_targs)
logger.log(eval_targs)
make_model_f = get_make_mlp_model(num_fc=num_fc,
num_fwd_hidden=num_fwd_hidden,
layer_norm=use_layer_norm)
mbl = MBL(env=eval_env,
env_id=env_id,
make_model=make_model_f,
num_warm_start=num_warm_start,
init_epochs=init_epochs,
update_epochs=update_epochs,
batch_size=batch_size,
**network_kwargs)
val_dataset = {'ob': None, 'ac': None, 'ob_next': None}
if update_with_validation:
logger.log('Update with validation')
val_dataset = load_val_data(validation_set_path)
if eval_val_err:
logger.log('Log val error')
eval_val_dataset = load_val_data(eval_val_err_path)
if collect_val_data:
logger.log('Collect validation data')
val_dataset_collect = []
def _mf_pi(ob, t=None):
stochastic = True
ac, vpred, _, _ = pi.step(ob, stochastic=stochastic)
return ac, vpred
def _mf_det_pi(ob, t=None):
#ac, vpred, _, _ = pi.step(ob, stochastic=False)
ac, vpred = pi._evaluate([pi.pd.mode(), pi.vf], ob)
return ac, vpred
def _mf_ent_pi(ob, t=None):
mean, std, vpred = pi._evaluate([pi.pd.mode(), pi.pd.std, pi.vf], ob)
ac = np.random.normal(mean, std * adj_std_scale, size=mean.shape)
return ac, vpred
################### use_ent_adjust======> adj_std_scale????????pi action sample
def _mbmf_inner_pi(ob, t=0):
if use_ent_adjust:
return _mf_ent_pi(ob)
else:
#return _mf_pi(ob)
if t < mbl_sh: return _mf_pi(ob)
else: return _mf_det_pi(ob)
# ---------------------------------------
# Run multiple configuration once
all_eval_descs = []
def make_mbmf_pi(n, h, e, l):
def _mbmf_pi(ob):
ac, rew = mbl.step(ob=ob,
pi=_mbmf_inner_pi,
horizon=h,
num_samples=n,
num_elites=e,
gamma=mbl_gamma,
lamb=l,
use_mean_elites=use_mean_elites)
return ac[None], rew
return Policy(step=_mbmf_pi, reset=None)
for n in num_samples:
for h in horizon:
for l in mbl_lamb:
for e in num_elites:
if 'mbmf' in eval_targs:
all_eval_descs.append(('MeanRew', 'MBL_TRPO_SIL',
make_mbmf_pi(n, h, e, l)))
#if 'mbmf' in eval_targs: all_eval_descs.append(('MeanRew-n-{}-h-{}-e-{}-l-{}-sh-{}-me-{}'.format(n, h, e, l, mbl_sh, use_mean_elites), 'MBL_TRPO-n-{}-h-{}-e-{}-l-{}-sh-{}-me-{}'.format(n, h, e, l, mbl_sh, use_mean_elites), make_mbmf_pi(n, h, e, l)))
if 'mf' in eval_targs:
all_eval_descs.append(
('MeanRew', 'TRPO_SIL', Policy(step=_mf_pi, reset=None)))
logger.log('List of evaluation targets')
for it in all_eval_descs:
logger.log(it[0])
pool = Pool(mp.cpu_count())
warm_start_done = False
# ----------------------------------------
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
vferr = tf.reduce_mean(tf.square(pi.vf - ret))
ratio = tf.exp(pi.pd.logp(ac) -
oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = get_trainable_variables("pi")
# var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("pol")]
# vf_var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("vf")]
var_list = get_pi_trainable_variables("pi")
vf_var_list = get_vf_trainable_variables("pi")
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function(
[], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(get_variables("oldpi"), get_variables("pi"))
])
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses +
[U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret],
U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
U.initialize()
if load_path is not None:
pi.load(load_path)
th_init = get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
if traj_collect == 'mf':
seg_gen = traj_segment_generator(env,
timesteps_per_batch,
model,
stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
if sum([max_iters > 0, total_timesteps > 0, max_episodes > 0]) == 0:
# noththing to be done
return pi
assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \
'out of max_iters, total_timesteps, and max_episodes only one should be specified'
while True:
if callback: callback(locals(), globals())
if total_timesteps and timesteps_so_far >= total_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************" % iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
if traj_collect == 'mf-random' or traj_collect == 'mf-mb':
seg_mbl = seg_gen_mbl.__next__()
else:
seg_mbl = seg
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
# Val data collection
if collect_val_data:
for ob_, ac_, ob_next_ in zip(ob[:-1, 0, ...], ac[:-1, ...],
ob[1:, 0, ...]):
val_dataset_collect.append(
(copy.copy(ob_), copy.copy(ac_), copy.copy(ob_next_)))
# -----------------------------
# MBL update
else:
ob_mbl, ac_mbl = seg_mbl["ob"], seg_mbl["ac"]
mbl.add_data_batch(ob_mbl[:-1, 0, ...], ac_mbl[:-1, ...],
ob_mbl[1:, 0, ...])
mbl.update_forward_dynamic(require_update=iters_so_far %
mbl_train_freq == 0,
ob_val=val_dataset['ob'],
ac_val=val_dataset['ac'],
ob_next_val=val_dataset['ob_next'])
# -----------------------------
if traj_collect == 'mf':
#if traj_collect == 'mf' or traj_collect == 'mf-random' or traj_collect == 'mf-mb':
vpredbefore = seg[
"vpred"] # predicted value function before udpate
model = seg["model"]
atarg = (atarg - atarg.mean()) / atarg.std(
) # standardized advantage function estimate
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "rms"):
pi.rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
assign_old_eq_new(
) # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(),
vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False,
batch_size=64):
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
with timed("SIL"):
lrnow = lr(1.0 - timesteps_so_far / total_timesteps)
l_loss, sil_adv, sil_samples, sil_nlogp = model.sil_train(
lrnow)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
if MPI is not None:
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
else:
listoflrpairs = [lrlocal]
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if sil_update > 0:
logger.record_tabular("SilSamples", sil_samples)
if rank == 0:
# MBL evaluation
if not collect_val_data:
#set_global_seeds(seed)
default_sess = tf.get_default_session()
def multithread_eval_policy(env_, pi_, num_episodes_,
vis_eval_, seed):
with default_sess.as_default():
if hasattr(env, 'ob_rms') and hasattr(env_, 'ob_rms'):
env_.ob_rms = env.ob_rms
res = eval_policy(env_, pi_, num_episodes_, vis_eval_,
seed, measure_time, measure_rew)
try:
env_.close()
except:
pass
return res
if mbl.is_warm_start_done() and iters_so_far % eval_freq == 0:
warm_start_done = mbl.is_warm_start_done()
if num_eval_episodes > 0:
targs_names = {}
with timed('eval'):
num_descs = len(all_eval_descs)
list_field_names = [e[0] for e in all_eval_descs]
list_legend_names = [e[1] for e in all_eval_descs]
list_pis = [e[2] for e in all_eval_descs]
list_eval_envs = [
make_eval_env() for _ in range(num_descs)
]
list_seed = [seed for _ in range(num_descs)]
list_num_eval_episodes = [
num_eval_episodes for _ in range(num_descs)
]
print(list_field_names)
print(list_legend_names)
list_vis_eval = [
vis_eval for _ in range(num_descs)
]
for i in range(num_descs):
field_name, legend_name = list_field_names[
i], list_legend_names[i],
res = multithread_eval_policy(
list_eval_envs[i], list_pis[i],
list_num_eval_episodes[i],
list_vis_eval[i], seed)
#eval_results = pool.starmap(multithread_eval_policy, zip(list_eval_envs, list_pis, list_num_eval_episodes, list_vis_eval,list_seed))
#for field_name, legend_name, res in zip(list_field_names, list_legend_names, eval_results):
perf, elapsed_time, eval_rew = res
logger.record_tabular(field_name, perf)
if measure_time:
logger.record_tabular(
'Time-%s' % (field_name), elapsed_time)
if measure_rew:
logger.record_tabular(
'SimRew-%s' % (field_name), eval_rew)
targs_names[field_name] = legend_name
if eval_val_err:
fwd_dynamics_err = mbl.eval_forward_dynamic(
obs=eval_val_dataset['ob'],
acs=eval_val_dataset['ac'],
obs_next=eval_val_dataset['ob_next'])
logger.record_tabular('FwdValError', fwd_dynamics_err)
logger.dump_tabular()
#print(logger.get_dir())
#print(targs_names)
#if num_eval_episodes > 0:
# win = plot(viz, win, logger.get_dir(), targs_names=targs_names, quant=quant, opt='best')
# -----------
#logger.dump_tabular()
yield pi
if collect_val_data:
with open(validation_set_path, 'wb') as f:
pickle.dump(val_dataset_collect, f)
logger.log('Save {} validation data'.format(len(val_dataset_collect)))
def learn(
*,
network,
env,
total_timesteps,
timesteps_per_batch=1024, # what to train on
max_kl=0.002,
cg_iters=10,
gamma=0.99,
lam=1.0, # advantage estimation
seed=None,
ent_coef=0.00,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters=3,
max_episodes=0,
max_iters=0, # time constraint
callback=None,
load_path=None,
num_reward=1,
**network_kwargs):
'''
learn a policy function with TRPO algorithm
Parameters:
----------
network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
or function that takes input placeholder and returns tuple (output, None) for feedforward nets
or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets
env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class
timesteps_per_batch timesteps per gradient estimation batch
max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )
ent_coef coefficient of policy entropy term in the optimization objective
cg_iters number of iterations of conjugate gradient algorithm
cg_damping conjugate gradient damping
vf_stepsize learning rate for adam optimizer used to optimie value function loss
vf_iters number of iterations of value function optimization iterations per each policy optimization step
total_timesteps max number of timesteps
max_episodes max number of episodes
max_iters maximum number of policy optimization iterations
callback function to be called with (locals(), globals()) each policy optimization step
load_path str, path to load the model from (default: None, i.e. no model is loaded)
**network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
Returns:
-------
learnt model
'''
if MPI is not None:
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
else:
nworkers = 1
rank = 0
cpus_per_worker = 1
U.get_session(
config=tf.ConfigProto(allow_soft_placement=True,
inter_op_parallelism_threads=cpus_per_worker,
intra_op_parallelism_threads=cpus_per_worker))
set_global_seeds(seed)
# 创建policy
policy = build_policy(env,
network,
value_network='copy',
num_reward=num_reward,
**network_kwargs)
process_dir = logger.get_dir()
save_dir = process_dir.split(
'Data')[-2] + 'log/l2/seed' + process_dir[-1] + '/'
os.makedirs(save_dir, exist_ok=True)
coe_save = []
impro_save = []
grad_save = []
adj_save = []
coe = np.ones((num_reward)) / num_reward
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
#################################################################
# ob ac ret atarg 都是 placeholder
# ret atarg 此处应该是向量形式
ob = observation_placeholder(ob_space)
# 创建pi和oldpi
with tf.variable_scope("pi"):
pi = policy(observ_placeholder=ob)
with tf.variable_scope("oldpi"):
oldpi = policy(observ_placeholder=ob)
# 每个reward都可以算一个atarg
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32,
shape=[None, num_reward]) # Empirical return
ac = pi.pdtype.sample_placeholder([None])
#此处的KL div和entropy与reward无关
##################################
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
# entbonus 是entropy loss
entbonus = ent_coef * meanent
#################################
###########################################################
# vferr 用来更新 v 网络
vferr = tf.reduce_mean(tf.square(pi.vf - ret))
ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac))
# advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
# optimgain 用来更新 policy 网络, 应该每个reward有一个
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
###########################################################
dist = meankl
# 定义要优化的变量和 V 网络 adam 优化器
all_var_list = get_trainable_variables("pi")
# var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("pol")]
# vf_var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("vf")]
var_list = get_pi_trainable_variables("pi")
vf_var_list = get_vf_trainable_variables("pi")
vfadam = MpiAdam(vf_var_list)
# 把变量展开成一个向量的类
get_flat = U.GetFlat(var_list)
# 这个类可以把一个向量分片赋值给var_list里的变量
set_from_flat = U.SetFromFlat(var_list)
# kl散度的梯度
klgrads = tf.gradients(dist, var_list)
####################################################################
# 拉直的向量
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
# 把拉直的向量重新分成很多向量
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
####################################################################
####################################################################
# 把kl散度梯度与变量乘积相加
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
]) #pylint: disable=E1111
# 把gvp的梯度展成向量
fvp = U.flatgrad(gvp, var_list)
####################################################################
# 用学习后的策略更新old策略
assign_old_eq_new = U.function(
[], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(get_variables("oldpi"), get_variables("pi"))
])
# 计算loss
compute_losses = U.function([ob, ac, atarg], losses)
# 计算loss和梯度
compute_lossandgrad = U.function([ob, ac, atarg], losses +
[U.flatgrad(optimgain, var_list)])
# 计算fvp
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
# 计算值网络的梯度
compute_vflossandgrad = U.function([ob, ret],
U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
if MPI is not None:
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
else:
out = np.copy(x)
return out
# 初始化variable
U.initialize()
if load_path is not None:
pi.load(load_path)
# 得到初始化的参数向量
th_init = get_flat()
if MPI is not None:
MPI.COMM_WORLD.Bcast(th_init, root=0)
# 把向量the_init的值分片赋值给var_list
set_from_flat(th_init)
#同步
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
# 这是一个生成数据的迭代器
seg_gen = traj_segment_generator(pi,
env,
timesteps_per_batch,
stochastic=True,
num_reward=num_reward)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
# 双端队列
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
if sum([max_iters > 0, total_timesteps > 0, max_episodes > 0]) == 0:
# noththing to be done
return pi
assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \
'out of max_iters, total_timesteps, and max_episodes only one should be specified'
while True:
if callback: callback(locals(), globals())
if total_timesteps and timesteps_so_far >= total_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************" % iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
# 计算累积回报
add_vtarg_and_adv(seg, gamma, lam, num_reward=num_reward)
###########$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ToDo
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
# ob, ac, atarg, tdlamret 的类型都是ndarray
#ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg["tdlamret"]
_, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
#print(seg['ob'].shape,type(seg['ob']))
#print(seg['ac'],type(seg['ac']))
#print(seg['adv'],type(seg['adv']))
#print(seg["tdlamret"].shape,type(seg['tdlamret']))
vpredbefore = seg["vpred"] # predicted value function before udpate
# 标准化
#print("============================== atarg =========================================================")
#print(atarg)
atarg = (atarg - np.mean(atarg, axis=0)) / np.std(
atarg, axis=0) # standardized advantage function estimate
#atarg = (atarg) / np.max(np.abs(atarg),axis=0)
#print('======================================= standardized atarg ====================================')
#print(atarg)
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(ob) # update running mean/std for policy
## set old parameter values to new parameter values
assign_old_eq_new()
G = None
S = None
mr_lossbefore = np.zeros((num_reward, len(loss_names)))
grad_norm = np.zeros((num_reward + 1))
for i in range(num_reward):
args = seg["ob"], seg["ac"], atarg[:, i]
#print(atarg[:,i])
# 算是args的一个sample，每隔5个取出一个
fvpargs = [arr[::5] for arr in args]
# 这个函数计算fisher matrix 与向量 p 的 乘积
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
with timed("computegrad of " + str(i + 1) + ".th reward"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
mr_lossbefore[i] = lossbefore
g = allmean(g)
#print("***************************************************************")
#print(g)
if isinstance(G, np.ndarray):
G = np.vstack((G, g))
else:
G = g
# g是目标函数的梯度
# 利用共轭梯度获得更新方向
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg of " + str(i + 1) + ".th reward"):
# stepdir 是更新方向
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
grad_norm[i] = np.linalg.norm(fullstep)
assert np.isfinite(stepdir).all()
if isinstance(S, np.ndarray):
S = np.vstack((S, stepdir))
else:
S = stepdir
#print('======================================= G ====================================')
#print(G)
#print('======================================= S ====================================')
#print(S)
try:
new_coe = get_coefficient(G, S)
#coe = 0.99 * coe + 0.01 * new_coe
coe = new_coe
coe_save.append(coe)
#根据梯度的夹角调整参数
# GG = np.dot(S, S.T)
# D = np.sqrt(np.diag(1/np.diag(GG)))
# GG = np.dot(np.dot(D,GG),D)
# #print('======================================= inner product ====================================')
# #print(GG)
# adj = np.sum(GG) / (num_reward ** 2)
adj = 1
#print('======================================= adj ====================================')
#print(adj)
adj_save.append(adj)
adj_max_kl = adj * max_kl
#################################################################
grad_norm = grad_norm * np.sqrt(adj)
stepdir = np.dot(coe, S)
g = np.dot(coe, G)
lossbefore = np.dot(coe, mr_lossbefore)
#################################################################
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / adj_max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
grad_norm[num_reward] = np.linalg.norm(fullstep)
grad_save.append(grad_norm)
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
def compute_mr_losses():
mr_losses = np.zeros((num_reward, len(loss_names)))
for i in range(num_reward):
args = seg["ob"], seg["ac"], atarg[:, i]
one_reward_loss = allmean(np.array(compute_losses(*args)))
mr_losses[i] = one_reward_loss
mr_loss = np.dot(coe, mr_losses)
return mr_loss, mr_losses
# 做10次搜索
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
mr_loss_new, mr_losses_new = compute_mr_losses()
mr_impro = mr_losses_new - mr_lossbefore
meanlosses = surr, kl, *_ = allmean(np.array(mr_loss_new))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > adj_max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
impro_save.append(np.hstack((mr_impro[:, 0], improve)))
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
#print('======================================= tdlamret ====================================')
#print(seg["tdlamret"])
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False,
batch_size=64):
#with tf.Session() as sess:
# sess.run(tf.global_variables_initializer())
# aaa = sess.run(pi.vf,feed_dict={ob:mbob,ret:mbret})
# print("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
# print(aaa.shape)
# print(mbret.shape)
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
except:
print('error')
#print(mbob,mbret)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
if MPI is not None:
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
else:
listoflrpairs = [lrlocal]
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank == 0:
logger.dump_tabular()
#pdb.set_trace()
np.save(save_dir + 'coe.npy', coe_save)
np.save(save_dir + 'grad.npy', grad_save)
np.save(save_dir + 'improve.npy', impro_save)
np.save(save_dir + 'adj.npy', adj_save)
return pi
def run_hoof_no_lamgam(
network,
env,
total_timesteps,
timesteps_per_batch, # what to train on
kl_range,
gamma_range,
lam_range, # advantage estimation
num_kl,
num_gamma_lam,
cg_iters=10,
seed=None,
ent_coef=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters=3,
max_episodes=0,
max_iters=0, # time constraint
callback=None,
load_path=None,
**network_kwargs):
'''
learn a policy function with TRPO algorithm
Parameters:
----------
network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
or function that takes input placeholder and returns tuple (output, None) for feedforward nets
or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets
env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class
timesteps_per_batch timesteps per gradient estimation batch
max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )
ent_coef coefficient of policy entropy term in the optimization objective
cg_iters number of iterations of conjugate gradient algorithm
cg_damping conjugate gradient damping
vf_stepsize learning rate for adam optimizer used to optimie value function loss
vf_iters number of iterations of value function optimization iterations per each policy optimization step
total_timesteps max number of timesteps
max_episodes max number of episodes
max_iters maximum number of policy optimization iterations
callback function to be called with (locals(), globals()) each policy optimization step
load_path str, path to load the model from (default: None, i.e. no model is loaded)
**network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
Returns:
-------
learnt model
'''
MPI = None
nworkers = 1
rank = 0
cpus_per_worker = 1
U.get_session(
config=tf.ConfigProto(allow_soft_placement=True,
inter_op_parallelism_threads=cpus_per_worker,
intra_op_parallelism_threads=cpus_per_worker))
policy = build_policy(env, network, value_network='copy', **network_kwargs)
set_global_seeds(seed)
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
# +2 for gamma, lambda
ob = tf.placeholder(shape=(None, env.observation_space.shape[0] + 2),
dtype=env.observation_space.dtype,
name='Ob')
with tf.variable_scope("pi"):
pi = policy(observ_placeholder=ob)
with tf.variable_scope("oldpi"):
oldpi = policy(observ_placeholder=ob)
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
vferr = tf.reduce_mean(tf.square(pi.vf - ret))
ratio = tf.exp(pi.pd.logp(ac) -
oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = get_trainable_variables("pi")
var_list = get_pi_trainable_variables("pi")
vf_var_list = get_vf_trainable_variables("pi")
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function(
[], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(get_variables("oldpi"), get_variables("pi"))
])
compute_ratio = U.function(
[ob, ac, atarg], ratio) # IS ratio - used for computing IS weights
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses +
[U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret],
U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
if MPI is not None:
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
else:
out = np.copy(x)
return out
U.initialize()
if load_path is not None:
pi.load(load_path)
th_init = get_flat()
if MPI is not None:
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator_with_gl(pi,
env,
timesteps_per_batch,
stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
if sum([max_iters > 0, total_timesteps > 0, max_episodes > 0]) == 0:
# noththing to be done
return pi
assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \
'out of max_iters, total_timesteps, and max_episodes only one should be specified'
kl_range = np.atleast_1d(kl_range)
gamma_range = np.atleast_1d(gamma_range)
lam_range = np.atleast_1d(lam_range)
while True:
if callback: callback(locals(), globals())
if total_timesteps and timesteps_so_far >= total_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************" % iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
thbefore = get_flat()
rand_gamma = gamma_range[0] + (
gamma_range[-1] - gamma_range[0]) * np.random.rand(num_gamma_lam)
rand_lam = lam_range[0] + (
lam_range[-1] - lam_range[0]) * np.random.rand(num_gamma_lam)
rand_kl = kl_range[0] + (kl_range[-1] -
kl_range[0]) * np.random.rand(num_kl)
opt_polval = -10**8
est_polval = np.zeros((num_gamma_lam, num_kl))
ob_lam_gam = []
tdlamret = []
vpred = []
for gl in range(num_gamma_lam):
oblg, vpredbefore, atarg, tdlr = add_vtarg_and_adv_without_gl(
pi, seg, rand_gamma[gl], rand_lam[gl])
ob_lam_gam += [oblg]
tdlamret += [tdlr]
vpred += [vpredbefore]
atarg = (atarg - atarg.mean()) / atarg.std(
) # standardized advantage function estimate
pol_ob = np.concatenate(
(seg['ob'], np.zeros(seg['ob'].shape[:-1] + (2, ))), axis=-1)
args = pol_ob, seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
assign_old_eq_new(
) # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=False)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
surrbefore = lossbefore[0]
for m, kl in enumerate(rand_kl):
lm = np.sqrt(shs / kl)
fullstep = stepdir / lm
thnew = thbefore + fullstep
set_from_flat(thnew)
# compute the IS estimates
lik_ratio = compute_ratio(*args)
est_polval[gl, m] = wis_estimate(seg, lik_ratio)
# update best policy found so far
if est_polval[gl, m] > opt_polval:
opt_polval = est_polval[gl, m]
opt_th = thnew
opt_kl = kl
opt_gamma = rand_gamma[gl]
opt_lam = rand_lam[gl]
opt_vpredbefore = vpredbefore
opt_tdlr = tdlr
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses(*args)))
improve = surr - surrbefore
expectedimprove = g.dot(fullstep)
set_from_flat(thbefore)
logger.log("Expected: %.3f Actual: %.3f" % (expectedimprove, improve))
set_from_flat(opt_th)
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
ob_lam_gam = np.concatenate(ob_lam_gam, axis=0)
tdlamret = np.concatenate(tdlamret, axis=0)
vpred = np.concatenate(vpred, axis=0)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(ob_lam_gam, tdlamret),
include_final_partial_batch=False,
batch_size=num_gamma_lam * 64):
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpred, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
if MPI is not None:
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
else:
listoflrpairs = [lrlocal]
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
logger.record_tabular("Opt_KL", opt_kl)
logger.record_tabular("gamma", opt_gamma)
logger.record_tabular("lam", opt_lam)
if rank == 0:
logger.dump_tabular()
return pi
def learn(
env,
policy_fn,
*,
timesteps_per_batch, # what to train on
epsilon,
beta,
cg_iters,
gamma,
lam, # advantage estimation
entcoeff=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters=3,
max_timesteps=0,
max_episodes=0,
max_iters=0, # time constraint
callback=None,
TRPO=False,
n_policy=1,
policy_type=0,
filepath='',
session,
retrace=False):
'''
:param TRPO: True: TRPO, False: COPOS
:param n_policy: Number of periodic policy parts
:param policy_type: 0: Optimize 'n_policy' policies that are executed periodically. All the policies are updated.
1: The last 'n_policy' policies are executed periodically but only the last one is optimized.
2: The policy is spread over 'n_policy' time steps.
'''
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
pis = [
policy_fn("pi_" + str(i), ob_space, ac_space) for i in range(n_policy)
]
oldpis = [
policy_fn("oldpi_" + str(i), ob_space, ac_space)
for i in range(n_policy)
]
pi_vf = policy_fn("pi_vf", ob_space, ac_space)
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ob = U.get_placeholder_cached(name="ob")
if policy_type == 0:
print(
"Policy type: " + str(policy_type) +
". Optimize 'n_policy' policies that are executed periodically. All the policies are updated."
)
elif policy_type == 1:
print(
"Policy type: " + str(policy_type) +
". The last 'n_policy' policies are executed periodically but only the last one is optimized."
)
elif policy_type == 2:
print("Policy type: " + str(policy_type) +
". The policy is spread over 'n_policy' time steps.")
else:
print("Policy type: " + str(policy_type) + " is not supported.")
# Compute variables for each policy separately
old_entropy = []
get_flat = []
set_from_flat = []
assign_old_eq_new = []
copy_policy_back = []
compute_losses = []
compute_lossandgrad = []
compute_fvp = []
for i in range(n_policy):
pi = pis[i]
oldpi = oldpis[i]
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
old_entropy.append(oldpi.pd.entropy())
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = entcoeff * meanent
ratio = tf.exp(pi.pd.logp(ac) -
oldpi.pd.logp(ac)) # advantage * pnew / pold
if retrace:
surrgain = tf.reduce_mean(
atarg) # atarg incorporates pnew / pold already
else:
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "Entropy"]
dist = meankl
all_var_list = pi.get_trainable_variables()
all_var_list = [
v for v in all_var_list if v.name.split("/")[0].startswith("pi")
]
var_list = [
v for v in all_var_list if v.name.split("/")[1].startswith("pol")
]
#
# fvp: Fisher Information Matrix / vector product based on Hessian of KL-divergence
# fvp = F * v, where F = - E \partial_1 \partial_2 KL_div(p1 || p2)
#
get_flat.append(U.GetFlat(var_list))
set_from_flat.append(U.SetFromFlat(var_list))
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
#
# fvpll: Fisher Information Matrix / vector product based on exact FIM
# fvpll = F * v, where F = E[\partial_1 \log p * \partial_2 \log p]
#
# Mean: (\partial \mu^T / \partial param1) * Precision * (\partial \mu / \partial param1)
# Covariance: 0.5 * Trace[Precision * (\partial Cov / \partial param1) *
# Precision * (\partial Cov / \partial param2)]
if i > 0:
# Only needed for policy_type == 1 for copying policy 'i' to policy 'i-1'
copy_policy_back.append(
U.function(
[], [],
updates=[
tf.assign(oldv, newv) for (oldv, newv) in zipsame(
pis[i - 1].get_variables(), pi.get_variables())
]))
assign_old_eq_new.append(
U.function([], [],
updates=[
tf.assign(oldv, newv) for (oldv, newv) in zipsame(
oldpi.get_variables(), pi.get_variables())
]))
compute_losses.append(U.function([ob, ac, atarg], losses))
compute_lossandgrad.append(
U.function([ob, ac, atarg],
losses + [U.flatgrad(optimgain, var_list)]))
compute_fvp.append(U.function([flat_tangent, ob, ac, atarg], fvp))
# Value function is global to all policies
vferr = tf.reduce_mean(tf.square(pi_vf.vpred - ret))
all_var_list = pi_vf.get_trainable_variables()
vf_var_list = [
v for v in all_var_list if v.name.split("/")[1].startswith("vf")
]
vfadam = MpiAdam(vf_var_list)
compute_vflossandgrad = U.function([ob, ret],
U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
U.initialize()
if policy_type == 1:
# Initialize policies to identical values
th_init = get_flat[0]()
for i in range(n_policy):
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat[i](th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
else:
for i in range(n_policy):
th_init = get_flat[i]()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat[i](th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Initialize eta, omega optimizer
init_eta = 0.5
init_omega = 2.0
eta_omega_optimizer = EtaOmegaOptimizer(beta, epsilon, init_eta,
init_omega)
# Prepare for rollouts
# ----------------------------------------
seg_gen = []
for i in range(len(pis)):
seg_gen.append(
traj_segment_generator(pis,
i + 1,
pi_vf,
env,
timesteps_per_batch,
stochastic=True))
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
assert sum([max_iters > 0, max_timesteps > 0, max_episodes > 0]) == 1
n_saves = 0
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************" % iters_so_far)
if max_timesteps > 0 and (timesteps_so_far >=
(n_saves * max_timesteps // 5)):
# Save policy
saver = tf.train.Saver()
saver.save(session, filepath + "_" + str(iters_so_far))
n_saves += 1
with timed("sampling"):
if policy_type == 1 and iters_so_far < len(pis):
all_seg = seg_gen[iters_so_far].__next__(
) # For four time steps use the four policies
else:
all_seg = seg_gen[-1].__next__()
if policy_type == 1 and retrace:
act_pi_ids = np.empty_like(all_seg["vpred"], dtype=int)
act_pi_ids[:] = n_policy - 1 # Always update the last policy
add_vtarg_and_adv_retrace(all_seg, gamma, lam, act_pi_ids)
else:
add_vtarg_and_adv(all_seg, gamma, lam)
# Split the advantage functions etc. among the policies
segs = split_traj_segment(pis, all_seg)
# Update all policies
for pi_id in range(n_policy):
if policy_type == 1:
# Update only last policy
pi_id = n_policy - 1
# Using all the samples
seg = all_seg
else:
seg = segs[pi_id]
pi = pis[pi_id]
oldpi = oldpis[pi_id]
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
vpredbefore = seg[
"vpred"] # predicted value function before update
atarg = (atarg - atarg.mean()) / atarg.std(
) # standardized advantage function estimate
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp[pi_id](p, *
fvpargs)) + cg_damping * p
assign_old_eq_new[pi_id](
) # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad[pi_id](*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
if policy_type == 1:
# Update only the last policy
break
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
if TRPO:
#
# TRPO specific code.
# Find correct step size using line search
#
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / epsilon)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat[pi_id]()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat[pi_id](thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses[pi_id](*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log(
"Got non-finite value of losses -- bad!")
elif kl > epsilon * 1.5:
logger.log(
"violated KL constraint. shrinking step.")
elif improve < 0:
logger.log(
"surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat[pi_id](thbefore)
else:
#
# COPOS specific implementation.
#
copos_update_dir = stepdir
# Split direction into log-linear 'w_theta' and non-linear 'w_beta' parts
w_theta, w_beta = pi.split_w(copos_update_dir)
# q_beta(s,a) = \grad_beta \log \pi(a|s) * w_beta
# = features_beta(s) * K^T * Prec * a
# q_beta = self.target.get_q_beta(features_beta, actions)
Waa, Wsa = pi.w2W(w_theta)
wa = pi.get_wa(ob, w_beta)
varphis = pi.get_varphis(ob)
# Optimize eta and omega
tmp_ob = np.zeros(
(1, ) + env.observation_space.shape
) # We assume that entropy does not depend on the NN
old_ent = old_entropy[pi_id].eval({oldpi.ob: tmp_ob})[0]
eta, omega = eta_omega_optimizer.optimize(
w_theta, Waa, Wsa, wa, varphis, pi.get_kt(),
pi.get_prec_matrix(), pi.is_new_policy_valid, old_ent)
logger.log("Initial eta: " + str(eta) + " and omega: " +
str(omega))
current_theta_beta = get_flat[pi_id]()
prev_theta, prev_beta = pi.all_to_theta_beta(
current_theta_beta)
for i in range(2):
# Do a line search for both theta and beta parameters by adjusting only eta
eta = eta_search(w_theta, w_beta, eta, omega, allmean,
compute_losses[pi_id],
get_flat[pi_id], set_from_flat[pi_id],
pi, epsilon, args)
logger.log("Updated eta, eta: " + str(eta) +
" and omega: " + str(omega))
# Find proper omega for new eta. Use old policy parameters first.
set_from_flat[pi_id](pi.theta_beta_to_all(
prev_theta, prev_beta))
eta, omega = \
eta_omega_optimizer.optimize(w_theta, Waa, Wsa, wa, varphis, pi.get_kt(),
pi.get_prec_matrix(), pi.is_new_policy_valid, old_ent, eta)
logger.log("Updated omega, eta: " + str(eta) +
" and omega: " + str(omega))
# Use final policy
logger.log("Final eta: " + str(eta) + " and omega: " +
str(omega))
cur_theta = (eta * prev_theta +
w_theta.reshape(-1, )) / (eta + omega)
cur_beta = prev_beta + w_beta.reshape(-1, ) / eta
thnew = pi.theta_beta_to_all(cur_theta, cur_beta)
set_from_flat[pi_id](thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses[pi_id](*args)))
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(),
vfadam[pi_id].getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
logger.record_tabular(
"ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
if policy_type == 1:
# Update only the last policy
break
if policy_type == 1:
# Copy policies 1, ..., i to 0, ..., i-1
for j in range(n_policy - 1):
copy_policy_back[j]()
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(all_seg["ob"], all_seg["tdlamret"]),
include_final_partial_batch=False,
batch_size=64):
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
lrlocal = (all_seg["ep_lens"], all_seg["ep_rets"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("AverageReturn", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank == 0:
logger.dump_tabular()
def learn(env, policy_func, reward_giver, expert_dataset, rank,
pretrained, pretrained_weight, *,
g_step, d_step, entcoeff, save_per_iter,
ckpt_dir, timesteps_per_batch, task_name,
gamma, lam,
max_kl, cg_iters, cg_damping=1e-2,
vf_stepsize=3e-4, d_stepsize=3e-4, vf_iters=3,
max_timesteps=0, max_episodes=0, max_iters=0,
callback=None
):
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
pi = policy_func("pi", ob_space, ac_space, reuse=(pretrained_weight != None))
oldpi = policy_func("oldpi", ob_space, ac_space)
atarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ob = U.get_placeholder_cached(name="ob")
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = entcoeff * meanent
vferr = tf.reduce_mean(tf.square(pi.vpred - ret))
ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = pi.get_trainable_variables()
var_list = [v for v in all_var_list if v.name.startswith("pi/pol") or v.name.startswith("pi/logstd")]
vf_var_list = [v for v in all_var_list if v.name.startswith("pi/vff")]
assert len(var_list) == len(vf_var_list) + 1
d_adam = MpiAdam(reward_giver.get_trainable_variables())
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))
start += sz
gvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) # pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function([], [], updates=[tf.assign(oldv, newv)
for (oldv, newv) in zipsame(oldpi.get_variables(), pi.get_variables())])
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
U.initialize()
th_init = get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
d_adam.sync()
vfadam.sync()
if rank == 0:
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi, env, reward_giver, timesteps_per_batch, stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
true_rewbuffer = deque(maxlen=40)
assert sum([max_iters > 0, max_timesteps > 0, max_episodes > 0]) == 1
# g_loss_stats = stats(loss_names)
# d_loss_stats = stats(reward_giver.loss_name)
ep_stats = stats(["True_rewards", "Rewards", "Episode_length"])
# if provide pretrained weight
if pretrained_weight is not None:
U.load_variables(pretrained_weight, variables=pi.get_variables())
best=-2000
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************" % iters_so_far)
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
# ------------------ Update G ------------------
logger.log("Optimizing Policy...")
for _ in range(g_step):
seg = seg_gen.__next__()
#report stats and save policy if any good
lrlocal = (seg["ep_lens"], seg["ep_rets"], seg["ep_true_rets"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews, true_rets = map(flatten_lists, zip(*listoflrpairs))
true_rewbuffer.extend(true_rets)
lenbuffer.extend(lens)
rewbuffer.extend(rews)
true_rew_avg = np.mean(true_rewbuffer)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpTrueRewMean", true_rew_avg)
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
logger.record_tabular("Best so far", best)
# Save model
if ckpt_dir is not None and true_rew_avg >= best and len(true_rewbuffer) > 30:
best = true_rew_avg
fname = os.path.join(ckpt_dir, task_name)
os.makedirs(os.path.dirname(fname), exist_ok=True)
pi.save_policy(fname)
#compute gradient towards next policy
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg["tdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()) / atarg.std() # standardized advantage function estimate
if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
assign_old_eq_new() # set old parameter values to new parameter values
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=False)
assert np.isfinite(stepdir).all()
shs = .5*stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" % (expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches((seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False, batch_size=128):
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(mbob) # update running mean/std for policy
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
g_losses = meanlosses
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
logger.record_tabular("ev_tdlam_before", explained_variance(vpredbefore, tdlamret))
# ------------------ Update D ------------------
batch_size = len(ob) // d_step
d_losses = [] # list of tuples, each of which gives the loss for a minibatch
for ob_batch, ac_batch in dataset.iterbatches((ob, ac),
include_final_partial_batch=False,
batch_size=batch_size):
ob_expert, ac_expert = expert_dataset.next_batch(len(ob_batch))
# update running mean/std for reward_giver
if hasattr(reward_giver, "obs_rms"): reward_giver.obs_rms.update(np.concatenate((ob_batch, ob_expert), 0))
*newlosses, g = reward_giver.lossandgrad(ob_batch, ac_batch, ob_expert, ac_expert)
d_adam.update(allmean(g), d_stepsize)
d_losses.append(newlosses)
logger.log(fmt_row(13, reward_giver.loss_name))
logger.log(fmt_row(13, np.mean(d_losses, axis=0)))
if rank == 0:
logger.dump_tabular()
def learn(
*,
network,
env,
eval_env,
total_timesteps,
timesteps_per_batch=1024, # what to train on
max_kl=0.001,
cg_iters=10,
gamma=0.99,
lam=1.0, # advantage estimation
seed=None,
ent_coef=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters=3,
log_path=None,
max_episodes=0,
max_iters=0, # time constraint
callback=None,
load_path=None,
**network_kwargs):
'''
learn a policy function with TRPO algorithm
Parameters:
----------
network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
or function that takes input placeholder and returns tuple (output, None) for feedforward nets
or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets
env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class
timesteps_per_batch timesteps per gradient estimation batch
max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )
ent_coef coefficient of policy entropy term in the optimization objective
cg_iters number of iterations of conjugate gradient algorithm
cg_damping conjugate gradient damping
vf_stepsize learning rate for adam optimizer used to optimie value function loss
vf_iters number of iterations of value function optimization iterations per each policy optimization step
total_timesteps max number of timesteps
max_episodes max number of episodes
max_iters maximum number of policy optimization iterations
callback function to be called with (locals(), globals()) each policy optimization step
load_path str, path to load the model from (default: None, i.e. no model is loaded)
**network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
Returns:
-------
learnt model
'''
if MPI is not None:
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
else:
nworkers = 1
rank = 0
set_global_seeds(seed)
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
if isinstance(network, str):
network = get_network_builder(network)(**network_kwargs)
with tf.name_scope("pi"):
pi_policy_network = network(ob_space.shape)
pi_value_network = network(ob_space.shape)
pi = PolicyWithValue(ac_space, pi_policy_network, pi_value_network)
with tf.name_scope("oldpi"):
old_pi_policy_network = network(ob_space.shape)
old_pi_value_network = network(ob_space.shape)
oldpi = PolicyWithValue(ac_space, old_pi_policy_network,
old_pi_value_network)
pi_var_list = pi_policy_network.trainable_variables + list(
pi.pdtype.trainable_variables)
old_pi_var_list = old_pi_policy_network.trainable_variables + list(
oldpi.pdtype.trainable_variables)
vf_var_list = pi_value_network.trainable_variables + pi.value_fc.trainable_variables
old_vf_var_list = old_pi_value_network.trainable_variables + oldpi.value_fc.trainable_variables
if load_path is not None:
load_path = osp.expanduser(load_path)
ckpt = tf.train.Checkpoint(model=pi)
manager = tf.train.CheckpointManager(ckpt, load_path, max_to_keep=None)
ckpt.restore(manager.latest_checkpoint)
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(pi_var_list)
set_from_flat = U.SetFromFlat(pi_var_list)
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
shapes = [var.get_shape().as_list() for var in pi_var_list]
def assign_old_eq_new():
for pi_var, old_pi_var in zip(pi_var_list, old_pi_var_list):
old_pi_var.assign(pi_var)
for vf_var, old_vf_var in zip(vf_var_list, old_vf_var_list):
old_vf_var.assign(vf_var)
@tf.function
def compute_lossandgrad(ob, ac, atarg):
with tf.GradientTape() as tape:
old_policy_latent = oldpi.policy_network(ob)
old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent)
policy_latent = pi.policy_network(ob)
pd, _ = pi.pdtype.pdfromlatent(policy_latent)
kloldnew = old_pd.kl(pd)
ent = pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
ratio = tf.exp(pd.logp(ac) - old_pd.logp(ac))
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
gradients = tape.gradient(optimgain, pi_var_list)
return losses + [U.flatgrad(gradients, pi_var_list)]
@tf.function
def compute_losses(ob, ac, atarg):
old_policy_latent = oldpi.policy_network(ob)
old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent)
policy_latent = pi.policy_network(ob)
pd, _ = pi.pdtype.pdfromlatent(policy_latent)
kloldnew = old_pd.kl(pd)
ent = pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
ratio = tf.exp(pd.logp(ac) - old_pd.logp(ac))
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
return losses
#ob shape should be [batch_size, ob_dim], merged nenv
#ret shape should be [batch_size]
@tf.function
def compute_vflossandgrad(ob, ret):
with tf.GradientTape() as tape:
pi_vf = pi.value(ob)
vferr = tf.reduce_mean(tf.square(pi_vf - ret))
return U.flatgrad(tape.gradient(vferr, vf_var_list), vf_var_list)
@tf.function
def compute_fvp(flat_tangent, ob, ac, atarg):
with tf.GradientTape() as outter_tape:
with tf.GradientTape() as inner_tape:
old_policy_latent = oldpi.policy_network(ob)
old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent)
policy_latent = pi.policy_network(ob)
pd, _ = pi.pdtype.pdfromlatent(policy_latent)
kloldnew = old_pd.kl(pd)
meankl = tf.reduce_mean(kloldnew)
klgrads = inner_tape.gradient(meankl, pi_var_list)
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(
tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
])
hessians_products = outter_tape.gradient(gvp, pi_var_list)
fvp = U.flatgrad(hessians_products, pi_var_list)
return fvp
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
if MPI is not None:
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
else:
out = np.copy(x)
return out
th_init = get_flat()
if MPI is not None:
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi, env, timesteps_per_batch)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
logdir = log_path + '/evaluator'
modeldir = log_path + '/models'
if not os.path.exists(logdir):
os.makedirs(logdir)
if not os.path.exists(modeldir):
os.makedirs(modeldir)
evaluator = Evaluator(env=eval_env, model=pi, logdir=logdir)
max_inner_iter = 500000 if env.spec.id == 'InvertedDoublePendulum-v2' else 3000000
epoch = vf_iters
batch_size = timesteps_per_batch
mb_size = 256
inner_iter_per_iter = epoch * int(batch_size / mb_size)
max_iter = int(max_inner_iter / inner_iter_per_iter)
eval_num = 150
eval_interval = save_interval = int(
int(max_inner_iter / eval_num) / inner_iter_per_iter)
if sum([max_iters > 0, total_timesteps > 0, max_episodes > 0]) == 0:
# noththing to be done
return pi
assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \
'out of max_iters, total_timesteps, and max_episodes only one should be specified'
for update in range(1, max_iter + 1):
if callback: callback(locals(), globals())
# if total_timesteps and timesteps_so_far >= total_timesteps:
# break
# elif max_episodes and episodes_so_far >= max_episodes:
# break
# elif max_iters and iters_so_far >= max_iters:
# break
logger.log("********** Iteration %i ************" % iters_so_far)
if (update - 1) % eval_interval == 0:
evaluator.run_evaluation(update - 1)
if (update - 1) % save_interval == 0:
ckpt = tf.train.Checkpoint(model=pi)
ckpt.save(modeldir + '/ckpt_ite' + str((update - 1)))
with timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
ob = sf01(ob)
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()
) / atarg.std() # standardized advantage function estimate
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(ob) # update running mean/std for policy
args = ob, ac, atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs).numpy()) + cg_damping * p
assign_old_eq_new() # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = g.numpy()
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False,
batch_size=mb_size):
mbob = sf01(mbob)
g = allmean(compute_vflossandgrad(mbob, mbret).numpy())
vfadam.update(g, vf_stepsize)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
if MPI is not None:
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
else:
listoflrpairs = [lrlocal]
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank == 0:
logger.dump_tabular()
return pi
def run_pg(
algo,
network,
env,
total_timesteps,
timesteps_per_batch,
max_kl=0.001,
cg_iters=10,
gamma=0.99,
lam=1.0, # advantage estimation
seed=None,
ent_coef=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters =3,
max_episodes=0, max_iters=0, # time constraint
callback=None,
load_path=None,
**network_kwargs
):
'''
learn a policy function with TRPO algorithm
Parameters:
----------
network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
or function that takes input placeholder and returns tuple (output, None) for feedforward nets
or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets
env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class
timesteps_per_batch timesteps per gradient estimation batch
max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )
ent_coef coefficient of policy entropy term in the optimization objective
cg_iters number of iterations of conjugate gradient algorithm
cg_damping conjugate gradient damping
vf_stepsize learning rate for adam optimizer used to optimie value function loss
vf_iters number of iterations of value function optimization iterations per each policy optimization step
total_timesteps max number of timesteps
max_episodes max number of episodes
max_iters maximum number of policy optimization iterations
callback function to be called with (locals(), globals()) each policy optimization step
load_path str, path to load the model from (default: None, i.e. no model is loaded)
**network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
Returns:
-------
learnt model
'''
MPI = None
nworkers = 1
rank = 0
cpus_per_worker = 1
U.get_session(config=tf.ConfigProto(
allow_soft_placement=True,
inter_op_parallelism_threads=cpus_per_worker,
intra_op_parallelism_threads=cpus_per_worker
))
policy = build_policy(env, network, value_network='copy', **network_kwargs)
set_global_seeds(seed)
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
ob = observation_placeholder(ob_space)
with tf.variable_scope("pi"):
pi = policy(observ_placeholder=ob)
with tf.variable_scope("oldpi"):
oldpi = policy(observ_placeholder=ob)
atarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
vferr = tf.reduce_mean(tf.square(pi.vf - ret))
ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = get_trainable_variables("pi")
# var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("pol")]
# vf_var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("vf")]
var_list = get_pi_trainable_variables("pi")
vf_var_list = get_vf_trainable_variables("pi")
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))
start += sz
gvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function([],[], updates=[tf.assign(oldv, newv)
for (oldv, newv) in zipsame(get_variables("oldpi"), get_variables("pi"))])
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(colorize("done in %.3f seconds"%(time.time() - tstart), color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
if MPI is not None:
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
else:
out = np.copy(x)
return out
U.initialize()
if load_path is not None:
pi.load(load_path)
th_init = get_flat()
if MPI is not None:
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi, env, timesteps_per_batch, stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
if sum([max_iters>0, total_timesteps>0, max_episodes>0])==0:
# noththing to be done
return pi
assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \
'out of max_iters, total_timesteps, and max_episodes only one should be specified'
while True:
if callback: callback(locals(), globals())
if total_timesteps and timesteps_so_far >= total_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************"%iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg["tdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()) / atarg.std() # standardized advantage function estimate
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
assign_old_eq_new() # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank==0)
assert np.isfinite(stepdir).all()
shs = .5*stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
""" changes here """
if algo=='TNPG':
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f"%(expectedimprove, improve))
else:
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f"%(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .8
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
# changes end
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches((seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False, batch_size=64):
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
logger.record_tabular("ev_tdlam_before", explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
if MPI is not None:
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
else:
listoflrpairs = [lrlocal]
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("Opt_KL", max_kl)
logger.record_tabular("gamma", gamma)
logger.record_tabular("lam", lam)
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank==0:
logger.dump_tabular()
return pi
def learn(env,
policy_func,
reward_giver,
expert_dataset,
rank,
g_step,
d_step,
entcoeff,
save_per_iter,
timesteps_per_batch,
ckpt_dir,
log_dir,
task_name,
gamma,
lam,
max_kl,
cg_iters,
cg_damping=1e-2,
vf_stepsize=3e-4,
d_stepsize=3e-4,
vf_iters=3,
max_timesteps=0,
max_episodes=0,
max_iters=0,
callback=None):
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
pi = policy_func("pi", ob_space, ac_space)
saver = tf.train.Saver(
var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='pi'))
saver.restore(tf.get_default_session(), U_.getPath() + '/model/bc.ckpt')
oldpi = policy_func("oldpi", ob_space, ac_space)
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ob = U.get_placeholder_cached(name="ob")
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = entcoeff * meanent
vferr = tf.reduce_mean(tf.square(pi.vpred - ret))
ratio = tf.exp(pi.pd.logp(ac) -
oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = pi.get_trainable_variables()
var_list = [
v for v in all_var_list
if v.name.startswith("pi/pol") or v.name.startswith("pi/logstd")
]
vf_var_list = [v for v in all_var_list if v.name.startswith("pi/vff")]
assert len(var_list) == len(vf_var_list) + 1
d_adam = MpiAdam(reward_giver.get_trainable_variables())
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
]) # pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function(
[], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(oldpi.get_variables(), pi.get_variables())
])
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses +
[U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret],
U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
U.initialize()
th_init = get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
d_adam.sync()
vfadam.sync()
if rank == 0:
print("Init param sum", th_init.sum())
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi,
env,
reward_giver,
timesteps_per_batch,
stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
true_rewbuffer = deque(maxlen=40)
assert sum([max_iters > 0, max_timesteps > 0, max_episodes > 0]) == 1
g_loss_stats = stats(loss_names)
d_loss_stats = stats(reward_giver.loss_name)
ep_stats = stats(["True_rewards", "Rewards", "Episode_length"])
# if provide pretrained weight
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
# Save model
if rank == 0 and iters_so_far % save_per_iter == 0 and ckpt_dir is not None:
fname = os.path.join(ckpt_dir, task_name)
print('save model as ', fname)
try:
os.makedirs(os.path.dirname(fname))
except OSError:
# folder already exists
pass
saver = tf.train.Saver()
saver.save(tf.get_default_session(), fname)
print("********** Iteration %i ************" % iters_so_far)
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
# ------------------ Update G ------------------
print("Optimizing Policy...")
for _ in range(g_step):
with timed("sampling"):
seg = seg_gen.next()
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
vpredbefore = seg[
"vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()) / atarg.std(
) # standardized advantage function estimate
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
assign_old_eq_new(
) # set old parameter values to new parameter values
with timed("computegrad"):
tmp_result = compute_lossandgrad(seg["ob"], seg["ac"], atarg)
lossbefore = tmp_result[:-1]
g = tmp_result[-1]
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
print("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# print("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = allmean(
np.array(compute_losses(seg["ob"], seg["ac"], atarg)))
surr = meanlosses[0]
kl = meanlosses[1]
improve = surr - surrbefore
print("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
print("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
print("violated KL constraint. shrinking step.")
elif improve < 0:
print("surrogate didn't improve. shrinking step.")
else:
print("Stepsize OK!")
break
stepsize *= .5
else:
print("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(),
vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False,
batch_size=128):
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(
mbob) # update running mean/std for policy
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
print("ev_tdlam_before", explained_variance(vpredbefore, tdlamret))
# ------------------ Update D ------------------
print("Optimizing Discriminator...")
print(fmt_row(13, reward_giver.loss_name))
ob_expert, ac_expert = expert_dataset.get_next_batch(len(ob))
batch_size = len(ob) // d_step
d_losses = [
] # list of tuples, each of which gives the loss for a minibatch
for ob_batch, ac_batch in tqdm(
dataset.iterbatches((ob, ac),
include_final_partial_batch=False,
batch_size=batch_size)):
ob_expert, ac_expert = expert_dataset.get_next_batch(len(ob_batch))
# update running mean/std for reward_giver
if hasattr(reward_giver, "obs_rms"):
reward_giver.obs_rms.update(
np.concatenate((ob_batch, ob_expert), 0))
tmp_result = reward_giver.lossandgrad(ob_batch, ac_batch,
ob_expert, ac_expert)
newlosses = tmp_result[:-1]
g = tmp_result[-1]
d_adam.update(allmean(g), d_stepsize)
d_losses.append(newlosses)
print(fmt_row(13, np.mean(d_losses, axis=0)))
timesteps_so_far += len(seg['ob'])
iters_so_far += 1
print("EpisodesSoFar", episodes_so_far)
print("TimestepsSoFar", timesteps_so_far)
print("TimeElapsed", time.time() - tstart)
def learn(
env,
policy_fn,
*,
timesteps_per_batch, # what to train on
epsilon,
beta,
cg_iters,
gamma,
lam, # advantage estimation
entcoeff=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters=3,
max_timesteps=0,
max_episodes=0,
max_iters=0, # time constraint
callback=None,
TRPO=False):
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
discrete_ac_space = isinstance(ac_space, gym.spaces.Discrete)
print("ob_space: " + str(ob_space))
print("ac_space: " + str(ac_space))
pi = policy_fn("pi", ob_space, ac_space)
oldpi = policy_fn("oldpi", ob_space, ac_space)
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ob = U.get_placeholder_cached(name="ob")
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
old_entropy = oldpi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = entcoeff * meanent
vferr = tf.reduce_mean(tf.square(pi.vpred - ret))
ratio = tf.exp(pi.pd.logp(ac) -
oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "Entropy"]
dist = meankl
all_var_list = pi.get_trainable_variables()
all_var_list = [
v for v in all_var_list if v.name.split("/")[0].startswith("pi")
]
var_list = [
v for v in all_var_list if v.name.split("/")[1].startswith("pol")
]
vf_var_list = [
v for v in all_var_list if v.name.split("/")[1].startswith("vf")
]
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
#????gvp and fvp???
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function(
[], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(oldpi.get_variables(), pi.get_variables())
])
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses +
[U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret],
U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
U.initialize()
th_init = get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Initialize eta, omega optimizer
if discrete_ac_space:
init_eta = 1
init_omega = 0.5
eta_omega_optimizer = EtaOmegaOptimizerDiscrete(
beta, epsilon, init_eta, init_omega)
else:
init_eta = 0.5
init_omega = 2.0
#????eta_omega_optimizer details?????
eta_omega_optimizer = EtaOmegaOptimizer(beta, epsilon, init_eta,
init_omega)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi,
env,
timesteps_per_batch,
stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
assert sum([max_iters > 0, max_timesteps > 0, max_episodes > 0]) == 1
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************" % iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam)
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()
) / atarg.std() # standardized advantage function estimate
#print(ob[:20])
#print(ac[:20])
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"):
print(pi.ob_rms.mean)
pi.ob_rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
assign_old_eq_new() # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
if TRPO:
#
# TRPO specific code.
# Find correct step size using line search
#
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / epsilon)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > epsilon * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
'''else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)'''
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(),
vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
else:
#
# COPOS specific implementation.
#
copos_update_dir = stepdir
# Split direction into log-linear 'w_theta' and non-linear 'w_beta' parts
w_theta, w_beta = pi.split_w(copos_update_dir)
tmp_ob = np.zeros(
(1, ) + env.observation_space.shape
) # We assume that entropy does not depend on the NN
# Optimize eta and omega
if discrete_ac_space:
entropy = lossbefore[4]
#entropy = - 1/timesteps_per_batch * np.sum(np.sum(pi.get_action_prob(ob) * pi.get_log_action_prob(ob), axis=1))
eta, omega = eta_omega_optimizer.optimize(
pi.compute_F_w(ob, copos_update_dir),
pi.get_log_action_prob(ob), timesteps_per_batch,
entropy)
else:
Waa, Wsa = pi.w2W(w_theta)
wa = pi.get_wa(ob, w_beta)
varphis = pi.get_varphis(ob)
#old_ent = old_entropy.eval({oldpi.ob: tmp_ob})[0]
old_ent = lossbefore[4]
eta, omega = eta_omega_optimizer.optimize(
w_theta, Waa, Wsa, wa, varphis, pi.get_kt(),
pi.get_prec_matrix(), pi.is_new_policy_valid, old_ent)
logger.log("Initial eta: " + str(eta) + " and omega: " +
str(omega))
current_theta_beta = get_flat()
prev_theta, prev_beta = pi.all_to_theta_beta(
current_theta_beta)
if discrete_ac_space:
# Do a line search for both theta and beta parameters by adjusting only eta
eta = eta_search(w_theta, w_beta, eta, omega, allmean,
compute_losses, get_flat, set_from_flat,
pi, epsilon, args, discrete_ac_space)
logger.log("Updated eta, eta: " + str(eta))
set_from_flat(pi.theta_beta_to_all(prev_theta, prev_beta))
# Find proper omega for new eta. Use old policy parameters first.
eta, omega = eta_omega_optimizer.optimize(
pi.compute_F_w(ob, copos_update_dir),
pi.get_log_action_prob(ob), timesteps_per_batch,
entropy, eta)
logger.log("Updated omega, eta: " + str(eta) +
" and omega: " + str(omega))
# do line search for ratio for non-linear "beta" parameter values
#ratio = beta_ratio_line_search(w_theta, w_beta, eta, omega, allmean, compute_losses, get_flat, set_from_flat, pi,
# epsilon, beta, args)
# set ratio to 1 if we do not use beta ratio line search
ratio = 1
#print("ratio from line search: " + str(ratio))
cur_theta = (eta * prev_theta +
w_theta.reshape(-1, )) / (eta + omega)
cur_beta = prev_beta + ratio * w_beta.reshape(-1, ) / eta
else:
for i in range(2):
# Do a line search for both theta and beta parameters by adjusting only eta
eta = eta_search(w_theta, w_beta, eta, omega, allmean,
compute_losses, get_flat,
set_from_flat, pi, epsilon, args)
logger.log("Updated eta, eta: " + str(eta) +
" and omega: " + str(omega))
# Find proper omega for new eta. Use old policy parameters first.
set_from_flat(
pi.theta_beta_to_all(prev_theta, prev_beta))
eta, omega = \
eta_omega_optimizer.optimize(w_theta, Waa, Wsa, wa, varphis, pi.get_kt(),
pi.get_prec_matrix(), pi.is_new_policy_valid, old_ent, eta)
logger.log("Updated omega, eta: " + str(eta) +
" and omega: " + str(omega))
# Use final policy
logger.log("Final eta: " + str(eta) + " and omega: " +
str(omega))
cur_theta = (eta * prev_theta +
w_theta.reshape(-1, )) / (eta + omega)
cur_beta = prev_beta + w_beta.reshape(-1, ) / eta
paramnew = allmean(pi.theta_beta_to_all(cur_theta, cur_beta))
set_from_flat(paramnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses(*args)))
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(paramnew.sum(),
vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
##copos specific over
#cg over
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
#policy update over
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False,
batch_size=64):
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
print("Reward max: " + str(max(rewbuffer)))
print("Reward min: " + str(min(rewbuffer)))
logger.record_tabular(
"EpLenMean",
np.mean(lenbuffer) if np.sum(lenbuffer) != 0.0 else 0.0)
logger.record_tabular(
"EpRewMean",
np.mean(rewbuffer) if np.sum(rewbuffer) != 0.0 else 0.0)
logger.record_tabular(
"AverageReturn",
np.mean(rewbuffer) if np.sum(rewbuffer) != 0.0 else 0.0)
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank == 0:
logger.dump_tabular()
def dynamics_train(index, state, action, next_state, dyn_weights, alpha):
j = index
size_num = np.size(state, axis=0)
dynamics_state_and_action = np.concatenate((state, action), axis=1)
args = dynamics_state_and_action, next_state
fvpargs = [arr[::5] for arr in args]
def MDP_fisher_vector_product(p):
return allmean(MDP_compute_fvp[j](p, *fvpargs)) + cg_damping * p
MDP_assign_old_eq_new[j]()
surr_begin = allmean(np.array(compute_likelihood(dyn_weights, K, *args, size_num)))
surr_begin_l2_norm = allmean(np.array(MDP_compute_l2_norm_sum()))
surr_begin = surr_begin - alpha*surr_begin_l2_norm
g = ensemble_compute_grad(dyn_weights, K, *args, size_num, j)
g_l2_norm = MDP_compute_l2_norm_grad[j]()
g = g - alpha*g_l2_norm
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
stepdir = cg(MDP_fisher_vector_product, g, cg_iters=cg_iters, verbose=rank == 0) # s \simeq A^{-1}g
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(MDP_fisher_vector_product(stepdir)) # shs = 1/2 s^{T}As
lm = np.sqrt(shs / max_kl) # lm = 1 / \beta
fullstep = stepdir / lm # fullstep = \beta s, and then shrink exponentially (10 times)
expectedimprove = g.dot(fullstep)
surrbefore = surr_begin
stepsize = 1.0
phibefore = MDP_get_flat[j]()
for _ in range(10):
phinew = phibefore + fullstep * stepsize
MDP_set_from_flat[j](phinew)
surr = allmean(np.array(compute_likelihood(dyn_weights, K, *args, size_num)))
surr_l2_norm = allmean(np.array(MDP_compute_l2_norm_sum()))
surr = surr - alpha * surr_l2_norm
kl = allmean(np.array(MDP_compute_kl[j](*args)))
meanlosses = surr, kl
improve = surr - surrbefore
if not np.isfinite(meanlosses).all():
pass
elif kl > max_kl * 1.5:
pass
elif improve < 0:
pass
else:
break
stepsize *= .5
else:
MDP_set_from_flat[j](phibefore)
print("Updating the model %i is completed" % (j + 1))
def learn(env,
policy_func,
reward_giver,
expert_dataset,
rank,
pretrained,
pretrained_weight,
*,
g_step,
d_step,
entcoeff,
save_per_iter,
ckpt_dir,
log_dir,
timesteps_per_batch,
task_name,
gamma,
lam,
max_kl,
cg_iters,
cg_damping=1e-2,
vf_stepsize=3e-4,
d_stepsize=3e-4,
vf_iters=3,
max_timesteps=0,
max_episodes=0,
max_iters=0,
vf_batchsize=128,
callback=None,
freeze_g=False,
freeze_d=False,
semi_dataset=None,
semi_loss=False):
semi_loss = semi_loss and semi_dataset is not None
l2_w = 0.1
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
if rank == 0:
writer = U.file_writer(log_dir)
# print all the hyperparameters in the log...
log_dict = {
"expert trajectories": expert_dataset.num_traj,
"algo": "trpo",
"threads": nworkers,
"timesteps_per_batch": timesteps_per_batch,
"timesteps_per_thread": -(-timesteps_per_batch // nworkers),
"entcoeff": entcoeff,
"vf_iters": vf_iters,
"vf_batchsize": vf_batchsize,
"vf_stepsize": vf_stepsize,
"d_stepsize": d_stepsize,
"g_step": g_step,
"d_step": d_step,
"max_kl": max_kl,
"gamma": gamma,
"lam": lam,
"l2_weight": l2_w
}
if semi_dataset is not None:
log_dict["semi trajectories"] = semi_dataset.num_traj
if hasattr(semi_dataset, 'info'):
log_dict["semi_dataset_info"] = semi_dataset.info
# print them all together for csv
logger.log(",".join([str(elem) for elem in log_dict]))
logger.log(",".join([str(elem) for elem in log_dict.values()]))
# also print them separately for easy reading:
for elem in log_dict:
logger.log(str(elem) + ": " + str(log_dict[elem]))
# divide the timesteps to the threads
timesteps_per_batch = -(-timesteps_per_batch // nworkers
) # get ceil of division
# Setup losses and stuff
# ----------------------------------------
if semi_dataset:
ob_space = semi_ob_space(env, semi_size=semi_dataset.semi_size)
else:
ob_space = env.observation_space
ac_space = env.action_space
pi = policy_func("pi",
ob_space=ob_space,
ac_space=ac_space,
reuse=(pretrained_weight is not None))
oldpi = policy_func("oldpi", ob_space=ob_space, ac_space=ac_space)
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ob = U.get_placeholder_cached(name="ob")
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = entcoeff * meanent
vferr = tf.reduce_mean(tf.square(pi.vpred - ret))
ratio = tf.exp(pi.pd.logp(ac) -
oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
vf_losses = [vferr]
vf_loss_names = ["vf_loss"]
dist = meankl
all_var_list = pi.get_trainable_variables()
var_list = [
v for v in all_var_list
if v.name.startswith("pi/pol") or v.name.startswith("pi/logstd")
]
vf_var_list = [v for v in all_var_list if v.name.startswith("pi/vf")]
assert len(var_list) == len(vf_var_list) + 1
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
]) # pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function(
[], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(oldpi.get_variables(), pi.get_variables())
])
compute_losses = U.function([ob, ac, atarg], losses)
compute_vf_losses = U.function([ob, ac, atarg, ret], losses + vf_losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses +
[U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret], vf_losses +
[U.flatgrad(vferr, vf_var_list)])
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
true_rewbuffer = deque(maxlen=40)
success_buffer = deque(maxlen=40)
l2_rewbuffer = deque(
maxlen=40) if semi_loss and semi_dataset is not None else None
total_rewbuffer = deque(
maxlen=40) if semi_loss and semi_dataset is not None else None
assert sum([max_iters > 0, max_timesteps > 0, max_episodes > 0]) == 1
not_update = 1 if not freeze_d else 0 # do not update G before D the first time
# if provide pretrained weight
loaded = False
if not U.load_checkpoint_variables(pretrained_weight):
if U.load_checkpoint_variables(pretrained_weight,
check_prefix=get_il_prefix()):
if rank == 0:
logger.log("loaded checkpoint variables from " +
pretrained_weight)
loaded = True
else:
loaded = True
if loaded:
not_update = 0 if any(
[x.op.name.find("adversary") != -1
for x in U.ALREADY_INITIALIZED]) else 1
if pretrained_weight and pretrained_weight.rfind("iter_") and \
pretrained_weight[pretrained_weight.rfind("iter_") + len("iter_"):].isdigit():
curr_iter = int(
pretrained_weight[pretrained_weight.rfind("iter_") +
len("iter_"):]) + 1
print("loaded checkpoint at iteration: " + str(curr_iter))
iters_so_far = curr_iter
timesteps_so_far = iters_so_far * timesteps_per_batch
d_adam = MpiAdam(reward_giver.get_trainable_variables())
vfadam = MpiAdam(vf_var_list)
U.initialize()
th_init = get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
d_adam.sync()
vfadam.sync()
if rank == 0:
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(
pi,
env,
reward_giver,
timesteps_per_batch,
stochastic=True,
semi_dataset=semi_dataset,
semi_loss=semi_loss) # ADD L2 loss to semi trajectories
g_loss_stats = stats(loss_names + vf_loss_names)
d_loss_stats = stats(reward_giver.loss_name)
ep_names = ["True_rewards", "Rewards", "Episode_length", "Success"]
if semi_loss and semi_dataset is not None:
ep_names.append("L2_loss")
ep_names.append("total_rewards")
ep_stats = stats(ep_names)
if rank == 0:
start_time = time.time()
ch_count = 0
env_type = env.env.env.__class__.__name__
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
# Save model
if rank == 0 and iters_so_far % save_per_iter == 0 and ckpt_dir is not None:
fname = os.path.join(ckpt_dir, task_name)
if env_type.find(
"Pendulum"
) != -1 or save_per_iter != 1: # allow pendulum to save all iterations
fname = os.path.join(ckpt_dir, 'iter_' + str(iters_so_far),
'iter_' + str(iters_so_far))
os.makedirs(os.path.dirname(fname), exist_ok=True)
saver = tf.train.Saver()
saver.save(tf.get_default_session(), fname, write_meta_graph=False)
if rank == 0 and time.time(
) - start_time >= 3600 * ch_count: # save a different checkpoint every hour
fname = os.path.join(ckpt_dir, 'hour' + str(ch_count).zfill(3))
fname = os.path.join(fname, 'iter_' + str(iters_so_far))
os.makedirs(os.path.dirname(fname), exist_ok=True)
saver = tf.train.Saver()
saver.save(tf.get_default_session(), fname, write_meta_graph=False)
ch_count += 1
logger.log("********** Iteration %i ************" % iters_so_far)
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
# ------------------ Update G ------------------
logger.log("Optimizing Policy...")
for curr_step in range(g_step):
with timed("sampling"):
seg = seg_gen.__next__()
seg["rew"] = seg["rew"] - seg["l2_loss"] * l2_w
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret, full_ob = seg["ob"], seg["ac"], seg[
"adv"], seg["tdlamret"], seg["full_ob"]
vpredbefore = seg[
"vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()) / atarg.std(
) # standardized advantage function estimate
d = Dataset(dict(ob=full_ob, ac=ac, atarg=atarg, vtarg=tdlamret),
shuffle=True)
if not_update:
break # stop G from updating
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(full_ob) # update running mean/std for policy
args = seg["full_ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
assign_old_eq_new(
) # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses(*args)))
if rank == 0:
print("Generator entropy " + str(meanlosses[4]) +
", loss " + str(meanlosses[2]))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(),
vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
with timed("vf"):
logger.log(fmt_row(13, vf_loss_names))
for _ in range(vf_iters):
vf_b_losses = []
for batch in d.iterate_once(vf_batchsize):
mbob = batch["ob"]
mbret = batch["vtarg"]
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(
mbob) # update running mean/std for policy
*newlosses, g = compute_vflossandgrad(mbob, mbret)
g = allmean(g)
newlosses = allmean(np.array(newlosses))
vfadam.update(g, vf_stepsize)
vf_b_losses.append(newlosses)
logger.log(fmt_row(13, np.mean(vf_b_losses, axis=0)))
logger.log("Evaluating losses...")
losses = []
for batch in d.iterate_once(vf_batchsize):
newlosses = compute_vf_losses(batch["ob"], batch["ac"],
batch["atarg"], batch["vtarg"])
losses.append(newlosses)
meanlosses, _, _ = mpi_moments(losses, axis=0)
#########################
'''
For evaluation during training.
Can be commented out for faster training...
'''
for ob_batch, ac_batch, full_ob_batch in dataset.iterbatches(
(ob, ac, full_ob),
include_final_partial_batch=False,
batch_size=len(ob)):
ob_expert, ac_expert = expert_dataset.get_next_batch(
len(ob_batch))
exp_rew = 0
for obs, acs in zip(ob_expert, ac_expert):
exp_rew += 1 - np.exp(
-reward_giver.get_reward(obs, acs)[0][0])
mean_exp_rew = exp_rew / len(ob_expert)
gen_rew = 0
for obs, acs, full_obs in zip(ob_batch, ac_batch,
full_ob_batch):
gen_rew += 1 - np.exp(
-reward_giver.get_reward(obs, acs)[0][0])
mean_gen_rew = gen_rew / len(ob_batch)
if rank == 0:
logger.log("Generator step " + str(curr_step) +
": Dicriminator reward of expert traj " +
str(mean_exp_rew) + " vs gen traj " +
str(mean_gen_rew))
#########################
if not not_update:
g_losses = meanlosses
for (lossname, lossval) in zip(loss_names + vf_loss_names,
meanlosses):
logger.record_tabular(lossname, lossval)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
# ------------------ Update D ------------------
if not freeze_d:
logger.log("Optimizing Discriminator...")
batch_size = len(ob) // d_step
d_losses = [
] # list of tuples, each of which gives the loss for a minibatch
for ob_batch, ac_batch, full_ob_batch in dataset.iterbatches(
(ob, ac, full_ob),
include_final_partial_batch=False,
batch_size=batch_size):
ob_expert, ac_expert = expert_dataset.get_next_batch(
len(ob_batch))
#########################
'''
For evaluation during training.
Can be commented out for faster training...
'''
exp_rew = 0
for obs, acs in zip(ob_expert, ac_expert):
exp_rew += 1 - np.exp(
-reward_giver.get_reward(obs, acs)[0][0])
mean_exp_rew = exp_rew / len(ob_expert)
gen_rew = 0
for obs, acs in zip(ob_batch, ac_batch):
gen_rew += 1 - np.exp(
-reward_giver.get_reward(obs, acs)[0][0])
mean_gen_rew = gen_rew / len(ob_batch)
if rank == 0:
logger.log("Dicriminator reward of expert traj " +
str(mean_exp_rew) + " vs gen traj " +
str(mean_gen_rew))
#########################
# update running mean/std for reward_giver
if hasattr(reward_giver, "obs_rms"):
reward_giver.obs_rms.update(
np.concatenate((ob_batch, ob_expert), 0))
loss_input = (ob_batch, ac_batch, ob_expert, ac_expert)
*newlosses, g = reward_giver.lossandgrad(*loss_input)
d_adam.update(allmean(g), d_stepsize)
d_losses.append(newlosses)
logger.log(fmt_row(13, reward_giver.loss_name))
logger.log(fmt_row(13, np.mean(d_losses, axis=0)))
lrlocal = (seg["ep_lens"], seg["ep_rets"], seg["ep_true_rets"],
seg["ep_success"], seg["ep_semi_loss"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews, true_rets, success, semi_losses = map(
flatten_lists, zip(*listoflrpairs))
# success
success = [
float(elem) for elem in success
if isinstance(elem, (int, float, bool))
] # remove potential None types
if not success:
success = [-1] # set success to -1 if env has no success flag
success_buffer.extend(success)
true_rewbuffer.extend(true_rets)
lenbuffer.extend(lens)
rewbuffer.extend(rews)
if semi_loss and semi_dataset is not None:
semi_losses = [elem * l2_w for elem in semi_losses]
total_rewards = rews
total_rewards = [
re_elem - l2_elem
for re_elem, l2_elem in zip(total_rewards, semi_losses)
]
l2_rewbuffer.extend(semi_losses)
total_rewbuffer.extend(total_rewards)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpTrueRewMean", np.mean(true_rewbuffer))
logger.record_tabular("EpSuccess", np.mean(success_buffer))
if semi_loss and semi_dataset is not None:
logger.record_tabular("EpSemiLoss", np.mean(l2_rewbuffer))
logger.record_tabular("EpTotalReward", np.mean(total_rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
logger.record_tabular("ItersSoFar", iters_so_far)
if rank == 0:
logger.dump_tabular()
if not not_update:
g_loss_stats.add_all_summary(writer, g_losses, iters_so_far)
if not freeze_d:
d_loss_stats.add_all_summary(writer, np.mean(d_losses, axis=0),
iters_so_far)
# default buffers
ep_buffers = [
np.mean(true_rewbuffer),
np.mean(rewbuffer),
np.mean(lenbuffer),
np.mean(success_buffer)
]
if semi_loss and semi_dataset is not None:
ep_buffers.append(np.mean(l2_rewbuffer))
ep_buffers.append(np.mean(total_rewbuffer))
ep_stats.add_all_summary(writer, ep_buffers, iters_so_far)
if not_update and not freeze_g:
not_update -= 1
def learn(self):
# Prepare for rollouts
# ----------------------------------------
self.seg_gen = traj_segment_generator(self.pi, self.env, self.timesteps_per_batch, stochastic=True)
return None
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
assert sum([self.max_iters>0, self.max_timesteps>0, self.max_episodes>0])==1
while True:
if self.max_timesteps and timesteps_so_far >= self.max_timesteps:
break
elif self.max_episodes and episodes_so_far >= self.max_episodes:
break
elif self.max_iters and iters_so_far >= self.max_iters:
break
elif self.max_epi_avg and len(lenbuffer)>0 and np.mean(lenbuffer)>= self.max_epi_avg:
break
logger.log("********** Iteration %i ************"%iters_so_far)
with self.timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, self.gamma, self.lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
self.ob, self.ac, self.atarg, self.tdlamret = seg["ob"], seg["ac"], seg["adv"], seg["tdlamret"]
self.vpredbefore = seg["vpred"] # predicted value function before udpate
self.atarg = (self.atarg - self.atarg.mean()) / self.atarg.std() # standardized advantage function estimate
if hasattr(self.pi, "ret_rms"): self.pi.ret_rms.update(self.tdlamret)
if hasattr(self.pi, "ob_rms"): self.pi.ob_rms.update(self.ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], self.atarg
self.fvpargs = [arr[::5] for arr in args]
self.assign_old_eq_new() # set old parameter values to new parameter values
with self.timed("computegrad"):
*lossbefore, g = self.compute_lossandgrad(*args)
lossbefore = self.allmean(np.array(lossbefore))
g = self.allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with self.timed("cg"):
stepdir = cg(self.fisher_vector_product, g, cg_iters=self.cg_iters, verbose=self.rank==0)
assert np.isfinite(stepdir).all()
shs = .5*stepdir.dot(self.fisher_vector_product(stepdir))
lm = np.sqrt(shs / self.max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = self.get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
self.set_from_flat(thnew)
meanlosses = surr, kl, *_ = self.allmean(np.array(self.compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f"%(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > self.max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
self.set_from_flat(thbefore)
if self.nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), self.vfadam.getflat().sum())) # list of tuples
assert all(np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(self.loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with self.timed("vf"):
for _ in range(self.vf_iters):
for (mbob, mbret) in dataset.iterbatches((seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False, batch_size=64):
g = self.allmean(self.compute_vflossandgrad(mbob, mbret))
self.vfadam.update(g, self.vf_stepsize)
logger.record_tabular("ev_tdlam_before", explained_variance(self.vpredbefore, self.tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if self.rank==0:
logger.dump_tabular()
def update(self, obs, actions, atarg, returns, vpredbefore, nb):
# Prepare data
obs = tf.constant(obs)
actions = tf.constant(actions)
atarg = tf.constant(atarg)
returns = tf.constant(returns)
estimates = tf.constant(self.estimates[nb])
multipliers = tf.constant(self.multipliers[nb])
comm = self.comm_matrix[self.comm_matrix[:, nb] != 0][0, self.agent.id]
args, synargs = (obs, actions, atarg), (estimates, multipliers, comm)
# Sampling every 5
fvpargs = [arr[::1] for arr in args]
def hvp(p):
fvp = self.compute_fvp(p, *fvpargs).numpy()
jjvp = self.compute_jjvp(p, *fvpargs).numpy()
return self.allmean(fvp + jjvp) + self.cg_damping * p
self.assign_new_eq_old(
) # set old parameter values to new parameter values
with self.timed("computegrad"):
g = self.compute_vjp(*args, *synargs).numpy()
g = self.allmean(g)
lossbefore = self.allmean(
np.array(self.compute_losses(*args, *synargs)))
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with self.timed("cg"):
stepdir = cg(hvp, g, cg_iters=self.cg_iters)
assert np.isfinite(stepdir).all()
shs = 0.5 * g.dot(stepdir)
# shs = .5*stepdir.dot(fvp(stepdir))
lm = np.sqrt(shs / self.max_kl)
logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
lagrangebefore, surrbefore, syncbefore, klbefore, entbonusbefore, meanentbefore = lossbefore
stepsize = 1.0
thbefore = self.get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
self.set_from_flat(thnew)
meanlosses = lagrange, surr, syncloss, kl, entbonus, meanent = self.allmean(
np.array(self.compute_losses(*args, *synargs)))
improve = lagrangebefore - lagrange
performance_improve = surr - surrbefore
sync_improve = syncbefore - syncloss
print(lagrangebefore, surrbefore, syncbefore, meanentbefore)
print(lagrange, surr, syncloss, meanent)
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > self.max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
self.set_from_flat(thbefore)
# with self.timed("vf"):
for _ in range(self.vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(obs, returns),
include_final_partial_batch=False,
batch_size=64):
vg = self.allmean(
self.compute_vflossandgrad(mbob, mbret).numpy())
self.vfadam.update(vg, self.vf_stepsize)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, returns))
def learn(
*,
network,
env,
save,
total_timesteps,
timesteps_per_batch=1024, # what to train on
max_kl=0.001,
cg_iters=10,
gamma=0.99,
lam=1.0, # advantage estimation
seed=None,
ent_coef=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters=3,
max_episodes=0,
max_iters=0, # ttotal_timestepsime constraint
callback=None,
load_path=None,
**network_kwargs):
'''
learn a policy function with TRPO algorithm
Parameters:
----------
network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
or function that takes input placeholder and returns tuple (output, None) for feedforward nets
or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets
env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class
timesteps_per_batch timesteps per gradient estimation batch
max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )
ent_coef coefficient of policy entropy term in the optimization objective
cg_iters number of iterations of conjugate gradient algorithm
cg_damping conjugate gradient damping
vf_stepsize learning rate for adam optimizer used to optimie value function loss
vf_iters number of iterations of value function optimization iterations per each policy optimization step
total_timesteps max number of timesteps
max_episodes max number of episodes
max_iters maximum number of policy optimization iterations
callback function to be called with (locals(), globals()) each policy optimization step
load_path str, path to load the model from (default: None, i.e. no model is loaded)
**network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
Returns:
-------
learnt model
'''
if MPI is not None:
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
else:
nworkers = 1
rank = 0
set_global_seeds(seed)
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
if isinstance(network, str):
network, network_model = get_network_builder(network)(**network_kwargs)
with tf.name_scope("pi"):
pi_policy_network = network(ob_space.shape)
pi_value_network = network(ob_space.shape)
pi = PolicyWithValue(ac_space, pi_policy_network, pi_value_network)
with tf.name_scope("oldpi"):
old_pi_policy_network = network(ob_space.shape)
old_pi_value_network = network(ob_space.shape)
oldpi = PolicyWithValue(ac_space, old_pi_policy_network,
old_pi_value_network)
pi_var_list = pi_policy_network.trainable_variables + list(
pi.pdtype.trainable_variables)
old_pi_var_list = old_pi_policy_network.trainable_variables + list(
oldpi.pdtype.trainable_variables)
vf_var_list = pi_value_network.trainable_variables + pi.value_fc.trainable_variables
old_vf_var_list = old_pi_value_network.trainable_variables + oldpi.value_fc.trainable_variables
if load_path is not None:
load_path = osp.expanduser(load_path)
ckpt = tf.train.Checkpoint(model=pi)
manager = tf.train.CheckpointManager(ckpt, load_path, max_to_keep=None)
ckpt.restore(manager.latest_checkpoint)
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(pi_var_list)
set_from_flat = U.SetFromFlat(pi_var_list)
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
shapes = [var.get_shape().as_list() for var in pi_var_list]
def assign_old_eq_new():
for pi_var, old_pi_var in zip(pi_var_list, old_pi_var_list):
old_pi_var.assign(pi_var)
for vf_var, old_vf_var in zip(vf_var_list, old_vf_var_list):
old_vf_var.assign(vf_var)
@tf.function
def compute_lossandgrad(ob, ac, atarg):
with tf.GradientTape() as tape:
old_policy_latent = oldpi.policy_network(ob)
old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent)
policy_latent = pi.policy_network(ob)
pd, _ = pi.pdtype.pdfromlatent(policy_latent)
kloldnew = old_pd.kl(pd)
ent = pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
ratio = tf.exp(pd.logp(ac) - old_pd.logp(ac))
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
gradients = tape.gradient(optimgain, pi_var_list)
return losses + [U.flatgrad(gradients, pi_var_list)]
@tf.function
def compute_losses(ob, ac, atarg):
old_policy_latent = oldpi.policy_network(ob)
old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent)
policy_latent = pi.policy_network(ob)
pd, _ = pi.pdtype.pdfromlatent(policy_latent)
kloldnew = old_pd.kl(pd)
ent = pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
ratio = tf.exp(pd.logp(ac) - old_pd.logp(ac))
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
return losses
#ob shape should be [batch_size, ob_dim], merged nenv
#ret shape should be [batch_size]
@tf.function
def compute_vflossandgrad(ob, ret):
with tf.GradientTape() as tape:
pi_vf = pi.value(ob)
vferr = tf.reduce_mean(tf.square(pi_vf - ret))
return U.flatgrad(tape.gradient(vferr, vf_var_list), vf_var_list)
@tf.function
def compute_fvp(flat_tangent, ob, ac, atarg):
with tf.GradientTape() as outter_tape:
with tf.GradientTape() as inner_tape:
old_policy_latent = oldpi.policy_network(ob)
old_pd, _ = oldpi.pdtype.pdfromlatent(old_policy_latent)
policy_latent = pi.policy_network(ob)
pd, _ = pi.pdtype.pdfromlatent(policy_latent)
kloldnew = old_pd.kl(pd)
meankl = tf.reduce_mean(kloldnew)
klgrads = inner_tape.gradient(meankl, pi_var_list)
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(
tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
])
hessians_products = outter_tape.gradient(gvp, pi_var_list)
fvp = U.flatgrad(hessians_products, pi_var_list)
return fvp
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
if MPI is not None:
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
else:
out = np.copy(x)
return out
th_init = get_flat()
if MPI is not None:
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi, env, timesteps_per_batch)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
# ---------------------- New ----------------------
rewforbuffer = deque(maxlen=40)
rewctrlbuffer = deque(maxlen=40)
rewconbuffer = deque(maxlen=40)
rewsurbuffer = deque(maxlen=40)
rewformeanbuf = np.array([])
rewctrlmeanbuf = np.array([])
rewconmeanbuf = np.array([])
rewsurmeanbuf = np.array([])
# -------------------------------------------------
if sum([max_iters > 0, total_timesteps > 0, max_episodes > 0]) == 0:
# nothing to be done
return pi
assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \
'out of max_iters, total_timesteps, and max_episodes only one should be specified'
x_axis = 0
x_holder = np.array([])
rew_holder = np.array([])
while True:
if timesteps_so_far > total_timesteps - 1500: #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Set recording XXXX timesteps before ending
env = VecVideoRecorder(env,
osp.join(logger.get_dir(), "videos"),
record_video_trigger=lambda x: True,
video_length=200)
seg_gen = traj_segment_generator(pi, env, timesteps_per_batch)
if callback: callback(locals(), globals())
if total_timesteps and timesteps_so_far >= total_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************" % iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
ob = sf01(ob)
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()
) / atarg.std() # standardized advantage function estimate
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(ob) # update running mean/std for policy
args = ob, ac, atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs).numpy()) + cg_damping * p
assign_old_eq_new() # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = g.numpy()
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False,
batch_size=64):
mbob = sf01(mbob)
g = allmean(compute_vflossandgrad(mbob, mbret).numpy())
vfadam.update(g, vf_stepsize)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"], seg["ep_rets_for"],
seg["ep_rets_ctrl"], seg["ep_rets_con"], seg["ep_rets_sur"]
) # local values
if MPI is not None:
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
else:
listoflrpairs = [lrlocal]
lens, rews, rews_for, rews_ctrl, rews_con, rews_sur = map(
flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
# ---------------------- New ----------------------
rewforbuffer.extend(rews_for)
rewctrlbuffer.extend(rews_ctrl)
rewconbuffer.extend(rews_con)
rewsurbuffer.extend(rews_sur)
rewformeanbuf = np.append([rewformeanbuf], [np.mean(rewforbuffer)])
rewctrlmeanbuf = np.append([rewctrlmeanbuf], [np.mean(rewctrlbuffer)])
rewconmeanbuf = np.append([rewconmeanbuf], [np.mean(rewconbuffer)])
rewsurmeanbuf = np.append([rewsurmeanbuf], [np.mean(rewsurbuffer)])
# -------------------------------------------------
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank == 0:
logger.dump_tabular()
x_axis += 1
x_holder = np.append([x_holder], [x_axis])
rew_holder = np.append([rew_holder], [np.mean(rewbuffer)])
# --------------------------------------- NEW -----------------------------------------------------
with open("img_rec.txt", "r") as rec:
cur_gen = rec.read()
cur_gen = cur_gen.strip() # remove \n
dir_of_gens = [
'1_1', '2_1', '3_1', '1_2', '2_2', '3_2', '1_3', '2_3', '3_3', '1_4',
'2_4', '3_4', '1_5', '2_5', '3_5', '1_6', '2_6', '3_6', '1_7', '2_7',
'3_7', '1_8', '2_8', '3_8', '1_9', '2_9', '3_9', '1_10', '2_10',
'3_10', '1_11', '2_11', '3_11', '1_12', '2_12', '3_12'
]
# -------------------------------------------------------------------------------------------------
from matplotlib import pyplot as plt
f = plt.figure(1)
plt.plot(x_holder, rew_holder)
plt.title("Rewards for Ant v2")
plt.grid(True)
plt.savefig('rewards_for_antv2_{}'.format(cur_gen))
g = plt.figure(2)
plt.plot(x_holder, rewformeanbuf, label='Forward Reward')
plt.plot(x_holder, rewctrlmeanbuf, label='CTRL Cost')
plt.plot(x_holder, rewconmeanbuf, label='Contact Cost')
plt.plot(x_holder, rewsurmeanbuf, label='Survive Reward')
plt.title("Reward Breakdown")
plt.legend()
plt.grid(True)
plt.savefig('rewards_breakdown{}'.format(cur_gen))
# plt.show()
# --------------------------------------- NEW -----------------------------------------------------
elem = int(dir_of_gens.index(cur_gen))
with open("img_rec.txt", "w") as rec:
if elem == 35:
new_elem = 0
else:
new_elem = elem + 1
new_gen = cur_gen.replace(cur_gen, dir_of_gens[new_elem])
rec.write(new_gen)
# -------------------------------------------------------------------------------------------------
#----------------------------------------------------------- SAVE WEIGHTS ------------------------------------------------------------#
# np.save('val_weights_bias_2_c',val_weights_bias_2_c) # <-------------------------------------------------------------------------------------
# save = save.replace(save[0],'..',2)
# os.chdir(save)
# name = 'max_reward'
# completeName = os.path.join(name+".txt")
# file1 = open(completeName,"w")
# toFile = str(np.mean(rewbuffer))
# file1.write(toFile)
# file1.close()
# os.chdir('../../../baselines-tf2')
return pi
def learn(env, policy_fn, *,
timesteps_per_batch, # what to train on
epsilon, beta, cg_iters,
gamma, lam, # advantage estimation
trial, sess,
method,
entcoeff=0.0,
cg_damping=1e-2,
kl_target=0.01,
crosskl_coeff=0.01,
vf_stepsize=3e-4,
vf_iters =3,
max_timesteps=0, max_episodes=0, max_iters=0, # time constraint
callback=None,
TRPO=False
):
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
total_space = env.total_space
ob_space = env.observation_space
ac_space = env.action_space
pi = policy_fn("pi", ob_space, ac_space, ob_name="ob")
oldpi = policy_fn("oldpi", ob_space, ac_space, ob_name="ob")
gpi = policy_fn("gpi", total_space, ac_space, ob_name="gob")
goldpi = policy_fn("goldpi", total_space, ac_space, ob_name="gob")
atarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
gatarg = tf.placeholder(dtype=tf.float32, shape=[None])
gret = tf.placeholder(dtype=tf.float32, shape=[None])
ob = U.get_placeholder_cached(name="ob")
gob = U.get_placeholder_cached(name='gob')
ac = pi.pdtype.sample_placeholder([None])
crosskl_c = tf.placeholder(dtype=tf.float32, shape=[])
# crosskl_c = 0.01
kloldnew = oldpi.pd.kl(pi.pd)
gkloldnew = goldpi.pd.kl(gpi.pd)
#TODO: check if it can work in this way
# crosskl_ob = pi.pd.kl(goldpi.pd)
# crosskl_gob = gpi.pd.kl(oldpi.pd)
crosskl_gob = pi.pd.kl(gpi.pd)
crosskl_ob = gpi.pd.kl(pi.pd)
# crosskl
pdmean = pi.pd.mean
pdstd = pi.pd.std
gpdmean = gpi.pd.mean
gpdstd = gpi.pd.std
ent = pi.pd.entropy()
gent = gpi.pd.entropy()
old_entropy = oldpi.pd.entropy()
gold_entropy = goldpi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
meancrosskl = tf.reduce_mean(crosskl_ob)
# meancrosskl = tf.maximum(tf.reduce_mean(crosskl_ob - 100), 0)
gmeankl = tf.reduce_mean(gkloldnew)
gmeanent = tf.reduce_mean(gent)
gmeancrosskl = tf.reduce_mean(crosskl_gob)
vferr = tf.reduce_mean(tf.square(pi.vpred - ret))
gvferr = tf.reduce_mean(tf.square(gpi.vpred - gret))
ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) # advantage * pnew / pold
gratio = tf.exp(gpi.pd.logp(ac) - goldpi.pd.logp(ac))
# Ratio objective
# surrgain = tf.reduce_mean(ratio * atarg)
# gsurrgain = tf.reduce_mean(gratio * gatarg)
# Log objective
surrgain = tf.reduce_mean(pi.pd.logp(ac) * atarg)
gsurrgain = tf.reduce_mean(gpi.pd.logp(ac) * gatarg)
# optimgain = surrgain + crosskl_c * meancrosskl
optimgain = surrgain
losses = [optimgain, meankl, meancrosskl, surrgain, meanent, tf.reduce_mean(ratio)]
loss_names = ["optimgain", "meankl", "meancrosskl", "surrgain", "entropy", "ratio"]
# goptimgain = gsurrgain + crosskl_c * gmeancrosskl
goptimgain = gsurrgain
glosses = [goptimgain, gmeankl, gmeancrosskl, gsurrgain, gmeanent, tf.reduce_mean(gratio)]
gloss_names = ["goptimgain", "gmeankl","gmeancrosskl", "gsurrgain", "gentropy", "gratio"]
dist = meankl
gdist = gmeankl
all_pi_var_list = pi.get_trainable_variables()
all_var_list = [v for v in all_pi_var_list if v.name.split("/")[0].startswith("pi")]
var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("pol")]
vf_var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("vf")]
vfadam = MpiAdam(vf_var_list)
poladam = MpiAdam(var_list)
gall_gpi_var_list = gpi.get_trainable_variables()
gall_var_list = [v for v in gall_gpi_var_list if v.name.split("/")[0].startswith("gpi")]
gvar_list = [v for v in gall_var_list if v.name.split("/")[1].startswith("pol")]
gvf_var_list = [v for v in gall_var_list if v.name.split("/")[1].startswith("vf")]
gvfadam = MpiAdam(gvf_var_list)
# gpoladpam = MpiAdam(gvar_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
# crossklgrads = tf.gradients(meancrosskl, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))
start += sz
gvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
gget_flat = U.GetFlat(gvar_list)
gset_from_flat = U.SetFromFlat(gvar_list)
gklgrads = tf.gradients(gdist, gvar_list)
# gcrossklgrads = tf.gradients(gmeancrosskl, gvar_list)
gflat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="gflat_tan")
gshapes = [var.get_shape().as_list() for var in gvar_list]
gstart = 0
gtangents = []
for shape in gshapes:
sz = U.intprod(shape)
gtangents.append(tf.reshape(gflat_tangent[gstart:gstart+sz], shape))
gstart += sz
ggvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(gklgrads, gtangents)]) #pylint: disable=E1111
gfvp = U.flatgrad(ggvp, gvar_list)
assign_old_eq_new = U.function([],[], updates=[tf.assign(oldv, newv)
for (oldv, newv) in zipsame(oldpi.get_variables(), pi.get_variables())])
gassign_old_eq_new = U.function([], [], updates=[tf.assign(oldv, newv)
for (oldv, newv) in zipsame(goldpi.get_variables(), gpi.get_variables())])
compute_losses = U.function([crosskl_c, gob, ob, ac, atarg], losses)
compute_lossandgrad = U.function([crosskl_c, gob, ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))
compute_crossklandgrad = U.function([ob, gob],U.flatgrad(meancrosskl, var_list))
gcompute_losses = U.function([crosskl_c, ob, gob, ac, gatarg], glosses)
gcompute_lossandgrad = U.function([crosskl_c, ob, gob, ac, gatarg], glosses + [U.flatgrad(goptimgain, gvar_list)])
gcompute_fvp = U.function([gflat_tangent, gob, ac, gatarg], gfvp)
gcompute_vflossandgrad = U.function([gob, gret], U.flatgrad(gvferr, gvf_var_list))
# compute_gcrossklandgrad = U.function([gob, ob], U.flatgrad(gmeancrosskl, gvar_list))
saver = tf.train.Saver()
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(colorize("done in %.3f seconds"%(time.time() - tstart), color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
U.initialize()
guided_initilizer(gpol=gvar_list, gvf=gvf_var_list, fpol=var_list, fvf=vf_var_list)
th_init = get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
poladam.sync()
print("Init final policy param sum", th_init.sum(), flush=True)
gth_init = gget_flat()
MPI.COMM_WORLD.Bcast(gth_init, root=0)
gset_from_flat(gth_init)
gvfadam.sync()
# gpoladpam.sync()
print("Init guided policy param sum", gth_init.sum(), flush=True)
# Initialize eta, omega optimizer
init_eta = 0.5
init_omega = 2.0
eta_omega_optimizer = EtaOmegaOptimizer(beta, epsilon, init_eta, init_omega)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi, gpi, env, timesteps_per_batch, stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
num_iters = max_timesteps // timesteps_per_batch
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
assert sum([max_iters>0, max_timesteps>0, max_episodes>0])==1
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************"%iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg["tdlamret"]
gob, gatarg, gtdlamret = seg["gob"], seg["gadv"], seg["gtdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
gvpredbefore = seg["gvpred"]
atarg = (atarg - atarg.mean()) / atarg.std() # standardized advantage function estimate
gatarg = (gatarg - gatarg.mean()) / gatarg.std()
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy
if hasattr(gpi, "ret_rms"): gpi.ret_rms.update(gtdlamret)
if hasattr(gpi, "ob_rms"): gpi.ob_rms.update(gob)
args = crosskl_coeff, seg["gob"], seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args[2:]]
gargs = crosskl_coeff, seg["ob"], seg["gob"], seg["ac"], gatarg
gfvpargs = [arr[::5] for arr in gargs[2:]]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
def gfisher_vector_product(p):
return allmean(gcompute_fvp(p, *gfvpargs)) + cg_damping * p
assign_old_eq_new() # set old parameter values to new parameter values
gassign_old_eq_new()
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
*glossbefore, gg = gcompute_lossandgrad(*gargs)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
glossbefore = allmean(np.array(glossbefore))
gg = allmean(gg)
if np.allclose(g, 0) or np.allclose(gg, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank==0)
gstepdir = cg(gfisher_vector_product, gg, cg_iters=cg_iters, verbose=rank==0)
assert np.isfinite(gstepdir).all()
assert np.isfinite(stepdir).all()
if TRPO:
#
# TRPO specific code.
# Find correct step size using line search
#
#TODO: also enable guided learning for TRPO
shs = .5*stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / epsilon)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f"%(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > epsilon * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
else:
#
# COPOS specific implementation.
#
copos_update_dir = stepdir
gcopos_update_dir = gstepdir
# Split direction into log-linear 'w_theta' and non-linear 'w_beta' parts
w_theta, w_beta = pi.split_w(copos_update_dir)
gw_theta, gw_beta = gpi.split_w(gcopos_update_dir)
# q_beta(s,a) = \grad_beta \log \pi(a|s) * w_beta
# = features_beta(s) * K^T * Prec * a
# q_beta = self.target.get_q_beta(features_beta, actions)
Waa, Wsa = pi.w2W(w_theta)
wa = pi.get_wa(ob, w_beta)
gWaa, gWsa = gpi.w2W(gw_theta)
gwa = gpi.get_wa(gob, gw_beta)
varphis = pi.get_varphis(ob)
gvarphis = gpi.get_varphis(gob)
# Optimize eta and omega
tmp_ob = np.zeros((1,) + ob_space.shape) # We assume that entropy does not depend on the NN
old_ent = old_entropy.eval({oldpi.ob: tmp_ob})[0]
eta, omega = eta_omega_optimizer.optimize(w_theta, Waa, Wsa, wa, varphis, pi.get_kt(),
pi.get_prec_matrix(), pi.is_new_policy_valid, old_ent)
logger.log("Initial eta of final policy: " + str(eta) + " and omega: " + str(omega))
gtmp_ob = np.zeros((1,) + total_space.shape)
gold_ent = gold_entropy.eval({goldpi.ob: gtmp_ob})[0]
geta, gomega = eta_omega_optimizer.optimize(gw_theta, gWaa, gWsa, gwa, gvarphis, gpi.get_kt(),
gpi.get_prec_matrix(), gpi.is_new_policy_valid, gold_ent)
logger.log("Initial eta of guided policy: " + str(geta) + " and omega: " + str(gomega))
current_theta_beta = get_flat()
prev_theta, prev_beta = pi.all_to_theta_beta(current_theta_beta)
gcurrent_theta_beta = gget_flat()
gprev_theta, gprev_beta = gpi.all_to_theta_beta(gcurrent_theta_beta)
for i in range(2):
# Do a line search for both theta and beta parameters by adjusting only eta
eta = eta_search(w_theta, w_beta, eta, omega, allmean, compute_losses, get_flat, set_from_flat, pi,
epsilon, args)
logger.log("Updated eta of final policy, eta: " + str(eta) + " and omega: " + str(omega))
# Find proper omega for new eta. Use old policy parameters first.
set_from_flat(pi.theta_beta_to_all(prev_theta, prev_beta))
eta, omega = \
eta_omega_optimizer.optimize(w_theta, Waa, Wsa, wa, varphis, pi.get_kt(),
pi.get_prec_matrix(), pi.is_new_policy_valid, old_ent, eta)
logger.log("Updated omega of final policy, eta: " + str(eta) + " and omega: " + str(omega))
geta = eta_search(gw_theta, gw_beta, geta, gomega, allmean, gcompute_losses, gget_flat,
gset_from_flat, gpi, epsilon, gargs)
logger.log("updated eta of guided policy, eta:" + str(geta) + "and omega:" + str(gomega))
gset_from_flat(gpi.theta_beta_to_all(gprev_theta, gprev_beta))
geta, gomega = eta_omega_optimizer.optimize(gw_theta, gWaa, gWsa, gwa, gvarphis,
gpi.get_kt(), gpi.get_prec_matrix(), gpi.is_new_policy_valid, gold_ent, geta)
logger.log("Updated omega of guided policy, eta:" + str(geta) + "and omega:" + str(gomega))
# Use final policy
logger.log("Final eta of final policy: " + str(eta) + " and omega: " + str(omega))
logger.log("Final eta of guided policy: " + str(geta) + "and omega:" + str(gomega))
cur_theta = (eta * prev_theta + w_theta.reshape(-1, )) / (eta + omega)
cur_beta = prev_beta + w_beta.reshape(-1, ) / eta
set_from_flat(pi.theta_beta_to_all(cur_theta, cur_beta))
gcur_theta = (geta * gprev_theta + gw_theta.reshape(-1, )) / (geta + gomega)
gcur_beta = gprev_beta + gw_beta.reshape(-1, ) / geta
gset_from_flat(gpi.theta_beta_to_all(gcur_theta, gcur_beta))
meanlosses = surr, kl, crosskl, *_ = allmean(np.array(compute_losses(*args)))
gmeanlosses = gsurr, gkl, gcrosskl, *_ = allmean(np.array(gcompute_losses(*gargs)))
# poladam.update(allmean(compute_crossklandgrad(ob, gob)), vf_stepsize)
# gpoladpam.update(allmean(compute_gcrossklandgrad(gob, ob)), vf_stepsize)
for _ in range(vf_iters):
for (mbob, mbgob) in dataset.iterbatches((seg["ob"], seg["gob"]),
include_final_partial_batch=False, batch_size=64):
g = allmean(compute_crossklandgrad(mbob, mbgob))
poladam.update(g, vf_stepsize)
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
for (lossname, lossval) in zip(gloss_names, gmeanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches((seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False, batch_size=64):
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
for (mbob, mbret) in dataset.iterbatches((seg["gob"], seg["gtdlamret"]),
include_final_partial_batch=False, batch_size=64):
gg = allmean(gcompute_vflossandgrad(mbob, mbret))
gvfadam.update(gg, vf_stepsize)
logger.record_tabular("ev_tdlam_before", explained_variance(vpredbefore, tdlamret))
logger.record_tabular("gev_tdlam_before", explained_variance(gvpredbefore, gtdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
logger.record_tabular("CrossKLCoeff :", crosskl_coeff)
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
logger.record_tabular("Name", method)
logger.record_tabular("Iteration", iters_so_far)
logger.record_tabular("trial", trial)
if rank==0:
logger.dump_tabular()
if iters_so_far % 100 == 0 or iters_so_far == 1 or iters_so_far == num_iters:
# sess = tf.get_default_session()
checkdir = get_dir(osp.join(logger.get_dir(), 'checkpoints'))
savepath = osp.join(checkdir, '%.5i.ckpt'%iters_so_far)
saver.save(sess, save_path=savepath)
print("save model to path:", savepath)
def learn(*,
network,
env,
total_timesteps,
timesteps_per_batch=1024, # what to train on
max_kl=0.001,
cg_iters=10,
gamma=0.99,
lam=1.0, # advantage estimation
seed=None,
ent_coef=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters =3,
max_episodes=0, max_iters=0, # time constraint
callback=None,
load_path=None,
**network_kwargs
):
'''
learn a policy function with TRPO algorithm
Parameters:
----------
network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
or function that takes input placeholder and returns tuple (output, None) for feedforward nets
or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets
env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class
timesteps_per_batch timesteps per gradient estimation batch
max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )
ent_coef coefficient of policy entropy term in the optimization objective
cg_iters number of iterations of conjugate gradient algorithm
cg_damping conjugate gradient damping
vf_stepsize learning rate for adam optimizer used to optimie value function loss
vf_iters number of iterations of value function optimization iterations per each policy optimization step
total_timesteps max number of timesteps
max_episodes max number of episodes
max_iters maximum number of policy optimization iterations
callback function to be called with (locals(), globals()) each policy optimization step
load_path str, path to load the model from (default: None, i.e. no model is loaded)
**network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
Returns:
-------
learnt model
'''
if MPI is not None:
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
else:
nworkers = 1
rank = 0
cpus_per_worker = 1
U.get_session(config=tf.ConfigProto(
allow_soft_placement=True,
inter_op_parallelism_threads=cpus_per_worker,
intra_op_parallelism_threads=cpus_per_worker
))
policy = build_policy(env, network, value_network='copy', **network_kwargs)
set_global_seeds(seed)
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
ob = observation_placeholder(ob_space)
with tf.variable_scope("pi"):
pi = policy(observ_placeholder=ob)
with tf.variable_scope("oldpi"):
oldpi = policy(observ_placeholder=ob)
atarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
vferr = tf.reduce_mean(tf.square(pi.vf - ret))
ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = get_trainable_variables("pi")
# var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("pol")]
# vf_var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("vf")]
var_list = get_pi_trainable_variables("pi")
vf_var_list = get_vf_trainable_variables("pi")
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))
start += sz
gvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function([],[], updates=[tf.assign(oldv, newv)
for (oldv, newv) in zipsame(get_variables("oldpi"), get_variables("pi"))])
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(colorize("done in %.3f seconds"%(time.time() - tstart), color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
if MPI is not None:
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
else:
out = np.copy(x)
return out
U.initialize()
if load_path is not None:
pi.load(load_path)
th_init = get_flat()
if MPI is not None:
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi, env, timesteps_per_batch, stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
if sum([max_iters>0, total_timesteps>0, max_episodes>0])==0:
# noththing to be done
return pi
assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \
'out of max_iters, total_timesteps, and max_episodes only one should be specified'
while True:
if callback: callback(locals(), globals())
if total_timesteps and timesteps_so_far >= total_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************"%iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg["tdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()) / atarg.std() # standardized advantage function estimate
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
assign_old_eq_new() # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank==0)
assert np.isfinite(stepdir).all()
shs = .5*stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f"%(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches((seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False, batch_size=64):
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
logger.record_tabular("ev_tdlam_before", explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
if MPI is not None:
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
else:
listoflrpairs = [lrlocal]
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank==0:
logger.dump_tabular()
return pi
def evaluate_natural_gradient(g):
return cg(evaluate_fisher_vector_prod,
g,
cg_iters=10,
verbose=0)
def learn(env, last_ob, last_jpos, run_reach, policy_func, reward_giver, expert_dataset, rank,
pretrained, pretrained_weight, *,
g_step, d_step, entcoeff, save_per_iter,
ckpt_dir, log_dir, timesteps_per_batch, task_name,
gamma, lam,
max_kl, cg_iters, cg_damping=1e-2,
vf_stepsize=3e-4, d_stepsize=3e-4, vf_iters=3,
max_timesteps=0, max_episodes=0, max_iters=0,
callback=None
):
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
pi = policy_func("pi_grasp", ob_space, ac_space, reuse=(pretrained_weight != None))
oldpi = policy_func("oldpi", ob_space, ac_space)
atarg = tf.compat.v1.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)
ret = tf.compat.v1.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ob = U.get_placeholder_cached(name="ob")
ac = pi.pdtype.sample_placeholder([None])
# Changes are made in order to use tensorboard
# -------------------------------------------
#train_writer = tf.compat.v1.summary.FileWriter('../../logs/trpo_mpi') # sets log dir to GailPart folder
#sess = tf.compat.v1.Session() # create a session??
# -------------------------------------------
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = entcoeff * meanent
vferr = tf.reduce_mean(tf.square(pi.vpred - ret))
ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = pi.get_trainable_variables()
var_list = [v for v in all_var_list if v.name.startswith("pi_grasp/pol") or v.name.startswith("pi_grasp/logstd")]
vf_var_list = [v for v in all_var_list if v.name.startswith("pi_grasp/vff")]
assert len(var_list) == len(vf_var_list) + 1
d_adam = MpiAdam(reward_giver.get_trainable_variables())
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.compat.v1.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))
start += sz
gvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) # pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function([], [], updates=[tf.compat.v1.assign(oldv, newv)
for (oldv, newv) in zipsame(oldpi.get_variables(), pi.get_variables())])
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(colorize("done in %.3f seconds" % (time.time() - tstart), color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
U.initialize()
th_init = get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
d_adam.sync()
vfadam.sync()
if rank == 0:
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi, last_ob, last_jpos, run_reach, policy_func, env, reward_giver, timesteps_per_batch, stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
true_rewbuffer = deque(maxlen=40)
assert sum([max_iters > 0, max_timesteps > 0, max_episodes > 0]) == 1
g_loss_stats = stats(loss_names)
d_loss_stats = stats(reward_giver.loss_name)
ep_stats = stats(["True_rewards", "Rewards", "Episode_length"])
# if provide pretrained weight
if pretrained_weight is not None:
U.load_state(pretrained_weight, var_list=pi.get_variables())
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
# Save model
if rank == 0 and iters_so_far % save_per_iter == 0 and ckpt_dir is not None:
t_name = task_name + "_" + str(iters_so_far)
fname = os.path.join(ckpt_dir, t_name) # changed from task_name
os.makedirs(os.path.dirname(fname), exist_ok=True)
saver = tf.compat.v1.train.Saver()
saver.save(tf.compat.v1.get_default_session(), fname)
logger.log("********** Iteration %i ************" % iters_so_far)
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
# ------------------ Update G ------------------
logger.log("Optimizing Policy...")
for _ in range(g_step):
with timed("sampling"):
seg = seg_gen.__next__()
#print("trpo_mpi, seg = seg_gen.__next__() call output: ", seg )
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg["tdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()) / atarg.std() # standardized advantage function estimate
if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
assign_old_eq_new() # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank == 0)
assert np.isfinite(stepdir).all()
shs = .5*stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))
#logger.log("trpo_mpi.py, what should be logged with loss names ie. meanlosses:_", meanlosses)
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" % (expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches((seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False, batch_size=128):
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(mbob) # update running mean/std for policy
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
g_losses = meanlosses
#logger.log("trpo_mpi.py, mean losses before logging wiht loss names: \n")
#logger.log(meanlosses)
# This is where the nan values are tabulated for some of the entries
#logger.log("trpo_mpi.py, view whats being printed with (loss_names, lossvalues)")
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
logger.record_tabular("ev_tdlam_before", explained_variance(vpredbefore, tdlamret))
# ------------------ Update D ------------------
logger.log("Optimizing Discriminator...")
logger.log(fmt_row(13, reward_giver.loss_name))
ob_expert, ac_expert = expert_dataset.get_next_batch(len(ob))
batch_size = len(ob) // d_step
d_losses = [] # list of tuples, each of which gives the loss for a minibatch
for ob_batch, ac_batch in dataset.iterbatches((ob, ac),
include_final_partial_batch=False,
batch_size=batch_size):
ob_expert, ac_expert = expert_dataset.get_next_batch(len(ob_batch))
# update running mean/std for reward_giver
if hasattr(reward_giver, "obs_rms"): reward_giver.obs_rms.update(np.concatenate((ob_batch, ob_expert), 0))
*newlosses, g = reward_giver.lossandgrad(ob_batch, ac_batch, ob_expert, ac_expert)
d_adam.update(allmean(g), d_stepsize)
d_losses.append(newlosses)
# This is to see what the d_losses are
#logger.log("trpo_mpi.py, see what is being logged in d_losses")
#logger.log("trpo_mpi.py, d_losses")
#logger.log(d_losses)
logger.log(fmt_row(13, np.mean(d_losses, axis=0)))
# For Tensorboard Logging
# ---------------------------
#tf.compat.v1.summary.scalar("Generator Accuracy", tf.convert_to_tensor( np.mean(d_losses, axis=0)[4] ) ) # 5 position
#tf.compat.v1.summary.scalar("Expert Accuracy", tf.convert_to_tensor( np.mean(d_losses, axis=0)[5] ) ) # 6 position
#tf.compat.v1.summary.scalar("Entropy Loss", tf.convert_to_tensor( np.mean(d_losses, axis=0)[3] ) ) # 4 position
#merge = tf.compat.v1.summary.merge_all() # merge summaries
#summary = sess.run([merge])
#train_writer.add_summary(summary, iters_so_far)
# Is there a need to reset metric after every epoch? I dont think so?
# ---------------------------
#logger.log("trpo_mpi.py, after logging, but before recordeing timesteps so far")
lrlocal = (seg["ep_lens"], seg["ep_rets"], seg["ep_true_rets"]) # local values, truly confirmed is empty after call
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews, true_rets = map(flatten_lists, zip(*listoflrpairs))
true_rewbuffer.extend(true_rets)
lenbuffer.extend(lens)
rewbuffer.extend(rews)
# Could it be that the seg locals for lens and rets are ommitted since has no use in gail algorithm?
# Probably dont have to worry about it, check the scalar part
logger.record_tabular("EpLenMean", np.mean(lenbuffer)) # This has nan values
logger.record_tabular("EpRewMean", np.mean(rewbuffer)) # This has nan values
logger.record_tabular("EpTrueRewMean", np.mean(true_rewbuffer)) # This has nan values
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
#timesteps_so_far += sum(lens)
timesteps_so_far += seg["steps"] # changed to match setup with no finishing condition
iters_so_far += 1
#env.reset() #reset the environment after a new iteration, therefore in traj generator check ob
logger.record_tabular("EpisodesSoFar", episodes_so_far) # This is 0 ? if lens which is the number of entries for episode length doesnt exist, doesnt make sense for it to have a return.
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
# I think the entloss, entrpoy, ev_.... and the useful ones arent from the environment called using the trpo
if rank == 0:
logger.dump_tabular()
def learn(env, policy_func, *,
timesteps_per_batch, # what to train on
max_kl, cg_iters,
gamma, lam, # advantage estimation
entcoeff=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters =3,
max_timesteps=0, max_episodes=0, max_iters=0, # time constraint
callback=None
):
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
pi = policy_func("pi", ob_space, ac_space)
oldpi = policy_func("oldpi", ob_space, ac_space)
atarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ob = U.get_placeholder_cached(name="ob")
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = U.mean(kloldnew)
meanent = U.mean(ent)
entbonus = entcoeff * meanent
vferr = U.mean(tf.square(pi.vpred - ret))
ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = U.mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = pi.get_trainable_variables()
var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("pol")]
vf_var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("vf")]
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))
start += sz
gvp = tf.add_n([U.sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function([],[], updates=[tf.assign(oldv, newv)
for (oldv, newv) in zipsame(oldpi.get_variables(), pi.get_variables())])
compute_losses = U.function([ob, ac, atarg], losses)
compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])
compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(colorize("done in %.3f seconds"%(time.time() - tstart), color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
U.initialize()
th_init = get_flat()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi, env, timesteps_per_batch, stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
assert sum([max_iters>0, max_timesteps>0, max_episodes>0])==1
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************"%iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg["tdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()) / atarg.std() # standardized advantage function estimate
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy
args = seg["ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
assign_old_eq_new() # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank==0)
assert np.isfinite(stepdir).all()
shs = .5*stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f"%(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches((seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False, batch_size=64):
g = allmean(compute_vflossandgrad(mbob, mbret))
vfadam.update(g, vf_stepsize)
logger.record_tabular("ev_tdlam_before", explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank==0:
logger.dump_tabular()
def learn(
env,
policy_func,
reward_giver,
semi_dataset,
rank,
pretrained_weight,
*,
g_step,
d_step,
entcoeff,
save_per_iter,
ckpt_dir,
log_dir,
timesteps_per_batch,
task_name,
gamma,
lam,
max_kl,
cg_iters,
cg_damping=1e-2,
vf_stepsize=3e-4,
d_stepsize=3e-4,
vf_iters=3,
max_timesteps=0,
max_episodes=0,
max_iters=0,
vf_batchsize=128,
callback=None,
freeze_g=False,
freeze_d=False,
pretrained_il=None,
pretrained_semi=None,
semi_loss=False,
expert_reward_threshold=None, # filter experts based on reward
expert_label=get_semi_prefix(),
sparse_reward=False # filter experts based on success flag (sparse reward)
):
semi_loss = semi_loss and semi_dataset is not None
l2_w = 0.1
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
np.set_printoptions(precision=3)
if rank == 0:
writer = U.file_writer(log_dir)
# print all the hyperparameters in the log...
log_dict = {
# "expert trajectories": expert_dataset.num_traj,
"expert model": pretrained_semi,
"algo": "trpo",
"threads": nworkers,
"timesteps_per_batch": timesteps_per_batch,
"timesteps_per_thread": -(-timesteps_per_batch // nworkers),
"entcoeff": entcoeff,
"vf_iters": vf_iters,
"vf_batchsize": vf_batchsize,
"vf_stepsize": vf_stepsize,
"d_stepsize": d_stepsize,
"g_step": g_step,
"d_step": d_step,
"max_kl": max_kl,
"gamma": gamma,
"lam": lam,
}
if semi_dataset is not None:
log_dict["semi trajectories"] = semi_dataset.num_traj
if hasattr(semi_dataset, 'info'):
log_dict["semi_dataset_info"] = semi_dataset.info
if expert_reward_threshold is not None:
log_dict["expert reward threshold"] = expert_reward_threshold
log_dict["sparse reward"] = sparse_reward
# print them all together for csv
logger.log(",".join([str(elem) for elem in log_dict]))
logger.log(",".join([str(elem) for elem in log_dict.values()]))
# also print them separately for easy reading:
for elem in log_dict:
logger.log(str(elem) + ": " + str(log_dict[elem]))
# divide the timesteps to the threads
timesteps_per_batch = -(-timesteps_per_batch // nworkers
) # get ceil of division
# Setup losses and stuff
# ----------------------------------------
ob_space = OrderedDict([(label, env[label].observation_space)
for label in env])
if semi_dataset and get_semi_prefix() in env: # semi ob space is different
semi_obs_space = semi_ob_space(env[get_semi_prefix()],
semi_size=semi_dataset.semi_size)
ob_space[get_semi_prefix()] = semi_obs_space
else:
print("no semi dataset")
# raise RuntimeError
vf_stepsize = {label: vf_stepsize for label in env}
ac_space = {label: env[label].action_space for label in ob_space}
pi = {
label: policy_func("pi",
ob_space=ob_space[label],
ac_space=ac_space[label],
prefix=label)
for label in ob_space
}
oldpi = {
label: policy_func("oldpi",
ob_space=ob_space[label],
ac_space=ac_space[label],
prefix=label)
for label in ob_space
}
atarg = {
label: tf.placeholder(dtype=tf.float32, shape=[None])
for label in ob_space
} # Target advantage function (if applicable)
ret = {
label: tf.placeholder(dtype=tf.float32, shape=[None])
for label in ob_space
} # Empirical return
ob = {
label: U.get_placeholder_cached(name=label + "ob")
for label in ob_space
}
ac = {
label: pi[label].pdtype.sample_placeholder([None])
for label in ob_space
}
kloldnew = {label: oldpi[label].pd.kl(pi[label].pd) for label in ob_space}
ent = {label: pi[label].pd.entropy() for label in ob_space}
meankl = {label: tf.reduce_mean(kloldnew[label]) for label in ob_space}
meanent = {label: tf.reduce_mean(ent[label]) for label in ob_space}
entbonus = {label: entcoeff * meanent[label] for label in ob_space}
vferr = {
label: tf.reduce_mean(tf.square(pi[label].vpred - ret[label]))
for label in ob_space
}
ratio = {
label:
tf.exp(pi[label].pd.logp(ac[label]) - oldpi[label].pd.logp(ac[label]))
for label in ob_space
} # advantage * pnew / pold
surrgain = {
label: tf.reduce_mean(ratio[label] * atarg[label])
for label in ob_space
}
optimgain = {
label: surrgain[label] + entbonus[label]
for label in ob_space
}
losses = {
label: [
optimgain[label], meankl[label], entbonus[label], surrgain[label],
meanent[label]
]
for label in ob_space
}
loss_names = {
label: [
label + name for name in
["optimgain", "meankl", "entloss", "surrgain", "entropy"]
]
for label in ob_space
}
vf_losses = {label: [vferr[label]] for label in ob_space}
vf_loss_names = {label: [label + "vf_loss"] for label in ob_space}
dist = {label: meankl[label] for label in ob_space}
all_var_list = {
label: pi[label].get_trainable_variables()
for label in ob_space
}
var_list = {
label: [
v for v in all_var_list[label]
if "pol" in v.name or "logstd" in v.name
]
for label in ob_space
}
vf_var_list = {
label: [v for v in all_var_list[label] if "vf" in v.name]
for label in ob_space
}
for label in ob_space:
assert len(var_list[label]) == len(vf_var_list[label]) + 1
get_flat = {label: U.GetFlat(var_list[label]) for label in ob_space}
set_from_flat = {
label: U.SetFromFlat(var_list[label])
for label in ob_space
}
klgrads = {
label: tf.gradients(dist[label], var_list[label])
for label in ob_space
}
flat_tangent = {
label: tf.placeholder(dtype=tf.float32,
shape=[None],
name=label + "flat_tan")
for label in ob_space
}
fvp = {}
for label in ob_space:
shapes = [var.get_shape().as_list() for var in var_list[label]]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(
tf.reshape(flat_tangent[label][start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads[label], tangents)
]) # pylint: disable=E1111
fvp[label] = U.flatgrad(gvp, var_list[label])
assign_old_eq_new = {
label:
U.function([], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(oldpi[label].get_variables(),
pi[label].get_variables())
])
for label in ob_space
}
compute_losses = {
label: U.function([ob[label], ac[label], atarg[label]], losses[label])
for label in ob_space
}
compute_vf_losses = {
label: U.function([ob[label], ac[label], atarg[label], ret[label]],
losses[label] + vf_losses[label])
for label in ob_space
}
compute_lossandgrad = {
label: U.function([ob[label], ac[label], atarg[label]], losses[label] +
[U.flatgrad(optimgain[label], var_list[label])])
for label in ob_space
}
compute_fvp = {
label:
U.function([flat_tangent[label], ob[label], ac[label], atarg[label]],
fvp[label])
for label in ob_space
}
compute_vflossandgrad = {
label: U.function([ob[label], ret[label]], vf_losses[label] +
[U.flatgrad(vferr[label], vf_var_list[label])])
for label in ob_space
}
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
return out
episodes_so_far = {label: 0 for label in ob_space}
timesteps_so_far = {label: 0 for label in ob_space}
iters_so_far = 0
tstart = time.time()
lenbuffer = {label: deque(maxlen=40)
for label in ob_space} # rolling buffer for episode lengths
rewbuffer = {label: deque(maxlen=40)
for label in ob_space} # rolling buffer for episode rewards
true_rewbuffer = {label: deque(maxlen=40) for label in ob_space}
success_buffer = {label: deque(maxlen=40) for label in ob_space}
# L2 only for semi network
l2_rewbuffer = deque(
maxlen=40) if semi_loss and semi_dataset is not None else None
total_rewbuffer = deque(
maxlen=40) if semi_loss and semi_dataset is not None else None
not_update = 1 if not freeze_d else 0 # do not update G before D the first time
loaded = False
# if provide pretrained weight
if not U.load_checkpoint_variables(pretrained_weight,
include_no_prefix_vars=True):
# if no general checkpoint available, check sub-checkpoints for both networks
if U.load_checkpoint_variables(pretrained_il,
prefix=get_il_prefix(),
include_no_prefix_vars=False):
if rank == 0:
logger.log("loaded checkpoint variables from " + pretrained_il)
loaded = True
elif expert_label == get_il_prefix():
logger.log("ERROR no available cat_dauggi expert model in ",
pretrained_il)
exit(1)
if U.load_checkpoint_variables(pretrained_semi,
prefix=get_semi_prefix(),
include_no_prefix_vars=False):
if rank == 0:
logger.log("loaded checkpoint variables from " +
pretrained_semi)
loaded = True
elif expert_label == get_semi_prefix():
if rank == 0:
logger.log("ERROR no available semi expert model in ",
pretrained_semi)
exit(1)
else:
loaded = True
if rank == 0:
logger.log("loaded checkpoint variables from " + pretrained_weight)
if loaded:
not_update = 0 if any(
[x.op.name.find("adversary") != -1
for x in U.ALREADY_INITIALIZED]) else 1
if pretrained_weight and pretrained_weight.rfind("iter_") and \
pretrained_weight[pretrained_weight.rfind("iter_") + len("iter_"):].isdigit():
curr_iter = int(
pretrained_weight[pretrained_weight.rfind("iter_") +
len("iter_"):]) + 1
if rank == 0:
print("loaded checkpoint at iteration: " + str(curr_iter))
iters_so_far = curr_iter
for label in timesteps_so_far:
timesteps_so_far[label] = iters_so_far * timesteps_per_batch
d_adam = MpiAdam(reward_giver.get_trainable_variables())
vfadam = {label: MpiAdam(vf_var_list[label]) for label in ob_space}
U.initialize()
d_adam.sync()
for label in ob_space:
th_init = get_flat[label]()
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat[label](th_init)
vfadam[label].sync()
if rank == 0:
print(label + "Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = {
label: traj_segment_generator(
pi[label],
env[label],
reward_giver,
timesteps_per_batch,
stochastic=True,
semi_dataset=semi_dataset if label == get_semi_prefix() else None,
semi_loss=semi_loss,
reward_threshold=expert_reward_threshold
if label == expert_label else None,
sparse_reward=sparse_reward if label == expert_label else False)
for label in ob_space
}
g_losses = {}
assert sum([max_iters > 0, max_timesteps > 0, max_episodes > 0]) == 1
g_loss_stats = {
label: stats(loss_names[label] + vf_loss_names[label])
for label in ob_space if label != expert_label
}
d_loss_stats = stats(reward_giver.loss_name)
ep_names = ["True_rewards", "Rewards", "Episode_length", "Success"]
ep_stats = {label: None for label in ob_space}
# cat_dauggi network stats
if get_il_prefix() in ep_stats:
ep_stats[get_il_prefix()] = stats([name for name in ep_names])
# semi network stats
if get_semi_prefix() in ep_stats:
if semi_loss and semi_dataset is not None:
ep_names.append("L2_loss")
ep_names.append("total_rewards")
ep_stats[get_semi_prefix()] = stats(
[get_semi_prefix() + name for name in ep_names])
if rank == 0:
start_time = time.time()
ch_count = 0
env_type = env[expert_label].env.env.__class__.__name__
while True:
if callback: callback(locals(), globals())
if max_timesteps and any(
[timesteps_so_far[label] >= max_timesteps for label in ob_space]):
break
elif max_episodes and any(
[episodes_so_far[label] >= max_episodes for label in ob_space]):
break
elif max_iters and iters_so_far >= max_iters:
break
# Save model
if rank == 0 and iters_so_far % save_per_iter == 0 and ckpt_dir is not None:
fname = os.path.join(ckpt_dir, task_name)
if env_type.find("Pendulum") != -1 or save_per_iter != 1:
fname = os.path.join(ckpt_dir, 'iter_' + str(iters_so_far),
'iter_' + str(iters_so_far))
os.makedirs(os.path.dirname(fname), exist_ok=True)
saver = tf.train.Saver()
saver.save(tf.get_default_session(), fname, write_meta_graph=False)
if rank == 0 and time.time(
) - start_time >= 3600 * ch_count: # save a different checkpoint every hour
fname = os.path.join(ckpt_dir, 'hour' + str(ch_count).zfill(3))
fname = os.path.join(fname, 'iter_' + str(iters_so_far))
os.makedirs(os.path.dirname(fname), exist_ok=True)
saver = tf.train.Saver()
saver.save(tf.get_default_session(), fname, write_meta_graph=False)
ch_count += 1
logger.log("********** Iteration %i ************" % iters_so_far)
def fisher_func_builder(label):
def fisher_vector_product(p):
return allmean(compute_fvp[label](p, *
fvpargs)) + cg_damping * p
return fisher_vector_product
# ------------------ Update G ------------------
d = {label: None for label in ob_space}
segs = {label: None for label in ob_space}
logger.log("Optimizing Policy...")
for curr_step in range(g_step):
for label in ob_space:
if curr_step and label == expert_label: # get expert trajectories only for one g_step which is same as d_step
continue
logger.log("Optimizing Policy " + label + "...")
with timed("sampling"):
segs[label] = seg = seg_gen[label].__next__()
seg["rew"] = seg["rew"] - seg["l2_loss"] * l2_w
add_vtarg_and_adv(seg, gamma, lam)
ob, ac, atarg, tdlamret, full_ob = seg["ob"], seg["ac"], seg[
"adv"], seg["tdlamret"], seg["full_ob"]
vpredbefore = seg[
"vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()) / atarg.std(
) # standardized advantage function estimate
d[label] = Dataset(dict(ob=ob,
ac=ac,
atarg=atarg,
vtarg=tdlamret),
shuffle=True)
if not_update or label == expert_label:
continue # stop G from updating
if hasattr(pi[label], "ob_rms"):
pi[label].ob_rms.update(
full_ob) # update running mean/std for policy
args = seg["full_ob"], seg["ac"], atarg
fvpargs = [arr[::5] for arr in args]
assign_old_eq_new[label](
) # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad[label](*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_func_builder(label),
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_func_builder(label)(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat[label]()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat[label](thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses[label](*args)))
if rank == 0:
print("Generator entropy " + str(meanlosses[4]) +
", loss " + str(meanlosses[2]))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log(
"Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log(
"violated KL constraint. shrinking step.")
elif improve < 0:
logger.log(
"surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat[label](thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(),
vfadam[label].getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0])
for ps in paramsums[1:])
expert_dataset = d[expert_label]
if not_update:
break
for label in ob_space:
if label == expert_label:
continue
with timed("vf"):
logger.log(fmt_row(13, vf_loss_names[label]))
for _ in range(vf_iters):
vf_b_losses = []
for batch in d[label].iterate_once(vf_batchsize):
mbob = batch["ob"]
mbret = batch["vtarg"]
*newlosses, g = compute_vflossandgrad[label](mbob,
mbret)
g = allmean(g)
newlosses = allmean(np.array(newlosses))
vfadam[label].update(g, vf_stepsize[label])
vf_b_losses.append(newlosses)
logger.log(fmt_row(13, np.mean(vf_b_losses, axis=0)))
logger.log("Evaluating losses...")
losses = []
for batch in d[label].iterate_once(vf_batchsize):
newlosses = compute_vf_losses[label](batch["ob"],
batch["ac"],
batch["atarg"],
batch["vtarg"])
losses.append(newlosses)
g_losses[label], _, _ = mpi_moments(losses, axis=0)
#########################
for ob_batch, ac_batch, full_ob_batch in dataset.iterbatches(
(segs[label]["ob"], segs[label]["ac"],
segs[label]["full_ob"]),
include_final_partial_batch=False,
batch_size=len(ob)):
expert_batch = expert_dataset.next_batch(len(ob))
ob_expert, ac_expert = expert_batch["ob"], expert_batch[
"ac"]
exp_rew = 0
exp_rews = None
for obs, acs in zip(ob_expert, ac_expert):
curr_rew = reward_giver.get_reward(obs, acs)[0][0] \
if not hasattr(reward_giver, '_labels') else \
reward_giver.get_reward(obs, acs, label)
if isinstance(curr_rew, tuple):
curr_rew, curr_rews = curr_rew
exp_rews = 1 - np.exp(
-curr_rews
) if exp_rews is None else exp_rews + 1 - np.exp(
-curr_rews)
exp_rew += 1 - np.exp(-curr_rew)
mean_exp_rew = exp_rew / len(ob_expert)
mean_exp_rews = exp_rews / len(
ob_expert) if exp_rews is not None else None
gen_rew = 0
gen_rews = None
for obs, acs, full_obs in zip(ob_batch, ac_batch,
full_ob_batch):
curr_rew = reward_giver.get_reward(obs, acs)[0][0] \
if not hasattr(reward_giver, '_labels') else \
reward_giver.get_reward(obs, acs, label)
if isinstance(curr_rew, tuple):
curr_rew, curr_rews = curr_rew
gen_rews = 1 - np.exp(
-curr_rews
) if gen_rews is None else gen_rews + 1 - np.exp(
-curr_rews)
gen_rew += 1 - np.exp(-curr_rew)
mean_gen_rew = gen_rew / len(ob_batch)
mean_gen_rews = gen_rews / len(
ob_batch) if gen_rews is not None else None
if rank == 0:
msg = "Network " + label + \
" Generator step " + str(curr_step) + ": Dicriminator reward of expert traj " \
+ str(mean_exp_rew) + " vs gen traj " + str(mean_gen_rew)
if mean_exp_rews is not None and mean_gen_rews is not None:
msg += "\nDiscriminator multi rewards of expert " + str(mean_exp_rews) + " vs gen " \
+ str(mean_gen_rews)
logger.log(msg)
#########################
if not not_update:
for label in g_losses:
for (lossname,
lossval) in zip(loss_names[label] + vf_loss_names[label],
g_losses[label]):
logger.record_tabular(lossname, lossval)
logger.record_tabular(
label + "ev_tdlam_before",
explained_variance(segs[label]["vpred"],
segs[label]["tdlamret"]))
# ------------------ Update D ------------------
if not freeze_d:
logger.log("Optimizing Discriminator...")
batch_size = len(list(segs.values())[0]['ob']) // d_step
expert_dataset = d[expert_label]
batch_gen = {
label: dataset.iterbatches(
(segs[label]["ob"], segs[label]["ac"]),
include_final_partial_batch=False,
batch_size=batch_size)
for label in segs if label != expert_label
}
d_losses = [
] # list of tuples, each of which gives the loss for a minibatch
for step in range(d_step):
g_ob = {}
g_ac = {}
for label in batch_gen: # get batches for different gens
g_ob[label], g_ac[label] = batch_gen[label].__next__()
expert_batch = expert_dataset.next_batch(batch_size)
ob_expert, ac_expert = expert_batch["ob"], expert_batch["ac"]
for label in g_ob:
#########################
exp_rew = 0
exp_rews = None
for obs, acs in zip(ob_expert, ac_expert):
curr_rew = reward_giver.get_reward(obs, acs)[0][0] \
if not hasattr(reward_giver, '_labels') else \
reward_giver.get_reward(obs, acs, label)
if isinstance(curr_rew, tuple):
curr_rew, curr_rews = curr_rew
exp_rews = 1 - np.exp(
-curr_rews
) if exp_rews is None else exp_rews + 1 - np.exp(
-curr_rews)
exp_rew += 1 - np.exp(-curr_rew)
mean_exp_rew = exp_rew / len(ob_expert)
mean_exp_rews = exp_rews / len(
ob_expert) if exp_rews is not None else None
gen_rew = 0
gen_rews = None
for obs, acs in zip(g_ob[label], g_ac[label]):
curr_rew = reward_giver.get_reward(obs, acs)[0][0] \
if not hasattr(reward_giver, '_labels') else \
reward_giver.get_reward(obs, acs, label)
if isinstance(curr_rew, tuple):
curr_rew, curr_rews = curr_rew
gen_rews = 1 - np.exp(
-curr_rews
) if gen_rews is None else gen_rews + 1 - np.exp(
-curr_rews)
gen_rew += 1 - np.exp(-curr_rew)
mean_gen_rew = gen_rew / len(g_ob[label])
mean_gen_rews = gen_rews / len(
g_ob[label]) if gen_rews is not None else None
if rank == 0:
msg = "Dicriminator reward of expert traj " + str(mean_exp_rew) + " vs " + label + \
"gen traj " + str(mean_gen_rew)
if mean_exp_rews is not None and mean_gen_rews is not None:
msg += "\nDiscriminator multi expert rewards " + str(mean_exp_rews) + " vs " + label + \
"gen " + str(mean_gen_rews)
logger.log(msg)
#########################
# update running mean/std for reward_giver
if hasattr(reward_giver, "obs_rms"):
reward_giver.obs_rms.update(
np.concatenate(list(g_ob.values()) + [ob_expert], 0))
*newlosses, g = reward_giver.lossandgrad(
*(list(g_ob.values()) + list(g_ac.values()) + [ob_expert] +
[ac_expert]))
d_adam.update(allmean(g), d_stepsize)
d_losses.append(newlosses)
logger.log(fmt_row(13, reward_giver.loss_name))
logger.log(fmt_row(13, np.mean(d_losses, axis=0)))
for label in ob_space:
lrlocal = (segs[label]["ep_lens"], segs[label]["ep_rets"],
segs[label]["ep_true_rets"], segs[label]["ep_success"],
segs[label]["ep_semi_loss"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews, true_rets, success, semi_losses = map(
flatten_lists, zip(*listoflrpairs))
# success
success = [
float(elem) for elem in success
if isinstance(elem, (int, float, bool))
] # remove potential None types
if not success:
success = [-1] # set success to -1 if env has no success flag
success_buffer[label].extend(success)
true_rewbuffer[label].extend(true_rets)
lenbuffer[label].extend(lens)
rewbuffer[label].extend(rews)
if semi_loss and semi_dataset is not None and label == get_semi_prefix(
):
semi_losses = [elem * l2_w for elem in semi_losses]
total_rewards = rews
total_rewards = [
re_elem - l2_elem
for re_elem, l2_elem in zip(total_rewards, semi_losses)
]
l2_rewbuffer.extend(semi_losses)
total_rewbuffer.extend(total_rewards)
logger.record_tabular(label + "EpLenMean",
np.mean(lenbuffer[label]))
logger.record_tabular(label + "EpRewMean",
np.mean(rewbuffer[label]))
logger.record_tabular(label + "EpTrueRewMean",
np.mean(true_rewbuffer[label]))
logger.record_tabular(label + "EpSuccess",
np.mean(success_buffer[label]))
if semi_loss and semi_dataset is not None and label == get_semi_prefix(
):
logger.record_tabular(label + "EpSemiLoss",
np.mean(l2_rewbuffer))
logger.record_tabular(label + "EpTotalLoss",
np.mean(total_rewbuffer))
logger.record_tabular(label + "EpThisIter", len(lens))
episodes_so_far[label] += len(lens)
timesteps_so_far[label] += sum(lens)
logger.record_tabular(label + "EpisodesSoFar",
episodes_so_far[label])
logger.record_tabular(label + "TimestepsSoFar",
timesteps_so_far[label])
logger.record_tabular("TimeElapsed", time.time() - tstart)
iters_so_far += 1
logger.record_tabular("ItersSoFar", iters_so_far)
if rank == 0:
logger.dump_tabular()
if not not_update:
for label in g_loss_stats:
g_loss_stats[label].add_all_summary(
writer, g_losses[label], iters_so_far)
if not freeze_d:
d_loss_stats.add_all_summary(writer, np.mean(d_losses, axis=0),
iters_so_far)
for label in ob_space:
# default buffers
ep_buffers = [
np.mean(true_rewbuffer[label]),
np.mean(rewbuffer[label]),
np.mean(lenbuffer[label]),
np.mean(success_buffer[label])
]
if semi_loss and semi_dataset is not None and label == get_semi_prefix(
):
ep_buffers.append(np.mean(l2_rewbuffer))
ep_buffers.append(np.mean(total_rewbuffer))
ep_stats[label].add_all_summary(writer, ep_buffers,
iters_so_far)
if not_update and not freeze_g:
not_update -= 1
def learn(
*,
network,
env,
total_timesteps,
timesteps_per_batch=1024, # what to train on
max_kl=0.001,
cg_iters=10,
gamma=0.99,
lam=1.0, # advantage estimation
seed=None,
ent_coef=0.0,
cg_damping=1e-2,
vf_stepsize=3e-4,
vf_iters=3,
max_episodes=0,
max_iters=0, # time constraint
callback=None,
load_path=None,
vf_batch_size=64,
**network_kwargs):
'''
learn a policy function with TRPO algorithm
Parameters:
----------
network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)
or function that takes input placeholder and returns tuple (output, None) for feedforward nets
or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets
env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class
timesteps_per_batch timesteps per gradient estimation batch
max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )
ent_coef coefficient of policy entropy term in the optimization objective
cg_iters number of iterations of conjugate gradient algorithm
cg_damping conjugate gradient damping
vf_stepsize learning rate for adam optimizer used to optimie value function loss
vf_iters number of iterations of value function optimization iterations per each policy optimization step
total_timesteps max number of timesteps
max_episodes max number of episodes
max_iters maximum number of policy optimization iterations
callback function to be called with (locals(), globals()) each policy optimization step
load_path str, path to load the model from (default: None, i.e. no model is loaded)
**network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network
Returns:
-------
learnt model
'''
if not isinstance(
env.action_space, spaces.Discrete
): #Atari envs are already parallel without using vecnormalize wrapper
nenvs = env.num_envs
else:
nenvs = 1
nbatch = nenvs * timesteps_per_batch
l1regpi = network_kwargs['l1regpi']
l2regpi = network_kwargs['l2regpi']
l1regvf = network_kwargs['l1regvf']
l2regvf = network_kwargs['l2regvf']
toregularizepi = l1regpi > 0 or l2regpi > 0
toregularizevf = l1regvf > 0 or l2regvf > 0
toweightclippi = False
toweightclipvf = False
weight_clip_range_pi = 0
weight_clip_range_vf = 0
if network_kwargs['wclippi'] > 0:
weight_clip_range_pi = network_kwargs['wclippi']
print("Clipping policy network = {}".format(weight_clip_range_pi))
toweightclippi = True
if network_kwargs['wclipvf'] > 0:
weight_clip_range_vf = network_kwargs['wclipvf']
print("Clipping value network = {}".format(weight_clip_range_vf))
toweightclipvf = True
if MPI is not None:
nworkers = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
else:
nworkers = 1
rank = 0
cpus_per_worker = 1
U.get_session(
config=tf.ConfigProto(allow_soft_placement=True,
inter_op_parallelism_threads=cpus_per_worker,
intra_op_parallelism_threads=cpus_per_worker))
batchnormpi = network_kwargs["batchnormpi"]
batchnormvf = network_kwargs["batchnormvf"]
dropoutpi_keep_prob = None
todropoutpi = network_kwargs["dropoutpi"] < 1.0
dropoutvf_keep_prob = None
todropoutvf = network_kwargs["dropoutvf"] < 1.0
if todropoutpi or todropoutvf:
policy, dropoutpi_keep_prob, dropoutvf_keep_prob = build_policy(
env, network, value_network="copy", **network_kwargs)
else:
policy = build_policy(env,
network,
value_network="copy",
**network_kwargs)
isbnpitrainmode = None
isbnvftrainmode = None
if batchnormpi and batchnormvf:
policy, isbnpitrainmode, isbnvftrainmode = policy
elif batchnormpi and not batchnormvf:
policy, isbnpitrainmode = policy
elif batchnormvf and not batchnormpi:
policy, isbnvftrainmode = policy
#policy = build_policy(env, network, value_network='copy', **network_kwargs)
set_global_seeds(seed)
np.set_printoptions(precision=3)
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
ob = observation_placeholder(ob_space)
with tf.variable_scope("pi"):
pi = policy(observ_placeholder=ob)
with tf.variable_scope("oldpi"):
oldpi = policy(observ_placeholder=ob)
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = tf.reduce_mean(kloldnew)
meanent = tf.reduce_mean(ent)
entbonus = ent_coef * meanent
vferr = tf.reduce_mean(tf.square(pi.vf - ret))
ratio = tf.exp(pi.pd.logp(ac) -
oldpi.pd.logp(ac)) # advantage * pnew / pold
surrgain = tf.reduce_mean(ratio * atarg)
optimgain = surrgain + entbonus
losses = [optimgain, meankl, entbonus, surrgain, meanent]
loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]
dist = meankl
all_var_list = get_trainable_variables("pi")
# var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("pol")]
# vf_var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("vf")]
var_list = get_pi_trainable_variables("pi")
vf_var_list = get_vf_trainable_variables("pi")
if toregularizepi:
print("Regularizing policy network: L1 = {}, L2 = {}".format(
l1regpi, l2regpi))
regularizerpi = tf.contrib.layers.l1_l2_regularizer(scale_l1=l1regpi,
scale_l2=l2regpi,
scope="pi")
all_trainable_weights_pi = var_list.copy()
regularization_penalty_pi = tf.contrib.layers.apply_regularization(
regularizerpi, all_trainable_weights_pi)
optimgain = optimgain - regularization_penalty_pi
if toregularizevf:
print("Regularizing value network: L1 = {}, L2 = {}".format(
l1regvf, l2regvf))
regularizervf = tf.contrib.layers.l1_l2_regularizer(scale_l1=l1regvf,
scale_l2=l2regvf,
scope="vf")
all_trainable_weights_vf = vf_var_list
regularization_penalty_vf = tf.contrib.layers.apply_regularization(
regularizervf, all_trainable_weights_vf)
vferr = vferr + regularization_penalty_vf
vfadam = MpiAdam(vf_var_list)
get_flat = U.GetFlat(var_list)
set_from_flat = U.SetFromFlat(var_list)
klgrads = tf.gradients(dist, var_list)
flat_tangent = tf.placeholder(dtype=tf.float32,
shape=[None],
name="flat_tan")
shapes = [var.get_shape().as_list() for var in var_list]
start = 0
tangents = []
for shape in shapes:
sz = U.intprod(shape)
tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
start += sz
gvp = tf.add_n([
tf.reduce_sum(g * tangent)
for (g, tangent) in zipsame(klgrads, tangents)
]) #pylint: disable=E1111
fvp = U.flatgrad(gvp, var_list)
assign_old_eq_new = U.function(
[], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(get_variables("oldpi"), get_variables("pi"))
])
lossespi_inputs = [flat_tangent, ob, ac, atarg]
lossesvf_inputs = [ob, ret]
if todropoutpi:
lossespi_inputs.append(dropoutpi_keep_prob)
if todropoutvf:
lossesvf_inputs.append(dropoutvf_keep_prob)
if batchnormpi:
lossespi_inputs.append(isbnpitrainmode)
if batchnormvf:
lossesvf_inputs.append(isbnvftrainmode)
compute_losses = U.function(lossespi_inputs[1:], losses)
compute_lossandgrad = U.function(
lossespi_inputs[1:], losses + [U.flatgrad(optimgain, var_list)])
compute_fvp = U.function(lossespi_inputs, fvp)
compute_vflossandgrad = U.function(lossesvf_inputs,
U.flatgrad(vferr, vf_var_list))
@contextmanager
def timed(msg):
if rank == 0:
print(colorize(msg, color='magenta'))
tstart = time.time()
yield
print(
colorize("done in %.3f seconds" % (time.time() - tstart),
color='magenta'))
else:
yield
def allmean(x):
assert isinstance(x, np.ndarray)
if MPI is not None:
out = np.empty_like(x)
MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
out /= nworkers
else:
out = np.copy(x)
return out
U.initialize()
if load_path is not None:
pi.load(load_path)
th_init = get_flat()
if MPI is not None:
MPI.COMM_WORLD.Bcast(th_init, root=0)
set_from_flat(th_init)
vfadam.sync()
print("Init param sum", th_init.sum(), flush=True)
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(
pi,
env,
timesteps_per_batch,
nenvs,
stochastic=True,
dropoutpi_keep_prob=(dropoutpi_keep_prob if todropoutpi else None),
dropoutvf_keep_prob=(dropoutvf_keep_prob if todropoutvf else None),
isbnpitrainmode=isbnpitrainmode,
isbnvftrainmode=isbnvftrainmode)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards
if sum([max_iters > 0, total_timesteps > 0, max_episodes > 0]) == 0:
# noththing to be done
return pi
assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \
'out of max_iters, total_timesteps, and max_episodes only one should be specified'
if toweightclippi:
_wclip_ops_pi = []
wclip_bounds_pi = [-weight_clip_range_pi, weight_clip_range_pi]
for toclipvar in var_list:
if 'logstd' in toclipvar.name:
continue
_wclip_ops_pi.append(
tf.assign(
toclipvar,
tf.clip_by_value(toclipvar, wclip_bounds_pi[0],
wclip_bounds_pi[1])))
_wclip_op_pi = tf.group(*_wclip_ops_pi)
if toweightclipvf:
_wclip_ops_vf = []
wclip_bounds_vf = [-weight_clip_range_vf, weight_clip_range_vf]
for toclipvar in vf_var_list:
_wclip_ops_vf.append(
tf.assign(
toclipvar,
tf.clip_by_value(toclipvar, wclip_bounds_vf[0],
wclip_bounds_vf[1])))
_wclip_op_vf = tf.group(*_wclip_ops_vf)
while True:
if callback: callback(locals(), globals())
if total_timesteps and timesteps_so_far >= total_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
logger.log("********** Iteration %i ************" % iters_so_far)
with timed("sampling"):
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam, nenvs)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()
) / atarg.std() # standardized advantage function estimate
if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
if hasattr(pi, "ob_rms"):
pi.ob_rms.update(ob) # update running mean/std for policy
if isinstance(env.action_space, spaces.Discrete):
seg["ob"] = seg["ob"].reshape(
np.concatenate([[-1, seg["ob"].shape[1]],
seg["ob"].shape[1:]]))
args = seg["ob"], seg["ac"], atarg
if todropoutpi:
args = args + (network_kwargs['dropoutpi'], )
if batchnormpi:
args = args + (True, )
fvpargs = [arr[::5] for arr in args if isinstance(arr, np.ndarray)]
if todropoutpi:
fvpargs.append(network_kwargs['dropoutpi'])
if batchnormpi:
fvpargs.append(True, )
def fisher_vector_product(p):
return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p
assign_old_eq_new() # set old parameter values to new parameter values
with timed("computegrad"):
*lossbefore, g = compute_lossandgrad(*args)
lossbefore = allmean(np.array(lossbefore))
g = allmean(g)
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with timed("cg"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=cg_iters,
verbose=rank == 0)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / max_kl)
# logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore[0]
stepsize = 1.0
thbefore = get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
set_from_flat(thnew)
meanlosses = surr, kl, *_ = allmean(
np.array(compute_losses(*args)))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
set_from_flat(thbefore)
if nworkers > 1 and iters_so_far % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(), vfadam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0]) for ps in paramsums[1:])
if toweightclippi:
U.get_session().run(_wclip_op_pi)
for (lossname, lossval) in zip(loss_names, meanlosses):
logger.record_tabular(lossname, lossval)
with timed("vf"):
for _ in range(vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(seg["ob"], seg["tdlamret"]),
include_final_partial_batch=False,
batch_size=vf_batch_size):
vflossandgrad_inputs = (mbob, mbret)
if todropoutvf:
vflossandgrad_inputs += (network_kwargs["dropoutvf"], )
if batchnormvf:
vflossandgrad_inputs += (True, )
g = allmean(compute_vflossandgrad(*vflossandgrad_inputs))
if callable(vf_stepsize):
vfadam.update(
g, vf_stepsize(timesteps_so_far / total_timesteps))
else:
vfadam.update(g, vf_stepsize)
if toweightclipvf:
U.get_session().run(_wclip_op_vf)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
meanret = [np.mean(arr) for arr in seg["ep_rets"]]
meanret = sum(meanret) / len(meanret)
totallen = [np.sum(arr) for arr in seg["ep_lens"]]
totallen = sum(totallen)
meanlen = [np.mean(arr) for arr in seg["ep_lens"]]
meanlen = sum(meanlen) / len(meanlen)
logger.record_tabular("EpLenMean", meanlen)
logger.record_tabular("EpRewMean", meanret)
totalepi = sum([len(arr) for arr in seg["ep_lens"]])
logger.record_tabular("EpThisIter", totalepi)
episodes_so_far += totalepi
timesteps_so_far += totallen
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if rank == 0:
logger.dump_tabular()
return pi
def _update_policy(self, rollouts, it):
pi = self._policy
seg = rollouts[self.id]
info = defaultdict(list)
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
atarg = (atarg - atarg.mean()) / atarg.std()
info['adv'] = np.mean(atarg)
other_ob_list = []
for i, other_pi in enumerate(self._pis):
other_ob_list.extend(other_pi.get_ob_list(rollouts[i]["ob"]))
ob_list = pi.get_ob_list(ob)
args = ob_list * self._config.num_contexts + \
other_ob_list * 2 + ob_list + [ac, atarg]
fvpargs = [arr[::5] for arr in args]
def fisher_vector_product(p):
return self._all_mean(self._compute_fvp(
p, *fvpargs)) + self._config.cg_damping * p
self._update_oldpi()
with self.timed("compute gradient"):
lossbefore = self._compute_lossandgrad(*args)
lossbefore = {
k: self._all_mean(np.array(lossbefore[k]))
for k in sorted(lossbefore.keys())
}
g = lossbefore['g']
if np.allclose(g, 0):
logger.log("Got zero gradient. not updating")
else:
with self.timed("compute conjugate gradient"):
stepdir = cg(fisher_vector_product,
g,
cg_iters=self._config.cg_iters,
verbose=False)
assert np.isfinite(stepdir).all()
shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
lm = np.sqrt(shs / self._config.max_kl)
fullstep = stepdir / lm
expectedimprove = g.dot(fullstep)
surrbefore = lossbefore['pol_loss']
stepsize = 1.0
thbefore = self._get_flat()
for _ in range(10):
thnew = thbefore + fullstep * stepsize
self._set_from_flat(thnew)
meanlosses = self._compute_losses(*args)
meanlosses = {
k: self._all_mean(np.array(meanlosses[k]))
for k in sorted(meanlosses.keys())
}
for key, value in meanlosses.items():
if key != 'g':
info[key].append(value)
surr = meanlosses['pol_loss']
kl = meanlosses['kl']
meanlosses = np.array(list(meanlosses.values()))
improve = surr - surrbefore
logger.log("Expected: %.3f Actual: %.3f" %
(expectedimprove, improve))
if not np.isfinite(meanlosses).all():
logger.log("Got non-finite value of losses -- bad!")
elif kl > self._config.max_kl * 1.5:
logger.log("violated KL constraint. shrinking step.")
elif improve < 0:
logger.log("surrogate didn't improve. shrinking step.")
else:
logger.log("Stepsize OK!")
break
stepsize *= .5
else:
logger.log("couldn't compute a good step")
self._set_from_flat(thbefore)
if self._num_workers > 1 and it % 20 == 0:
paramsums = MPI.COMM_WORLD.allgather(
(thnew.sum(),
self._vf_adam.getflat().sum())) # list of tuples
assert all(
np.allclose(ps, paramsums[0])
for ps in paramsums[1:]), paramsums
with self.timed("updating value function"):
for _ in range(self._config.vf_iters):
for (mbob, mbret) in dataset.iterbatches(
(ob, tdlamret),
include_final_partial_batch=False,
batch_size=self._config.vf_batch_size):
ob_list = pi.get_ob_list(mbob)
g = self._all_mean(
self._compute_vflossandgrad(*ob_list, mbret))
self._vf_adam.update(g, self._config.vf_stepsize)
vf_loss = self._all_mean(
np.array(self._compute_vfloss(*ob_list, mbret)))
info['vf_loss'].append(vf_loss)
for key, value in info.items():
info[key] = np.mean(value)
return info