def get_action(self, session, observations, action_dim):
if not self.state_initialized:
self._make_graph(observations, action_dim)
load_state(self.model_file)
self.state_initialized = True
return session.run([self.action], feed_dict={self.observation: observations})
def load(path, num_cpus=1):
with open(path, "rb") as f:
model_data, act_params = dill.load(f)
act = build_act(**act_params)
sess = U.make_session(num_cpus=num_cpus)
sess.__enter__()
with tempfile.TemporaryDirectory() as td:
filepath = os.path.join(td, "packed.zip")
with open(filepath, "wb") as f:
f.write(model_data)
zipfile.ZipFile(filepath, 'r', zipfile.ZIP_DEFLATED).extractall(td)
U.load_state(os.path.join(td, "model"))
return ActWrapper(act, act_params)
def load(path):
with open(path, "rb") as f:
model_data, act_params = cloudpickle.load(f)
act = deepq.build_act(**act_params)
sess = tf.Session()
sess.__enter__()
with tempfile.TemporaryDirectory() as td:
arc_path = os.path.join(td, "packed.zip")
with open(arc_path, "wb") as f:
f.write(model_data)
zipfile.ZipFile(arc_path, 'r', zipfile.ZIP_DEFLATED).extractall(td)
load_state(os.path.join(td, "model"))
return ActWrapper(act, act_params)
def run():
import mlp_policy_robo
U.make_session(num_cpu=1).__enter__()
env = gym.make("RoboschoolHumanoid-v1")
#env = wrappers.Monitor(env, directory="./video/HalfCheeta-v1", force=True)
def policy_fn(name, ob_space, ac_space):
return mlp_policy_robo.MlpPolicy(name=name,
ob_space=ob_space,
ac_space=ac_space,
hid_size=128,
num_hid_layers=2)
ob_space = env.observation_space
ac_space = env.action_space
pi = policy_fn("pi", ob_space, ac_space)
oldpi = policy_fn("oldpi", ob_space, ac_space)
U.load_state("save/Humanoid-v1")
for epi in range(100):
ob = env.reset()
total_reward = 0
step = 0
while True:
env.render("human")
ac, v = pi.act(True, ob)
ob, rew, new, info = env.step(ac)
step += 1
total_reward += rew
if new:
print("Reward: {}, Step: {}".format(total_reward, step))
break
def learn(
env,
policy_func,
*,
timesteps_per_batch, # timesteps per actor per update
clip_param,
entcoeff, # clipping parameter epsilon, entropy coeff
optim_epochs,
optim_stepsize,
optim_batchsize, # optimization hypers
gamma,
lam, # advantage estimation
max_timesteps=0,
max_episodes=0,
max_iters=0,
max_seconds=0, # time constraint
callback=None, # you can do anything in the callback, since it takes locals(), globals()
adam_epsilon=1e-5,
schedule='constant' # annealing for stepsize parameters (epsilon and adam)
):
# Setup losses and stuff
# ----------------------------------------
ob_space = env.observation_space
ac_space = env.action_space
pi = policy_func("pi", ob_space,
ac_space) # Construct network for new policy
oldpi = policy_func("oldpi", ob_space, ac_space) # Network for old policy
atarg = tf.placeholder(
dtype=tf.float32,
shape=[None]) # Target advantage function (if applicable)
ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return
lrmult = tf.placeholder(
name='lrmult', dtype=tf.float32,
shape=[]) # learning rate multiplier, updated with schedule
clip_param = clip_param * lrmult # Annealed cliping parameter epislon
ob = U.get_placeholder_cached(name="ob")
ac = pi.pdtype.sample_placeholder([None])
kloldnew = oldpi.pd.kl(pi.pd)
ent = pi.pd.entropy()
meankl = U.mean(kloldnew)
meanent = U.mean(ent)
pol_entpen = (-entcoeff) * meanent
ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) # pnew / pold
surr1 = ratio * atarg # surrogate from conservative policy iteration
surr2 = U.clip(ratio, 1.0 - clip_param, 1.0 + clip_param) * atarg #
pol_surr = -U.mean(tf.minimum(
surr1, surr2)) # PPO's pessimistic surrogate (L^CLIP)
vf_loss = U.mean(tf.square(pi.vpred - ret))
total_loss = pol_surr + pol_entpen + vf_loss
losses = [pol_surr, pol_entpen, vf_loss, meankl, meanent]
loss_names = ["pol_surr", "pol_entpen", "vf_loss", "kl", "ent"]
var_list = pi.get_trainable_variables()
lossandgrad = U.function([ob, ac, atarg, ret, lrmult],
losses + [U.flatgrad(total_loss, var_list)])
adam = MpiAdam(var_list, epsilon=adam_epsilon)
assign_old_eq_new = U.function(
[], [],
updates=[
tf.assign(oldv, newv)
for (oldv,
newv) in zipsame(oldpi.get_variables(), pi.get_variables())
])
compute_losses = U.function([ob, ac, atarg, ret, lrmult], losses)
U.initialize()
adam.sync()
U.load_state("save/Humanoid-v1")
# Prepare for rollouts
# ----------------------------------------
seg_gen = traj_segment_generator(pi,
env,
timesteps_per_batch,
stochastic=True)
episodes_so_far = 0
timesteps_so_far = 0
iters_so_far = 0
tstart = time.time()
lenbuffer = deque(maxlen=100) # rolling buffer for episode lengths
rewbuffer = deque(maxlen=100) # rolling buffer for episode rewards
assert sum(
[max_iters > 0, max_timesteps > 0, max_episodes > 0,
max_seconds > 0]) == 1, "Only one time constraint permitted"
while True:
if callback: callback(locals(), globals())
if max_timesteps and timesteps_so_far >= max_timesteps:
break
elif max_episodes and episodes_so_far >= max_episodes:
break
elif max_iters and iters_so_far >= max_iters:
break
elif max_seconds and time.time() - tstart >= max_seconds:
break
if schedule == 'constant':
cur_lrmult = 1.0
elif schedule == 'linear':
cur_lrmult = max(1.0 - float(timesteps_so_far) / max_timesteps, 0)
else:
raise NotImplementedError
logger.log("********** Iteration %i ************" % iters_so_far)
seg = seg_gen.__next__()
add_vtarg_and_adv(seg, gamma, lam)
# ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
ob, ac, atarg, tdlamret = seg["ob"], seg["ac"], seg["adv"], seg[
"tdlamret"]
vpredbefore = seg["vpred"] # predicted value function before udpate
atarg = (atarg - atarg.mean()
) / atarg.std() # standardized advantage function estimate
d = Dataset(dict(ob=ob, ac=ac, atarg=atarg, vtarg=tdlamret),
shuffle=not pi.recurrent)
optim_batchsize = optim_batchsize or ob.shape[0]
#if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy
assign_old_eq_new() # set old parameter values to new parameter values
logger.log("Optimizing...")
logger.log(fmt_row(13, loss_names))
# Here we do a bunch of optimization epochs over the data
for _ in range(optim_epochs):
losses = [
] # list of tuples, each of which gives the loss for a minibatch
for batch in d.iterate_once(optim_batchsize):
*newlosses, g = lossandgrad(batch["ob"], batch["ac"],
batch["atarg"], batch["vtarg"],
cur_lrmult)
adam.update(g, optim_stepsize * cur_lrmult)
losses.append(newlosses)
logger.log(fmt_row(13, np.mean(losses, axis=0)))
logger.log("Evaluating losses...")
losses = []
for batch in d.iterate_once(optim_batchsize):
newlosses = compute_losses(batch["ob"], batch["ac"],
batch["atarg"], batch["vtarg"],
cur_lrmult)
losses.append(newlosses)
meanlosses, _, _ = mpi_moments(losses, axis=0)
logger.log(fmt_row(13, meanlosses))
for (lossval, name) in zipsame(meanlosses, loss_names):
logger.record_tabular("loss_" + name, lossval)
logger.record_tabular("ev_tdlam_before",
explained_variance(vpredbefore, tdlamret))
lrlocal = (seg["ep_lens"], seg["ep_rets"]) # local values
listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
lens, rews = map(flatten_lists, zip(*listoflrpairs))
lenbuffer.extend(lens)
rewbuffer.extend(rews)
logger.record_tabular("EpLenMean", np.mean(lenbuffer))
logger.record_tabular("EpRewMean", np.mean(rewbuffer))
logger.record_tabular("EpThisIter", len(lens))
episodes_so_far += len(lens)
timesteps_so_far += sum(lens)
iters_so_far += 1
logger.record_tabular("EpisodesSoFar", episodes_so_far)
logger.record_tabular("TimestepsSoFar", timesteps_so_far)
logger.record_tabular("TimeElapsed", time.time() - tstart)
if MPI.COMM_WORLD.Get_rank() == 0:
logger.dump_tabular()
U.save_state("save/Humanoid-v1")
scenario.reset_world,
scenario.reward,
scenario.observation,
info_callback=None,
shared_viewer=True)
# render call to create viewer window (necessary only for interactive policies)
env.render_whole_field()
obs_shape_n = [env.observation_space[i].shape for i in range(env.n)]
num_adversaries = 7
trainers = get_trainers(env, num_adversaries, obs_shape_n, args)
# create interactive policies for each agent
policies = [InteractivePolicy(env, i) for i in range(env.n)]
# execution loop
# load session
saver = U.load_state(args.load_dir)
# TODO: Pick the latest?
# TODO: Do I need to make agents???
# So now the session hosted by U.single_threaded_session SHOULD be loaded?
obs_n = env.reset()
while True:
# query for action from each agent's policy
# act_n = []
# for i, policy in enumerate(policies):
# act_n.append(policy.action(obs_n[i]))
act_n = [agent.action(obs) for agent, obs in zip(trainers, obs_n)]
# environment step
# new_obs_n, rew_n, done_n, info_n = env.step(action_n)
def dist_learn(env,
q_dist_func,
num_atoms=51,
V_max=10,
lr=25e-5,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.01,
exploration_final_eps=0.008,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=2000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=1,
callback=None):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
num_cpu: int
number of cpus to use for training
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = U.single_threaded_session()
sess.__enter__()
def make_obs_ph(name):
print name
return U.BatchInput(env.observation_space.shape, name=name)
act, train, update_target, debug = build_dist_train(
make_obs_ph=make_obs_ph,
dist_func=q_dist_func,
num_actions=env.action_space.n,
num_atoms=num_atoms,
V_max=V_max,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10)
# act, train, update_target, debug = build_train(
# make_obs_ph=make_obs_ph,
# q_func=q_func,
# num_actions=env.action_space.n,
# optimizer=tf.train.AdamOptimizer(learning_rate=lr),
# gamma=gamma,
# grad_norm_clipping=10
# )
act_params = {
'make_obs_ph': make_obs_ph,
'q_dist_func': q_dist_func,
'num_actions': env.action_space.n,
}
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
print model_file
# mkdir_p(os.path.dirname(model_file))
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
action = act(np.array(obs)[None],
update_eps=exploration.value(t))[0]
new_obs, rew, done, _ = env.step(action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
# print "CCCC"
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
# print "Come1"
# print np.shape(obses_t), np.shape(actions), np.shape(rewards), np.shape(obses_tp1), np.shape(dones)
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
# print "Loss : {}".format(td_errors)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
print "steps : {}".format(t)
print "episodes : {}".format(num_episodes)
print "mean 100 episode reward: {}".format(mean_100ep_reward)
# print "mean 100 episode reward".format(mean_100ep_reward)
# logger.record_tabular("episodes", num_episodes)
# logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
# logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
# logger.dump_tabular()
# logger.record_tabular("steps", t)
# logger.record_tabular("episodes", num_episodes)
# logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
# logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
# logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and t % checkpoint_freq == 0):
print "=========================="
print "Error: {}".format(td_errors)
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
print "Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward)
# logger.log("Saving model due to mean reward increase: {} -> {}".format(
# saved_mean_reward, mean_100ep_reward))
U.save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
print "Restored model with mean reward: {}".format(
saved_mean_reward)
# logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
U.load_state(model_file)
return ActWrapper(act, act_params)
def __init__(self,
*,
scope,
ob_space,
ac_space,
stochpol_fn,
nsteps,
nepochs=4,
nminibatches=1,
gamma=0.99,
gamma_ext=0.99,
lam=0.95,
ent_coef=0,
cliprange=0.2,
max_grad_norm=1.0,
vf_coef=1.0,
lr=30e-5,
adam_hps=None,
testing=False,
comm=None,
comm_train=None,
use_news=False,
update_ob_stats_every_step=True,
int_coeff=None,
ext_coeff=None,
restore_model_path=None):
self.lr = lr
self.ext_coeff = ext_coeff
self.int_coeff = int_coeff
self.use_news = use_news
self.update_ob_stats_every_step = update_ob_stats_every_step
self.abs_scope = (tf.get_variable_scope().name + '/' +
scope).lstrip('/')
self.testing = testing
self.comm_log = MPI.COMM_SELF
if comm is not None and comm.Get_size() > 1:
self.comm_log = comm
assert not testing or comm.Get_rank(
) != 0, "Worker number zero can't be testing"
if comm_train is not None:
self.comm_train, self.comm_train_size = comm_train, comm_train.Get_size(
)
else:
self.comm_train, self.comm_train_size = self.comm_log, self.comm_log.Get_size(
)
self.is_log_leader = self.comm_log.Get_rank() == 0
self.is_train_leader = self.comm_train.Get_rank() == 0
with tf.variable_scope(scope):
self.best_ret = -np.inf
self.local_best_ret = -np.inf
self.rooms = []
self.local_rooms = []
self.scores = []
self.ob_space = ob_space
self.ac_space = ac_space
self.stochpol = stochpol_fn()
self.nepochs = nepochs
self.cliprange = cliprange
self.nsteps = nsteps
self.nminibatches = nminibatches
self.gamma = gamma
self.gamma_ext = gamma_ext
self.lam = lam
self.adam_hps = adam_hps or {}
self.ph_adv = tf.placeholder(tf.float32, [None, None])
self.ph_ret_int = tf.placeholder(tf.float32, [None, None])
self.ph_ret_ext = tf.placeholder(tf.float32, [None, None])
self.ph_oldnlp = tf.placeholder(tf.float32, [None, None])
self.ph_oldvpred = tf.placeholder(tf.float32, [None, None])
self.ph_lr = tf.placeholder(tf.float32, [])
self.ph_lr_pred = tf.placeholder(tf.float32, [])
self.ph_cliprange = tf.placeholder(tf.float32, [])
#Define loss; returns tf.nn.softmax_cross_entropy_with_logits_v2
neglogpac = self.stochpol.pd_opt.neglogp(self.stochpol.ph_ac)
entropy = tf.reduce_mean(self.stochpol.pd_opt.entropy())
vf_loss_int = (0.5 * vf_coef) * tf.reduce_mean(
tf.square(self.stochpol.vpred_int_opt - self.ph_ret_int))
vf_loss_ext = (0.5 * vf_coef) * tf.reduce_mean(
tf.square(self.stochpol.vpred_ext_opt - self.ph_ret_ext))
vf_loss = vf_loss_int + vf_loss_ext
ratio = tf.exp(self.ph_oldnlp - neglogpac) # p_new / p_old
negadv = -self.ph_adv
pg_losses1 = negadv * ratio
pg_losses2 = negadv * tf.clip_by_value(
ratio, 1.0 - self.ph_cliprange, 1.0 + self.ph_cliprange)
pg_loss = tf.reduce_mean(tf.maximum(pg_losses1, pg_losses2))
ent_loss = (-ent_coef) * entropy
approxkl = .5 * tf.reduce_mean(
tf.square(neglogpac - self.ph_oldnlp))
maxkl = .5 * tf.reduce_max(tf.square(neglogpac - self.ph_oldnlp))
clipfrac = tf.reduce_mean(
tf.to_float(tf.greater(tf.abs(ratio - 1.0),
self.ph_cliprange)))
loss = pg_loss + ent_loss + vf_loss + self.stochpol.aux_loss
#Create optimizer.
params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=self.abs_scope)
logger.info("PPO: using MpiAdamOptimizer connected to %i peers" %
self.comm_train_size)
trainer = MpiAdamOptimizer(self.comm_train,
learning_rate=self.ph_lr,
**self.adam_hps)
grads_and_vars = trainer.compute_gradients(loss, params)
grads, vars = zip(*grads_and_vars)
if max_grad_norm:
_, _grad_norm = tf.clip_by_global_norm(grads, max_grad_norm)
global_grad_norm = tf.global_norm(grads)
grads_and_vars = list(zip(grads, vars))
self._train = trainer.apply_gradients(grads_and_vars)
#Quantities for reporting.
self._losses = [
loss, pg_loss, vf_loss, entropy, clipfrac, approxkl, maxkl,
self.stochpol.aux_loss, self.stochpol.feat_var,
self.stochpol.max_feat, global_grad_norm
]
self.loss_names = [
'tot', 'pg', 'vf', 'ent', 'clipfrac', 'approxkl', 'maxkl',
"auxloss", "featvar", "maxfeat", "gradnorm"
]
self.I = None
self.disable_policy_update = None
allvars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope=self.abs_scope)
if self.is_log_leader:
tf_util.display_var_info(allvars)
model_path = os.path.join(logger.get_dir(), 'saved_model')
self.model_path = os.path.join(model_path, 'ppo.ckpt')
if restore_model_path:
tf_util.load_state(restore_model_path)
else:
#self.activate_graph_debugging()
tf.get_default_session().run(tf.variables_initializer(allvars))
#Syncs initialization across mpi workers.
sync_from_root(tf.get_default_session(), allvars)
self.t0 = time.time()
self.global_tcount = 0
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('expert_training_data', type=str)
parser.add_argument('--train', action='store_true')
parser.add_argument('--loadcheckpoint', type=str, default=None)
parser.add_argument('--savecheckpoint', type=str, default=None)
parser.add_argument('--summaries_dir', type=str, default="summaries/")
parser.add_argument('--num_epochs', type=int, default=100)
parser.add_argument('--minibatch_size', type=int, default=1000)
parser.add_argument('--nueral_net', action='store_true')
parser.add_argument('--render', action='store_true')
args = parser.parse_args()
env = gym.make("Humanoid-v1")
with tf.Session() as sess:
training_data = pickle.load(open(args.expert_training_data))
training_observations = training_data['observations']
training_actions = training_data['actions']
num_examples = training_observations.shape[0]
print "Loaded %d expert examples" % (num_examples)
obs_dim = training_observations.shape[1]
action_dim = training_actions.shape[2]
print "Observation dim %s" % obs_dim
print "Action dim %s" % action_dim
mean = np.mean(training_observations, axis=0)
std = np.std(training_observations, axis=0) + 1e-6
high = env.action_space.high
low = env.action_space.low
policy = None
if args.nueral_net:
policy = NueralNet(obs_dim, action_dim, hidden_dims=[64, 64])
else:
policy = AffinePolicy(obs_dim, action_dim)
merged_summaries = tf.summary.merge_all()
summary_writer = tf.train.SummaryWriter(args.summaries_dir, sess.graph)
tf_util.eval(tf.global_variables_initializer())
if args.loadcheckpoint:
tf_util.load_state(args.loadcheckpoint)
if args.train:
training_iterations = args.num_epochs * num_examples / args.minibatch_size
for t in xrange(training_iterations):
batch_idxs = np.random.choice(np.arange(num_examples),
args.minibatch_size)
batch_actions = training_actions[batch_idxs]
batch_observations = (training_observations[batch_idxs] -
mean) / std
feed_dict = {
policy.input: batch_observations,
policy.targets: batch_actions
}
loss = 0.0
if t % 10 == 0:
_, loss, summary = tf_util.eval(
[policy.optimizer, policy.loss, merged_summaries],
feed_dict=feed_dict)
summary_writer.add_summary(summary, t)
else:
_, loss = tf_util.eval([policy.optimizer, policy.loss],
feed_dict=feed_dict)
if (t * args.minibatch_size) % num_examples == 0:
epoch_number = int(t * args.minibatch_size / num_examples)
print "Epoch: %d, loss: %s" % (epoch_number,
loss / args.minibatch_size)
reward = run_policy(env,
policy,
render=args.render,
mean=mean,
std=std)
print "Reward: %s" % reward
if args.savecheckpoint:
tf_util.save_state(args.savecheckpoint)
print "Finished training after %s epochs" % (args.num_epochs)
trial_total_rewards = []
for i in xrange(1000):
if i % 100 == 0:
print "Collecting policy performance %s..." % (i)
trial_total_rewards.append(run_policy(env, policy))
trial_total_rewards = np.array(trial_total_rewards)
print "Policy results"
print "Mean: %s, Std: %s" % (trial_total_rewards.mean(),
trial_total_rewards.std())
def load_test(*, env_id, num_env, hps, num_timesteps, seed, fname):
venv = VecFrameStack(
make_atari_env(env_id,
num_env,
seed,
wrapper_kwargs=dict(),
start_index=num_env * MPI.COMM_WORLD.Get_rank(),
max_episode_steps=hps.pop('max_episode_steps')),
hps.pop('frame_stack'))
# venv.score_multiple = {'Mario': 500,
# 'MontezumaRevengeNoFrameskip-v4': 100,
# 'GravitarNoFrameskip-v4': 250,
# 'PrivateEyeNoFrameskip-v4': 500,
# 'SolarisNoFrameskip-v4': None,
# 'VentureNoFrameskip-v4': 200,
# 'PitfallNoFrameskip-v4': 100,
# }[env_id]
venv.score_multiple = 1
venv.record_obs = True
ob_space = venv.observation_space
ac_space = venv.action_space
gamma = hps.pop('gamma')
policy = {'rnn': CnnGruPolicy, 'cnn': CnnPolicy}[hps.pop('policy')]
agent = PpoAgent(
scope='ppo',
ob_space=ob_space,
ac_space=ac_space,
stochpol_fn=functools.partial(
policy,
scope='pol',
ob_space=ob_space,
ac_space=ac_space,
update_ob_stats_independently_per_gpu=hps.pop(
'update_ob_stats_independently_per_gpu'),
proportion_of_exp_used_for_predictor_update=hps.pop(
'proportion_of_exp_used_for_predictor_update'),
dynamics_bonus=hps.pop("dynamics_bonus")),
gamma=gamma,
gamma_ext=hps.pop('gamma_ext'),
lam=hps.pop('lam'),
nepochs=hps.pop('nepochs'),
nminibatches=hps.pop('nminibatches'),
lr=hps.pop('lr'),
cliprange=0.1,
nsteps=128,
ent_coef=0.001,
max_grad_norm=hps.pop('max_grad_norm'),
use_news=hps.pop("use_news"),
comm=MPI.COMM_WORLD if MPI.COMM_WORLD.Get_size() > 1 else None,
update_ob_stats_every_step=hps.pop('update_ob_stats_every_step'),
int_coeff=hps.pop('int_coeff'),
ext_coeff=hps.pop('ext_coeff'),
obs_save_flag=True)
tf_util.load_state("saved_states/save1")
agent.start_interaction([venv])
counter = 0
while True:
info = agent.step()
if agent.I.stats['epcount'] > 1:
with open("obs_acs.pickle", 'wb') as f1:
pickle.dump(agent.obs_rec, f1)
break
def learn(env,
q_func,
alpha=1e-5,
num_cpu=1,
n_steps=100000,
update_target_every=500,
train_main_every=1,
print_every=50,
checkpoint_every=10000,
buffer_size=50000,
gamma=1.0,
batch_size=32,
param_noise=False,
pre_run_steps=1000,
exploration_fraction=0.1,
final_epsilon=0.1,
callback=None):
"""
:param env: gym.Env, environment from OpenAI
:param q_func: (tf.Variable, int, str, bool) -> tf.Variable
the q function takes the following inputs:
input_ph: tf.placeholder, network input
n_actions: int, number of possible actions
scope: str, specifying the variable scope
reuse: bool, whether to reuse the variable given in `scope`
:param alpha: learning rate
:param num_cpu: number of cpu to use
:param n_steps: number of training steps
:param update_target_every: frequency to update the target network
:param train_main_every: frequency to update(train) the main network
:param print_every: how often to print message to console
:param checkpoint_every: how often to save the model.
:param buffer_size: size of the replay buffer
:param gamma: int, discount factor
:param batch_size: int, size of the input batch
:param param_noise: bool, whether to use parameter noise
:param pre_run_steps: bool, pre-run steps to fill in the replay buffer. And only
after `pre_run_steps` steps, will the main and target network begin to update.
:param exploration_fraction: float, between 0 and 1. Fraction of the `n_steps` to
linearly decrease the epsilon. After that, the epsilon will remain unchanged.
:param final_epsilon: float, final epsilon value, usually a very small number
towards zero.
:param callback: (dict, dict) -> bool
a function to decide whether it's time to stop training, takes following inputs:
local_vars: dict, the local variables in the current scope
global_vars: dict, the global variables in the current scope
:return: ActWrapper, a callable function
"""
n_actions = env.action_space.n
sess = U.make_session(num_cpu)
sess.__enter__()
def make_obs_ph(name):
return U.BatchInput(env.observation_space.shape, name=name)
act, train, update_target, debug = build_train(
make_obs_ph,
q_func,
n_actions,
optimizer=tf.train.AdamOptimizer(alpha),
gamma=gamma,
param_noise=param_noise,
grad_norm_clipping=10)
act_params = {
"q_func": q_func,
"n_actions": env.action_space.n,
"make_obs_ph": make_obs_ph,
}
buffer = ReplayBuffer(buffer_size)
exploration = LinearSchedule(schedule_steps=int(exploration_fraction *
n_steps),
final_p=final_epsilon,
initial_p=1.0)
# writer = tf.summary.FileWriter("./log", sess.graph)
U.initialize()
# writer.close()
update_target() # copy from the main network
episode_rewards = []
current_episode_reward = 0.0
model_saved = False
saved_mean_reward = 0.0
obs_t = env.reset()
with tempfile.TemporaryDirectory() as td:
model_file_path = os.path.join(td, "model")
for step in range(n_steps):
if callback is not None:
if callback(locals(), globals()):
break
kwargs = {}
if not param_noise:
epsilon = exploration.value(step)
else:
assert False, "Not implemented"
action = act(np.array(obs_t)[None], epsilon=epsilon, **kwargs)[0]
obs_tp1, reward, done, _ = env.step(action)
current_episode_reward += reward
buffer.add(obs_t, action, reward, obs_tp1, done)
obs_t = obs_tp1
if done:
obs_t = env.reset()
episode_rewards.append(current_episode_reward)
current_episode_reward = 0.0
# given sometime to fill in the buffer
if step < pre_run_steps:
continue
# q_value = debug["q_values"]
# if step % 1000 == 0:
# print(q_value(np.array(obs_t)[None]))
if step % train_main_every == 0:
obs_ts, actions, rewards, obs_tp1s, dones = buffer.sample(
batch_size)
weights = np.ones_like(dones)
td_error = train(obs_ts, actions, rewards, obs_tp1s, dones,
weights)
if step % update_target_every == 0:
update_target()
mean_100eps_reward = float(np.mean(episode_rewards[-101:-1]))
if done and print_every is not None and len(
episode_rewards) % print_every == 0:
print(
"step %d, episode %d, epsilon %.2f, running mean reward %.2f"
%
(step, len(episode_rewards), epsilon, mean_100eps_reward))
if checkpoint_every is not None and step % checkpoint_every == 0:
if saved_mean_reward is None or mean_100eps_reward > saved_mean_reward:
U.save_state(model_file_path)
model_saved = True
if print_every is not None:
print(
"Dump model to file due to mean reward increase: %.2f -> %.2f"
% (saved_mean_reward, mean_100eps_reward))
saved_mean_reward = mean_100eps_reward
if model_saved:
U.load_state(model_file_path)
if print_every:
print("Restore model from file with mean reward %.2f" %
(saved_mean_reward, ))
return ActWrapper(act, act_params)
def learn(env,
q_func,
lr=1e-2,
max_timesteps=1000000,
buffer_size=50000,
exploration_fraction=1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
def make_obs_ph(name):
return ObservationInput(env.observation_space, name=name)
act, train, update_target, debug = build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
#exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
# initial_p=0.7,
# final_p=0.15)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_state(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps,
**kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
#logger.record_tabular("replay buffer size", replay_buffer.__len__())
logger.dump_tabular()
#if done and num_episodes % 100 == 1:
# filehandler = open("cartpole_MDP_replay_buffer.obj","wb")
# pickle.dump(replay_buffer,filehandler)
# filehandler.close()
# print('MDP model samples saved',replay_buffer.__len__())
# file = open("cartpole_MDP_replay_buffer.obj",'rb')
# reloaded_replay_buffer = pickle.load(file)
# file.close()
# print('MDP model samples loaded',reloaded_replay_buffer.__len__())
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_state(model_file)
#file = open("cartpole_MDP_replay_buffer.obj",'rb')
#reloaded_replay_buffer = pickle.load(file)
#file.close()
#reloaded_replay_buffer.__len__()
filehandler = open("cartpole_MDP_replay_buffer.obj", "wb")
pickle.dump(replay_buffer, filehandler)
filehandler.close()
print('MDP model samples saved', replay_buffer.__len__())
file = open("cartpole_MDP_replay_buffer.obj", 'rb')
reloaded_replay_buffer = pickle.load(file)
file.close()
print('MDP model samples loaded', reloaded_replay_buffer.__len__())
return act
