def main(env_name, num_episodes, gamma, lam, kl_targ, batch_size, nprocs,
policy_hid_list, valfunc_hid_list, gpu_pct, restore_path, animate,
submit):
""" Main training loop
Args:
env_name: OpenAI Gym environment name, e.g. 'Hopper-v1'
num_episodes: maximum number of episodes to run
gamma: reward discount factor (float)
lam: lambda from Generalized Advantage Estimate
kl_targ: D_KL target for policy update [D_KL(pi_old || pi_new)
batch_size: number of episodes per policy training batch
"""
# killer = GracefulKiller()
env, obs_dim, act_dim = init_osim(animate)
env.seed(111 + mpi_util.rank)
mpi_util.set_global_seeds(111 + mpi_util.rank)
obs_dim += 1 # add 1 to obs dimension for time step feature (see run_episode())
now = datetime.utcnow().strftime(
"%b-%d_%H:%M:%S") # create unique directories
if mpi_util.rank == 0:
#aigym_path = os.path.join('/tmp', env_name, now)
#env = wrappers.Monitor(env, aigym_path, force=True)
logger = Logger(logname=env_name, now=now)
episode = 0
checkpoint = Checkpoint("saves", now)
# restore from checkpoint?
if restore_path:
(policy, val_func, scaler, episode, obs_dim, act_dim,
kl_targ) = checkpoint.restore(restore_path)
else:
policy = Policy(obs_dim, act_dim, kl_targ)
val_func = NNValueFunction(obs_dim)
scaler = Scaler(obs_dim)
if mpi_util.rank == 0:
# run a few episodes (on node 0) of untrained policy to initialize scaler:
trajectories = run_policy(env, policy, scaler, episodes=5)
unscaled = np.concatenate(
[t['unscaled_obs'] for t in trajectories])
scaler.update(
unscaled) # update running statistics for scaling observations
# broadcast policy weights, scaler, val_func
(policy, scaler, val_func) = mpi_util.broadcast_policy_scaler_val(
policy, scaler, val_func)
if mpi_util.rank == 0:
checkpoint.save(policy, val_func, scaler, episode)
if animate:
observes, actions, rewards, unscaled_obs = run_episode(env,
policy,
scaler,
animate=animate)
exit(0)
if submit:
# Settings
#remote_base = 'http://grader.crowdai.org:1729'
remote_base = 'http://grader.crowdai.org:1730'
token = 'a83412a94593cae3a491f3ee28ff44e1'
client = Client(remote_base)
# Create environment
observation = client.env_create(token)
step = 0.0
observes, actions, rewards, unscaled_obs = [], [], [], []
scale, offset = scaler.get()
scale[-1] = 1.0 # don't scale time step feature
offset[-1] = 0.0 # don't offset time step feature
# Run a single step
#
# The grader runs 3 simulations of at most 1000 steps each. We stop after the last one
while True:
obs = np.array(observation).astype(np.float32).reshape((1, -1))
print("OBSERVATION TYPE:", type(obs), obs.shape)
print(obs)
obs = np.append(obs, [[step]], axis=1) # add time step feature
unscaled_obs.append(obs)
obs = (obs - offset) * scale # center and scale observations
observes.append(obs)
action = policy.sample(obs).astype(np.float32).reshape((-1, 1))
print("ACTION TYPE:", type(action), action.shape)
print(action)
actions.append(action)
[observation, reward, done,
info] = client.env_step(action.tolist())
print("step:", step, "reward:", reward)
if not isinstance(reward, float):
reward = np.asscalar(reward)
rewards.append(reward)
step += 1e-3 # increment time step feature
if done:
print(
"================================== RESTARTING ================================="
)
observation = client.env_reset()
step = 0.0
observes, actions, rewards, unscaled_obs = [], [], [], []
scale, offset = scaler.get()
scale[-1] = 1.0 # don't scale time step feature
offset[-1] = 0.0 # don't offset time step feature
if not observation:
break
client.submit()
exit(0)
######
worker_batch_size = int(batch_size / mpi_util.nworkers) # HACK
if (worker_batch_size * mpi_util.nworkers != batch_size):
print("batch_size:", batch_size, " is not divisible by nworkers:",
mpi_util.nworkers)
exit(1)
batch = 0
while episode < num_episodes:
if mpi_util.rank == 0 and batch > 0 and batch % 10 == 0:
checkpoint.save(policy, val_func, scaler, episode)
batch = batch + 1
trajectories = run_policy(env,
policy,
scaler,
episodes=worker_batch_size)
trajectories = mpi_util.gather_trajectories(trajectories)
if mpi_util.rank == 0:
# concatentate trajectories into one list
trajectories = list(itertools.chain.from_iterable(trajectories))
print("did a batch of ", len(trajectories), " trajectories")
print([t['rewards'].sum() for t in trajectories])
episode += len(trajectories)
add_value(trajectories,
val_func) # add estimated values to episodes
add_disc_sum_rew(trajectories,
gamma) # calculated discounted sum of Rs
add_gae(trajectories, gamma, lam) # calculate advantage
# concatenate all episodes into single NumPy arrays
observes, actions, advantages, disc_sum_rew = build_train_set(
trajectories)
# add various stats to training log:
logger.log({
'_MeanReward':
np.mean([t['rewards'].sum() for t in trajectories]),
'Steps':
np.sum([t['observes'].shape[0] for t in trajectories])
})
log_batch_stats(observes, actions, advantages, disc_sum_rew,
logger, episode)
policy.update(observes, actions, advantages,
logger) # update policy
val_func.fit(observes, disc_sum_rew,
logger) # update value function
unscaled = np.concatenate(
[t['unscaled_obs'] for t in trajectories])
scaler.update(
unscaled) # update running statistics for scaling observations
logger.write(
display=True) # write logger results to file and stdout
# if mpi_util.rank == 0 and killer.kill_now:
# if input('Terminate training (y/[n])? ') == 'y':
# break
# killer.kill_now = False
# broadcast policy weights, scaler, val_func
(policy, scaler, val_func) = mpi_util.broadcast_policy_scaler_val(
policy, scaler, val_func)
if mpi_util.rank == 0: logger.close()
policy.close_sess()
if mpi_util.rank == 0: val_func.close_sess()
