def __init__(self, env, num_agents, config):
self.num_agent = num_agents
self.config = config
self.env = env
self.observation_space = env.observation_space
self.action_space = env.action_space
self.is_rnn = config["network"] == "rnn"
self.is_cnn = config["network"] == "cnn"
config["num_agent"] = num_agents
if isinstance(self.observation_space, gym.spaces.Box):
if config["share_policy"]:
pg_agent = PGAgent(
"shared",
self.observation_space,
gym.spaces.Discrete(self.action_space.shape[0] /
self.num_agent),
config,
)
self.agents = [pg_agent for i in range(num_agents)]
else:
self.agents = [
PGAgent(
i,
self.observation_space,
gym.spaces.Discrete(self.action_space.shape[0] /
self.num_agent),
config,
) for i in range(num_agents)
]
else:
if config["share_policy"]:
pg_agent = PGAgent(
"shared",
self.observation_space[0],
self.action_space[0],
config,
)
self.agents = [pg_agent for i in range(num_agents)]
else:
self.agents = [
PGAgent(i, self.observation_space[i], self.action_space[i],
config) for i in range(num_agents)
]
self.local_steps_per_epoch = int(config["steps_per_epoch"] /
num_procs())
if type(self.action_space[0]) is gym.spaces.Discrete:
action_dim = 1
else:
action_dim = self.action_space[0].shape[0]
if type(self.observation_space[0]) is gym.spaces.Discrete:
obs_dim = self.observation_space[0].n
else:
if self.is_cnn:
obs_dim = self.observation_space[0].shape
else:
obs_dim = self.observation_space[0].shape[0]
self.buf = new_buffer(
num_agents,
obs_dim,
action_dim,
size=self.local_steps_per_epoch,
type=config["algo"],
is_rnn=self.is_rnn,
is_cnn=self.is_cnn,
rnn_length=config["rnn_length"],
)
# init logger
self.logger = Logger(self.config)
# init session and sync params
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
self.sess.run(sync_all_params())
tf.io.write_graph(
graph_or_graph_def=self.sess.graph_def,
logdir=os.path.join(self.config["output_dir"]),
name="model",
)
# save model
self.savers = []
with tf.device('/cpu:0'):
for i in range(num_agents):
vars_list = tf.compat.v1.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope="Agent_{}".format(i))
self.savers.append(tf.train.Saver(vars_list, max_to_keep=100))
if self.config["save_path"] is None:
self.save_path = "/tmp/agents.pickle"
else:
self.save_path = self.config["save_path"]
self.save_path += 's/'
if not os.path.exists(self.save_path):
os.makedirs(self.save_path)
# load model
if self.config["load_model"]:
self.restore(self.config["restore_path"], [0, 1])
def ppo(env_fn, actor_critic=core.MLPActorCritic, ac_kwargs=dict(), seed=0,
steps_per_epoch=4000, epochs=50, gamma=0.99, clip_ratio=0.2, pi_lr=3e-4,
vf_lr=1e-3, train_pi_iters=80, train_v_iters=80, lam=0.97, max_ep_len=1000,
target_kl=0.01, logger_kwargs=dict(), save_freq=10):
"""
Proximal Policy Optimization (by clipping),
with early stopping based on approximate KL
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The constructor method for a PyTorch Module with a
``step`` method, an ``act`` method, a ``pi`` module, and a ``v``
module. The ``step`` method should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``a`` (batch, act_dim) | Numpy array of actions for each
| observation.
``v`` (batch,) | Numpy array of value estimates
| for the provided observations.
``logp_a`` (batch,) | Numpy array of log probs for the
| actions in ``a``.
=========== ================ ======================================
The ``act`` method behaves the same as ``step`` but only returns ``a``.
The ``pi`` module's forward call should accept a batch of
observations and optionally a batch of actions, and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` N/A | Torch Distribution object, containing
| a batch of distributions describing
| the policy for the provided observations.
``logp_a`` (batch,) | Optional (only returned if batch of
| actions is given). Tensor containing
| the log probability, according to
| the policy, of the provided actions.
| If actions not given, will contain
| ``None``.
=========== ================ ======================================
The ``v`` module's forward call should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``v`` (batch,) | Tensor containing the value estimates
| for the provided observations. (Critical:
| make sure to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object
you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.) Typically
denoted by :math:`\epsilon`.
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
# Special function to avoid certain slowdowns from PyTorch + MPI combo.
setup_pytorch_for_mpi()
# Set up logger and save configuration
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# Random seed
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# Instantiate environment
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Create actor-critic module
ac = actor_critic(env.observation_space, env.action_space, **ac_kwargs)
# Sync params across processes
sync_params(ac)
# Count variables
var_counts = tuple(core.count_vars(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n'%var_counts)
# Set up experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Set up function for computing PPO policy loss
def compute_loss_pi(data):
obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data['logp']
# Policy loss
pi, logp = ac.pi(obs, act)
ratio = torch.exp(logp - logp_old)
clip_adv = torch.clamp(ratio, 1-clip_ratio, 1+clip_ratio) * adv
loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()
# Useful extra info
approx_kl = (logp_old - logp).mean().item()
ent = pi.entropy().mean().item()
clipped = ratio.gt(1+clip_ratio) | ratio.lt(1-clip_ratio)
clipfrac = torch.as_tensor(clipped, dtype=torch.float32).mean().item()
pi_info = dict(kl=approx_kl, ent=ent, cf=clipfrac)
return loss_pi, pi_info
# Set up function for computing value loss
def compute_loss_v(data):
obs, ret = data['obs'], data['ret']
return ((ac.v(obs) - ret)**2).mean()
# Set up optimizers for policy and value function
pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)
# Set up model saving
logger.setup_pytorch_saver(ac)
def update():
data = buf.get()
pi_l_old, pi_info_old = compute_loss_pi(data)
pi_l_old = pi_l_old.item()
v_l_old = compute_loss_v(data).item()
# Train policy with multiple steps of gradient descent
for i in range(train_pi_iters):
pi_optimizer.zero_grad()
loss_pi, pi_info = compute_loss_pi(data)
kl = mpi_avg(pi_info['kl'])
if kl > 1.5 * target_kl:
logger.log('Early stopping at step %d due to reaching max kl.'%i)
break
loss_pi.backward()
mpi_avg_grads(ac.pi) # average grads across MPI processes
pi_optimizer.step()
logger.store(StopIter=i)
# Value function learning
for i in range(train_v_iters):
vf_optimizer.zero_grad()
loss_v = compute_loss_v(data)
loss_v.backward()
mpi_avg_grads(ac.v) # average grads across MPI processes
vf_optimizer.step()
# Log changes from update
kl, ent, cf = pi_info['kl'], pi_info_old['ent'], pi_info['cf']
logger.store(LossPi=pi_l_old, LossV=v_l_old,
KL=kl, Entropy=ent, ClipFrac=cf,
DeltaLossPi=(loss_pi.item() - pi_l_old),
DeltaLossV=(loss_v.item() - v_l_old))
# Prepare for interaction with environment
start_time = time.time()
o, ep_ret, ep_len = env.reset(), 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v, logp = ac.step(torch.as_tensor(o, dtype=torch.float32))
next_o, r, d, _ = env.step(a)
ep_ret += r
ep_len += 1
# save and log
buf.store(o, a, r, v, logp)
logger.store(VVals=v)
# Update obs (critical!)
o = next_o
timeout = ep_len == max_ep_len
terminal = d or timeout
epoch_ended = t==local_steps_per_epoch-1
if terminal or epoch_ended:
if epoch_ended and not(terminal):
print('Warning: trajectory cut off by epoch at %d steps.'%ep_len, flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
if timeout or epoch_ended:
_, v, _ = ac.step(torch.as_tensor(o, dtype=torch.float32))
else:
v = 0
buf.finish_path(v)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs-1):
logger.save_state({'env': env}, None)
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch+1)*steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time()-start_time)
logger.dump_tabular()
def ppo(env_fn,
actor_critic=core.MLPActorCritic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=80,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10):
# Special function to avoid certain slowdowns from PyTorch + MPI combination
setup_pytorch_for_mpi()
# Setup logger and save configuration
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# Random Seed
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# Instantiate Environment
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Create actor - critic module
ac = actor_critic(env.observation_space, env.action_space, **ac_kwargs)
# Sync parameters across processes
sync_params(ac)
# Count variables
var_counts = tuple(
core.count_variables(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# Set up experiment buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Set up a function for computing PPO Policy loss
def compute_loss_pi(data):
obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data[
'logp']
# Policy Loss
pi, log_p = ac.pi(obs, act)
ratio = torch.exp(log_p - logp_old)
clip_adv = torch.clamp(ratio, 1 - clip_ratio, 1 + clip_ratio) * adv
loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()
# Useful Extra Information
approx_kl = (logp_old - log_p).mean().item()
ent = pi.entropy().mean().item()
clipped = ratio.gt(1 + clip_ratio) | ratio.lt(1 - clip_ratio)
clip_fraction = torch.as_tensor(clipped,
dtype=torch.float32).mean().item()
pi_info = dict(kl=approx_kl, ent=ent, cf=clip_fraction)
return loss_pi, pi_info
# Setup function for computing value loss
def compute_loss_v(data):
obs, ret = data['obs'], data['ret']
return ((ac.v(obs) - ret)**2).mean()
# Setup optimizers for policy and value functions
pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)
# Set up model saving
logger.setup_pytorch_saver(ac)
def update():
data = buf.get()
pi_l_old, pi_info_old = compute_loss_pi(data)
pi_l_old = pi_l_old.item()
v_l_old = compute_loss_v(data).item()
# Train policy with multiple steps of gradient descent
for i in range(train_pi_iters):
pi_optimizer.zero_grad()
loss_pi, pi_info = compute_loss_pi(data)
kl = mpi_avg(pi_info['kl'])
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' % i)
break
loss_pi.backward()
mpi_avg_grads(ac.pi)
pi_optimizer.step()
logger.store(StopIter=i)
# Value function learning
for i in range(train_v_iters):
vf_optimizer.zero_grad()
loss_v = compute_loss_v(data)
loss_v.backward()
mpi_avg_grads(ac.v)
vf_optimizer.step()
# Log changes from update
kl, ent, cf = pi_info['kl'], pi_info_old['ent'], pi_info['cf']
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(loss_pi.item() - pi_l_old),
DeltaLossV=(loss_v.item() - v_l_old))
# Prepare for interaction with the environment
start_time = time.time()
o, ep_ret, ep_len = env.reset(), 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v, logp = ac.step(torch.as_tensor(o, dtype=torch.float32))
next_o, r, d, _ = env.step(a)
ep_ret += r
ep_len += 1
# save and log
buf.store(o, a, r, v, logp)
logger.store(VVals=v)
# Update obs(critical!)
o = next_o
timeout = ep_len == max_ep_len
terminal = d or time_out
epoch_ended = t == local_steps_per_epoch - 1
if terminal or epoch_ended:
if epoch_ended and not terminal:
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len,
flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
if timeout or epoch_ended:
_, v, _ = ac.step(torch.as_tensor(o, dtype=torch.float32))
else:
v = 0
buf.finish_path(v)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo_pyco(gym_or_pyco,
env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=473,
steps_per_epoch=200,
epochs=500,
gamma=0.99,
clip_ratio=0.1,
pi_lr=1e-2,
vf_lr=5e-3,
train_pi_iters=80,
train_v_iters=80,
lam=0.97,
max_ep_len=350,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10,
tensorboard_path='/home/clement/spinningup/tensorboard'):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
tensorboard_path: The path to the saved graphs&scalars in tensorboard
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
#seed += 10000 * proc_id()
seed = 2808
tf.set_random_seed(seed)
np.random.seed(seed)
dict_continous_gym = [
'CarRacing', 'LunarLander', 'Pong', 'AirRaid', 'Adventure', 'AirRaid',
'Alien', 'Amidar', 'Assault', 'Asterix', 'Asteroids', 'Atlantis',
'BankHeist', 'BattleZone', 'BeamRider', 'Berzerk', 'Bowling', 'Boxing',
'Breakout', 'Carnival', 'Centipede', 'ChopperCommand', 'CrazyClimber',
'Defender', 'Demon_attack', 'DoubleDunk', 'ElevatorAction', 'Enduro',
'FishingDerby', 'Freeway', 'Frostbite', 'Gopher', 'Gravitar', 'Hero',
'IceHockey', 'Jamesbond', 'JourneyEscape', 'Kangaroo', 'Krull',
'KungFuMaster', 'MpntezumaRevenge', 'MsPacman', 'NameThisGame',
'Phoenix', 'Pitfall', 'Pooyan', 'PrivateEye', 'Qbert', 'Riverraid',
'RoadRunner', 'Robotank', 'Seaquest', 'Skiing', 'Solaris',
'SpaceInvaders', 'StarGunner', 'Tennis', 'TimePilot', 'Tutankham',
'UpNDown', 'Venture', 'VideoPinball', 'WizardOfWor', 'VarsRevenge',
'Zaxxon', 'Numberlink'
]
dict_discrete_gym = []
env = env_fn()
dict_gym = [
'CarRacing', 'LunarLander', 'Pong', 'AirRaid', 'Adventure', 'AirRaid',
'Alien', 'Amidar', 'Assault', 'Asterix', 'Asteroids', 'Atlantis',
'BankHeist', 'BattleZone', 'BeamRider', 'Berzerk', 'Bowling', 'Boxing',
'Breakout', 'Carnival', 'Centipede', 'ChopperCommand', 'CrazyClimber',
'Defender', 'Demon_attack', 'DoubleDunk', 'ElevatorAction', 'Enduro',
'FishingDerby', 'Freeway', 'Frostbite', 'Gopher', 'Gravitar', 'Hero',
'IceHockey', 'Jamesbond', 'JourneyEscape', 'Kangaroo', 'Krull',
'KungFuMaster', 'MpntezumaRevenge', 'MsPacman', 'NameThisGame',
'Phoenix', 'Pitfall', 'Pooyan', 'PrivateEye', 'Qbert', 'Riverraid',
'RoadRunner', 'Robotank', 'Seaquest', 'Skiing', 'Solaris',
'SpaceInvaders', 'StarGunner', 'Tennis', 'TimePilot', 'Tutankham',
'UpNDown', 'Venture', 'VideoPinball', 'WizardOfWor', 'VarsRevenge',
'Zaxxon', 'Numberlink'
]
# This code is specific for pycolab
if gym_or_pyco == 'gym':
None
else:
env = env()
obs_dim = env.observation_space.shape
if env.action_space == 4:
act_dim = env.action_space
try:
act_dim = env.action_space.n
except:
act_dim = env.action_space.shape
#act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
# x_ph, a_ph = core.placeholders_from_spaces(env.observation_space, env.action_space)
if gym_or_pyco == 'pyco':
x_ph = tf.placeholder(tf.float32,
shape=(None, obs_dim[0], obs_dim[1], 1))
else:
x_ph = tf.placeholder(tf.float32,
shape=(None, obs_dim[0], obs_dim[1], obs_dim[2]))
# a_ph = core.placeholders_from_spaces(env.action_space)
if gym_or_pyco == 'gym' and isinstance(env.action_space, Discrete):
a_ph = tf.placeholder(tf.uint8, shape=(None))
elif gym_or_pyco == 'gym' and isinstance(env.action_space, Box):
a_ph = tf.placeholder(tf.float32, shape=(env.action_space.shape[0]))
else:
a_ph = tf.placeholder(tf.uint8, shape=(None))
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
# pi, logp, logp_pi, v, logits = actor_critic(x_ph, a_ph, **ac_kwargs)
# actor_critic with relational policy
# pi, logp, logp_pi, v, logits = actor_critic(x_ph, a_ph, policy='relational_categorical_policy', action_space = env.action_space.n, **ac_kwargs)
if gym_or_pyco == 'gym' and isinstance(env.action_space, Discrete):
pi, logp, logp_pi, v, logits = actor_critic(
x_ph,
a_ph,
policy='baseline_categorical_policy',
action_space=env.action_space.n)
elif gym_or_pyco == 'gym' and isinstance(env.action_space, Box):
pi, logp, logp_pi, v = actor_critic(
x_ph,
a_ph,
policy='baseline_gaussian_policy',
action_space=env.action_space.shape[0])
else:
pi, logp, logp_pi, v, logits = actor_critic(
x_ph,
a_ph,
policy='baseline_categorical_policy',
action_space=env.action_space.n)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(gym_or_pyco, obs_dim, act_dim, local_steps_per_epoch,
gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# PPO objectives
ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s)
min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph,
(1 - clip_ratio) * adv_ph)
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv))
# tensorboard test
with tf.name_scope('pi_loss'):
core.variable_summaries(pi_loss)
v_loss = tf.reduce_mean((ret_ph - v)**2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-logp) # a sample estimate for entropy, also easy to compute
clipped = tf.logical_or(ratio > (1 + clip_ratio), ratio < (1 - clip_ratio))
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32))
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
sess = tf.Session()
# sess = tf_debug.LocalCLIDebugWrapperSession(sess)
# tensorboard
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(tensorboard_path + '/train',
sess.graph)
test_writer = tf.summary.FileWriter(tensorboard_path + '/test')
sess.run(tf.global_variables_initializer())
# saver.restore(sess, "/home/clement/Documents/spinningup_instadeep/trained_params/model.ckpt")
#saver.restore(sess, "/home/clement/Documents/spinningup_instadeep/data/cmd_ppo_pyco/cmd_ppo_pyco_s0/simple_save")
#tf.reset_default_graph()
#export_dir = "/home/clement/Documents/spinningup_instadeep/data/cmd_ppo_pyco/cmd_ppo_pyco_s0/simple_save"
#tf.saved_model.loader.load(sess, ["serve"],export_dir)
#sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
#logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def update(epoch):
# inputs = {k:v for k,v in zip(all_phs, buf.get())}
if gym_or_pyco == 'gym' and isinstance(env.action_space, Discrete):
pi_l_old, v_l_old, ent = sess.run(
[pi_loss, v_loss, approx_ent],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
if gym_or_pyco == 'gym' and isinstance(env.action_space, Box):
pi_l_old, v_l_old, ent = sess.run(
[pi_loss, v_loss, approx_ent],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
else:
pi_l_old, v_l_old, ent = sess.run(
[pi_loss, v_loss, approx_ent],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
# pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent], feed_dict=inputs)
summary = tf.Summary(
value=[tf.Summary.Value(tag="loss", simple_value=pi_l_old)])
test_writer.add_summary(summary, epoch)
# Training
for i in range(train_pi_iters):
if gym_or_pyco == 'gym' and isinstance(env.action_space, Discrete):
_, kl = sess.run(
[train_pi, approx_kl],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' %
i)
break
if gym_or_pyco == 'gym' and isinstance(env.action_space, Box):
_, kl = sess.run(
[train_pi, approx_kl],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' %
i)
break
else:
_, kl = sess.run(
[train_pi, approx_kl],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' %
i)
break
logger.store(StopIter=i)
for _ in range(train_v_iters):
if gym_or_pyco == 'gym' and isinstance(env.action_space, Discrete):
sess.run(train_v,
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
if gym_or_pyco == 'gym' and isinstance(env.action_space, Box):
sess.run(train_v,
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
else:
#sess.run(train_v, feed_dict={logp_old_ph: buf.logp_buf, x_ph: o, a_ph: a, adv_ph: buf.adv_buf,
# ret_ph: buf.ret_buf})
sess.run(train_v,
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
# Log changes from update
if gym_or_pyco == 'gym' and isinstance(env.action_space, Discrete):
pi_l_new, v_l_new, kl, cf = sess.run(
[pi_loss, v_loss, approx_kl, clipfrac],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
if gym_or_pyco == 'gym' and isinstance(env.action_space, Box):
pi_l_new, v_l_new, kl, cf = sess.run(
[pi_loss, v_loss, approx_kl, clipfrac],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
else:
pi_l_new, v_l_new, kl, cf = sess.run(
[pi_loss, v_loss, approx_kl, clipfrac],
feed_dict={
logp_old_ph: buf.logp_buf,
x_ph: buf.obs_buf,
a_ph: buf.act_buf,
adv_ph: buf.adv_buf,
ret_ph: buf.ret_buf
})
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
if gym_or_pyco == 'gym':
o = o.reshape(1, obs_dim[0], obs_dim[1], obs_dim[2])
else:
o = rgb_input_pyco(o, obs_dim)
o = o.reshape(1, obs_dim[0], obs_dim[1], 1)
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
num_ep = 0
summary_ep = []
logits_debug = []
for t in range(local_steps_per_epoch):
# a, v_t, logp_t = sess.run(get_action_ops, feed_dict={x_ph: o.board.reshape(1, -1)})
a, v_t, logp_t = sess.run(get_action_ops, feed_dict={x_ph: o})
logits_debug.append(sess.run(logits, feed_dict={x_ph: o})[0])
# save and log
buf.store(o, a, r, v_t, logp_t)
# buf.store(o.board.reshape(1,-1), a, r, v_t, logp_t)
# obs, act, rew, val, logp
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
if gym_or_pyco == 'pyco':
o = rgb_input_pyco(o, obs_dim)
o = o.reshape(1, obs_dim[0], obs_dim[1], 1)
else:
o = o.reshape(1, obs_dim[0], obs_dim[1], obs_dim[2])
if r is None:
ep_ret += 0
r = 0
else:
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
num_ep += 1
last_val = r if d else sess.run(v, feed_dict={x_ph: o})
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
else:
last_val = 0
# if trajectory didn't reach terminal state, bootstrap value target
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
summary_ep = summary_ep + [ep_ret]
# summary = tf.Summary(value=[tf.Summary.Value(tag="mean_ep_ret", simple_value=summary_ep)])
# test_writer.add_summary(summary, num_ep)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
if gym_or_pyco == 'gym':
o = o.reshape(1, obs_dim[0], obs_dim[1], obs_dim[2])
else:
o = rgb_input_pyco(o, obs_dim)
o = o.reshape(1, obs_dim[0], obs_dim[1], 1)
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform PPO update!
min_summary_ep = min(summary_ep)
max_summary_ep = max(summary_ep)
summary_ep = np.mean(summary_ep)
value_summary = np.mean(buf.val_buf)
summary = tf.Summary(value=[
tf.Summary.Value(tag="mean_ep_ret", simple_value=summary_ep)
])
test_writer.add_summary(summary, epoch)
summary = tf.Summary(value=[
tf.Summary.Value(tag="min_ep_ret", simple_value=min_summary_ep)
])
test_writer.add_summary(summary, epoch)
summary = tf.Summary(value=[
tf.Summary.Value(tag="max_ep_ret", simple_value=max_summary_ep)
])
test_writer.add_summary(summary, epoch)
summary = tf.Summary(value=[
tf.Summary.Value(tag="mean_value", simple_value=value_summary)
])
test_writer.add_summary(summary, epoch)
update(epoch)
#saver = tf.train.Saver()
#save_path = saver.save(sess, "/home/clement/Documents/spinningup/trained_params/model.ckpt")
# If you want to reload saved variables :
# with tf.Session() as sess:
# Restore variables from disk.
# saver.restore(sess, "/home/clement/Documents/spinningup/trained_params/model.ckpt")
# since I changed my sess.run i have to reset the buffer myself:
buf.get()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def vpg(env,
hidden_sizes,
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
pi_lr=3e-4,
vf_lr=1e-3,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10):
"""
Vanilla Policy Gradient (with GAE-Lambda for advantage estimation)
Args:
env_fn : A function which creates a copy of the environment. The environment must satisfy the OpenAI Gym API.
actor_critic: The constructor method for a PyTorch Module with a ``step`` method, an ``act`` method, a ``pi`` module, and a ``v``
module. The ``step`` method should accept a batch of observations and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``a`` (batch, act_dim) | Numpy array of actions for each
| observation.
``v`` (batch,) | Numpy array of value estimates
| for the provided observations.
``logp_a`` (batch,) | Numpy array of log probs for the
| actions in ``a``.
=========== ================ ======================================
The ``act`` method behaves the same as ``step`` but only returns ``a``.
The ``pi`` module's forward call should accept a batch of observations and optionally a batch of actions, and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` N/A | Torch Distribution object, containing
| a batch of distributions describing
| the policy for the provided observations.
``logp_a`` (batch,) | Optional (only returned if batch of
| actions is given). Tensor containing
| the log probability, according to
| the policy, of the provided actions.
| If actions not given, will contain
| ``None``.
=========== ================ ======================================
The ``v`` module's forward call should accept a batch of observations and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``v`` (batch,) | Tensor containing the value estimates
| for the provided observations. (Critical:
| make sure to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object you provided to VPG.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs) for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_v_iters (int): Number of gradient descent steps to take on value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1, close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save the current policy and value function.
"""
# Special function to avoid certain slowdowns from PyTorch + MPI combo.
setup_pytorch_for_mpi()
# logger
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# random seeds
seed += 1000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# 环境
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# 创建模型
ac = core.MLPActorCritic(env.observation_space, env.action_space,
hidden_sizes)
# Sync params across processes
sync_params(ac)
# Count variables
var_counts = tuple(core.count_vars(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# Set up experience buffer. 如果有多个线程，每个线程的经验池长度为 local_steps_per_epoch
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = VPGBuffer(obs_dim,
act_dim,
size=local_steps_per_epoch,
gamma=gamma,
lam=lam)
# optimizer
pi_optimizer = torch.optim.Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = torch.optim.Adam(ac.v.parameters(), lr=vf_lr)
# setup model saving
# logger.setup_pytorch_for_mpi()
# interaction
start_time = time.time()
o, ep_ret, ep_len = env.reset(), 0, 0
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v, logp = ac.step(torch.as_tensor(
o, dtype=torch.float32)) # (act_dim,), (), ()
next_o, r, d, _ = env.step(a)
ep_ret += r
ep_len += 1
# save
buf.store(o, a, r, v, logp)
logger.store(VVals=v)
# update obs
o = next_o
timeout = ep_len == max_ep_len
terminal = d or timeout
epoch_ended = t == local_steps_per_epoch - 1
if terminal or epoch_ended: # timeout=True, terminal=True, epoch_ended=True/False
if epoch_ended and not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len,
flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
if timeout or epoch_ended:
_, v, _ = ac.step(torch.as_tensor(o, dtype=torch.float32))
else:
v = 0
buf.finish_path(v)
if terminal:
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0 # 重新初始化
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform VPG update!
update(buf, ac, train_v_iters, pi_optimizer, vf_optimizer, logger)
# # Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo(env_fn, actor_critic=core.MLPActorCritic, ac_kwargs=dict(), seed=0,
steps_per_epoch=4000, epochs=50, gamma=0.99, clip_ratio=0.2, pi_lr=3e-4,
vf_lr=1e-3, train_pi_iters=80, train_v_iters=80, lam=0.97, max_ep_len=1000,
target_kl=0.01, logger_kwargs=dict(), save_freq=10, resume=None,
reinitialize_optimizer_on_resume=True, render=False, notes='',
env_config=None, boost_explore=0, partial_net_load=False,
num_inputs_to_add=0, episode_cull_ratio=0, try_rollouts=0,
steps_per_try_rollout=0, take_worst_rollout=False, shift_advs_pct=0,
**kwargs):
"""
Proximal Policy Optimization (by clipping),
with early stopping based on approximate KL
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The constructor method for a PyTorch Module with a
``step`` method, an ``act`` method, a ``pi`` module, and a ``v``
module. The ``step`` method should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``a`` (batch, act_dim) | Numpy array of actions for each
| observation.
``v`` (batch,) | Numpy array of value estimates
| for the provided observations.
``logp_a`` (batch,) | Numpy array of log probs for the
| actions in ``a``.
=========== ================ ======================================
The ``act`` method behaves the same as ``step`` but only returns ``a``.
The ``pi`` module's forward call should accept a batch of
observations and optionally a batch of actions, and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` N/A | Torch Distribution object, containing
| a batch of distributions describing
| the policy for the provided observations.
``logp_a`` (batch,) | Optional (only returned if batch of
| actions is given). Tensor containing
| the log probability, according to
| the policy, of the provided actions.
| If actions not given, will contain
| ``None``.
=========== ================ ======================================
The ``v`` module's forward call should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``v`` (batch,) | Tensor containing the value estimates
| for the provided observations. (Critical:
| make sure to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object
you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.) Typically
denoted by :math:`\epsilon`.
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
resume (str): Path to directory with simple_save model info
you wish to resume from
reinitialize_optimizer_on_resume: (bool) Whether to initialize
training state in the optimizers, i.e. the individual learning
rates for weights in Adam
render: (bool) Whether to render the env during training. Useful for
checking that resumption of training caused visual performance
to carry over
notes: (str) Experimental notes on what this run is testing
env_config (dict): Environment configuration pass through
boost_explore (float): Amount to increase std of actions in order to
reinvigorate exploration.
partial_net_load (bool): Whether to partially load the network when
resuming. https://pytorch.org/tutorials/beginner/saving_loading_models.html#id4
num_inputs_to_add (int): Number of new inputs to add, if resuming and
partially loading a new network.
episode_cull_ratio (float): Ratio of bad episodes to cull
from epoch
try_rollouts (int): Number of times to sample actions
steps_per_try_rollout (int): Number of steps per attempted rollout
take_worst_rollout (bool): Use worst rollout in training
shift_advs_pct (float): Action should be better than this pct of actions
to be considered advantageous.
"""
config = deepcopy(locals())
# Special function to avoid certain slowdowns from PyTorch + MPI combo.
setup_pytorch_for_mpi()
# Random seed
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
import_custom_envs()
# Instantiate environment
env = env_fn()
if hasattr(env.unwrapped, 'configure_env'):
env.unwrapped.configure_env(env_config)
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
num_agents = getattr(env, 'num_agents', 1)
if hasattr(env.unwrapped, 'logger'):
print('Logger set by environment')
logger_kwargs['logger'] = env.unwrapped.logger
logger = EpochLogger(**logger_kwargs)
logger.add_key_stat('won')
logger.add_key_stat('trip_pct')
logger.add_key_stat('HorizonReturn')
logger.save_config(config)
# Create actor-critic module
ac = actor_critic(env.observation_space, env.action_space,
num_inputs_to_add=num_inputs_to_add, **ac_kwargs)
# Set up optimizers for policy and value function
pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)
# Resume
if resume is not None:
ac, pi_optimizer, vf_optimizer = get_model_to_resume(
resume, ac, pi_lr, vf_lr, reinitialize_optimizer_on_resume,
actor_critic, partial_net_load, num_inputs_to_add)
if num_inputs_to_add:
add_inputs(ac, ac_kwargs, num_inputs_to_add)
if boost_explore:
boost_exploration(ac, boost_explore)
# Count variables
var_counts = tuple(core.count_vars(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n'%var_counts)
# Set up experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = ppo_buffer_factory(obs_dim, act_dim, local_steps_per_epoch, gamma,
lam, num_agents, shift_advs_pct,
cull_ratio=episode_cull_ratio)
# Set up function for computing PPO policy loss
def compute_loss_pi(data):
obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data['logp']
# Policy loss
pi, logp = ac.pi(obs, act)
ratio = torch.exp(logp - logp_old)
clip_adv = torch.clamp(ratio, 1-clip_ratio, 1+clip_ratio) * adv
loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()
# Useful extra info
approx_kl = (logp_old - logp).mean().item()
ent = pi.entropy().mean().item()
clipped = ratio.gt(1+clip_ratio) | ratio.lt(1-clip_ratio)
clipfrac = torch.as_tensor(clipped, dtype=torch.float32).mean().item()
pi_info = dict(kl=approx_kl, ent=ent, cf=clipfrac)
return loss_pi, pi_info
# Set up function for computing value loss
def compute_loss_v(data):
obs, ret = data['obs'], data['ret']
return ((ac.v(obs) - ret)**2).mean()
# Sync params across processes
sync_params(ac)
# Set up model saving
logger.setup_pytorch_saver(ac)
def update():
data = buf.get()
pi_l_old, pi_info_old = compute_loss_pi(data)
pi_l_old = pi_l_old.item()
v_l_old = compute_loss_v(data).item()
# Train policy with multiple steps of gradient descent
for i in range(train_pi_iters):
pi_optimizer.zero_grad()
loss_pi, pi_info = compute_loss_pi(data)
kl = mpi_avg(pi_info['kl'])
if kl > 1.5 * target_kl:
logger.log('Early stopping at step %d due to reaching max kl.'%i)
break
loss_pi.backward()
mpi_avg_grads(ac.pi) # average grads across MPI processes
pi_optimizer.step()
logger.store(StopIter=i)
# Value function learning
for i in range(train_v_iters):
vf_optimizer.zero_grad()
loss_v = compute_loss_v(data)
loss_v.backward()
mpi_avg_grads(ac.v) # average grads across MPI processes
vf_optimizer.step()
# Log changes from update
kl, ent, cf = pi_info['kl'], pi_info_old['ent'], pi_info['cf']
logger.store(LossPi=pi_l_old, LossV=v_l_old,
KL=kl, Entropy=ent, ClipFrac=cf,
DeltaLossPi=(loss_pi.item() - pi_l_old),
DeltaLossV=(loss_v.item() - v_l_old))
# Prepare for interaction with environment
start_time = time.time()
o, r, d = reset(env)
effective_horizon = round(1 / (1 - gamma))
effective_horizon_rewards = []
for _ in range(num_agents):
effective_horizon_rewards.append(deque(maxlen=effective_horizon))
if hasattr(env, 'agent_index'):
agent_index = env.agent_index
agent = env.agents[agent_index]
is_multi_agent = True
else:
agent_index = 0
agent = None
is_multi_agent = False
def get_action_fn(_obz):
return ac.step(torch.as_tensor(_obz, dtype=torch.float32))
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
epoch_episode = 0
info = {}
epoch_ended = False
step_num = 0
ep_len = 0
ep_ret = 0
while not epoch_ended:
if try_rollouts != 0:
# a, v, logp, next_o, r, d, info
# a, v, logp, obs, r, done, info
rollout = do_rollouts(
get_action_fn, env, o, steps_per_try_rollout, try_rollouts,
take_worst_rollout)
else:
a, v, logp = get_action_fn(o)
# NOTE: For multi-agent, steps current agent,
# but returns values for next agent (from its previous action)!
# TODO: Just return multiple agents observations
next_o, r, d, info = env.step(a)
if render:
env.render()
curr_reward = env.curr_reward if is_multi_agent else r
# save and log
buf.store(o, a, curr_reward, v, logp, agent_index)
logger.store(VVals=v)
# Update obs (critical!)
o = next_o
if 'stats' in info and info['stats']: # TODO: Optimize this
logger.store(**info['stats'])
if is_multi_agent:
agent_index = env.agent_index
agent = env.agents[agent_index]
# TODO: Store vector of these for each agent when changing step API
ep_len = agent.episode_steps
ep_ret = agent.episode_reward
else:
ep_len += 1
ep_ret += r
calc_effective_horizon_reward(
agent_index, effective_horizon_rewards, logger, r)
timeout = ep_len == max_ep_len
terminal = d or timeout
epoch_ended = buf.epoch_ended(step_num)
if terminal or epoch_ended:
if epoch_ended and not terminal:
print('Warning: trajectory cut off by epoch at %d steps.'%ep_len, flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
if timeout or epoch_ended:
_, v, _ = ac.step(torch.as_tensor(o, dtype=torch.float32))
else:
v = 0
buf.finish_path(agent_index, v)
if terminal:
buf.record_episode(ep_len=ep_len, ep_ret=ep_ret, step_num=step_num)
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
if 'stats' in info and info['stats'] and info['stats']['done_only']:
logger.store(**info['stats']['done_only'])
o, r, d = reset(env)
if not is_multi_agent:
ep_len = 0
ep_ret = 0
step_num += 1
buf.prepare_for_update()
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('DateTime', get_date_str())
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.log_tabular('HorizonReturn', with_min_and_max=True)
if getattr(env.unwrapped, 'is_deepdrive', False):
logger.log_tabular('trip_pct', with_min_and_max=True)
logger.log_tabular('collided')
logger.log_tabular('harmful_gs')
logger.log_tabular('timeup')
logger.log_tabular('exited_lane')
logger.log_tabular('circles')
logger.log_tabular('skipped')
logger.log_tabular('backwards')
logger.log_tabular('won')
if 'stats' in info and info['stats']:
for stat, value in info['stats'].items():
logger.log_tabular(stat, with_min_and_max=True)
if logger.best_category or (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state(dict(env=env), pytorch_save=dict(
ac=ac.state_dict(),
pi_optimizer=pi_optimizer.state_dict(),
vf_optimizer=vf_optimizer.state_dict(),
epoch=epoch,
stats=logger.epoch_dict,
), itr=None, best_category=logger.best_category)
logger.dump_tabular()
def ppo(logger,
reuse,
message,
env_fn,
reward,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
steps_per_epoch=1000,
epochs=10,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=50,
train_v_iters=50,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict()):
#tf.reset_default_graph() # allows us to call ppo multiple rounds
with tf.variable_scope("main", reuse=reuse):
#reward = lambda s, r: r if custom_reward == None else custom_reward(s)
#logger = EpochLogger(**logger_kwargs)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' %
var_counts)
# PPO objectives
ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s)
min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph,
(1 - clip_ratio) * adv_ph)
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv))
v_loss = tf.reduce_mean((ret_ph - v)**2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-logp) # a sample estimate for entropy, also easy to compute
clipped = tf.logical_or(ratio > (1 + clip_ratio), ratio <
(1 - clip_ratio))
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32))
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess,
inputs={'x': x_ph},
outputs={
'pi': pi,
'v': v
})
def update():
inputs = {k: v for k, v in zip(all_phs, buf.get())}
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent],
feed_dict=inputs)
# Training
for i in range(train_pi_iters):
_, kl = sess.run([train_pi, approx_kl], feed_dict=inputs)
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' %
i)
break
logger.store(StopIter=i)
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
pi_l_new, v_l_new, kl, cf = sess.run(
[pi_loss, v_loss, approx_kl, clipfrac], feed_dict=inputs)
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
r = reward(o) # custom reward
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops,
feed_dict={x_ph: o.reshape(1, -1)})
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
r = reward(o)
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print(
'Warning: trajectory cut off by epoch at %d steps.'
% ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(
v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
r = reward(o)
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts',
(epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
print(message)
logger.save_state({'env': env}, None) # save final model
env.close()
def PPO_with_bc(env_fn,
traj_dir,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
d_hidden_size=64,
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=40,
train_v_iters=40,
lam=0.97,
max_ep_len=4000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=50,
r_env_ratio=0,
pretrain_bc=True,
bc_itr=1e4):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
D = Discriminator(env,
hidden_size=d_hidden_size) #!add Discriminator object
e_obs = np.loadtxt(traj_dir + '/observations.csv',
delimiter=',') #!何故かカンマなしのcsv
e_act = np.loadtxt(traj_dir + '/actions.csv',
delimiter=',') #Demo treajectory
print(e_obs.shape, e_act.shape)
assert e_obs.shape[1:] == obs_dim
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi, pi_std, entropy, v = actor_critic(
x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
#buf_gail = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)#add buffer with TRgail rewards
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# PPO objectives
ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s)
min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph,
(1 - clip_ratio) * adv_ph)
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv))
v_loss = tf.reduce_mean((ret_ph - v)**2) #ret_phには累積報酬のバッファが入る
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-logp) # a sample estimate for entropy, also easy to compute
clipped = tf.logical_or(ratio > (1 + clip_ratio), ratio < (1 - clip_ratio))
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32))
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
sess = tf.Session()
BC = BehavioralCloning(sess, pi, logp, x_ph, a_ph)
obs_def_shape = 1 if obs_dim == () else obs_dim[0] #1次元にも対応
act_def_shape = 1 if act_dim == () else act_dim[0]
e_obs_bc = e_obs.reshape(-1, obs_def_shape)
e_act_bc = e_act.reshape(-1, act_def_shape)
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def update():
inputs = {k: v
for k, v in zip(all_phs, buf.get())
} #all_phsは各バッファーに対応するプレースホルダー辞書
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent],
feed_dict=inputs)
# Training#ここも変える必要あり? おそらく変えなくて良い
for i in range(train_pi_iters):
_, kl = sess.run([train_pi, approx_kl], feed_dict=inputs)
kl = mpi_avg(kl)
if kl > 1.5 * target_kl: #更新時のklが想定の1.5倍大きいとログをだしてtrainループを着る
logger.log(
'Early stopping at step %d due to reaching max kl.' % i)
break
logger.store(StopIter=i)
for _ in range(train_v_iters): #ｖの更新
sess.run(train_v, feed_dict=inputs)
# Log changes from update（新しいロスの計算）
pi_l_new, v_l_new, kl, cf = sess.run(
[pi_loss, v_loss, approx_kl, clipfrac], feed_dict=inputs)
std, std_ent = sess.run([pi_std, entropy], feed_dict=inputs)
logger.store(
LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=std_ent,
ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old), #更新での改善量
DeltaLossV=(v_l_new - v_l_old),
Std=std)
start_time = time.time()
if pretrain_bc:
BC.learn(e_obs_bc, e_act_bc, max_itr=bc_itr)
o, r, d, ep_ret_task, ep_len = env.reset(), 0, False, 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops,
feed_dict={x_ph: o.reshape(1, -1)})
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
'''
if epoch >45:
env.render()
time.sleep(0.03)
'''
ep_ret_task += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
'''
if not(terminal):
print('Warning: trajectory cut off by epoch at %d steps.'%ep_len)
'''
#!add discriminator train
'''#終端も加えるならアリッチャあり
o_reshape = o.reshape(core.combined_shape(1,obs_dim))
a_reshape = a.reshape(core.combined_shape(1,act_dim))
agent_obs = np.append(buf.obs_buf[buf.path_slice()],o_reshape,axis = 0)#!o を(obspace,)→(1,obspace)に変換してからアペンド
agent_act = np.append(buf.act_buf[buf.path_slice()],a_reshape,axis = 0)#終端での状態行動対も加えてDを学習
'''
agent_obs = buf.obs_buf[buf.path_slice()]
agent_act = buf.act_buf[buf.path_slice()]
if d: # if trajectory didn't reach terminal state, bootstrap value target
last_val = r
else:
last_val = sess.run(v,
feed_dict={x_ph: o.reshape(1, -1)
}) #v_last=...だったけどこれで良さげ
#!until here
buf.finish_path(
last_val) #これの前にbuf.finish_add_r_vがなされていることを確認すべし
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret_task, EpLen=ep_len)
o, r, d, ep_ret_task, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, epoch)
# Perform PPO update!
for _t in range(train_pi_iters // 3):
D.train(sess, e_obs, e_act, agent_obs, agent_act)
update()
js_d = D.get_js_div(sess, e_obs, e_act, agent_obs, agent_act)
logger.store(JS=js_d)
# Log info about epoch
#if epoch%10 == 0:#logger print each 10 epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.log_tabular('JS', average_only=True)
logger.log_tabular('Std', average_only=True)
logger.dump_tabular()
def vpg(env_fn,
actor_critic=core.ActorCritic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
pi_lr=3e-4,
vf_lr=1e-3,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The agent's main model which is composed of
the policy and value function model, where the policy takes
some state, ``x``, and action, ``a``, and value function takes
the state ``x`` and returns a tuple of:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a``
| in states ``x``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x``. (Critical: make sure
| to flatten this via .item()!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
class you provided to VPG.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Main model
actor_critic = actor_critic(in_features=obs_dim[0], **ac_kwargs)
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = VPGBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(
core.count_vars(module)
for module in [actor_critic.policy, actor_critic.value_function])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# Optimizers
train_pi = torch.optim.Adam(actor_critic.policy.parameters(), lr=pi_lr)
train_v = torch.optim.Adam(actor_critic.value_function.parameters(),
lr=vf_lr)
# Sync params across processes
sync_all_params(actor_critic.state_dict())
def update():
obs, act, adv, ret, logp_old = [torch.Tensor(x) for x in buf.get()]
# Policy gradient step
_, logp, _ = actor_critic.policy(obs, act)
ent = (-logp).mean() # a sample estimate for entropy
# VPG policy objective
pi_loss = -(logp * adv).mean()
# Policy gradient step
train_pi.zero_grad()
pi_loss.backward()
average_gradients(train_pi.param_groups)
train_pi.step()
# Value function learning
v = actor_critic.value_function(obs)
v_l_old = F.mse_loss(v, ret)
for _ in range(train_v_iters):
# Output from value function graph
v = actor_critic.value_function(obs)
# VPG value objective
v_loss = F.mse_loss(v, ret)
# Value function gradient step
train_v.zero_grad()
v_loss.backward()
average_gradients(train_v.param_groups)
train_v.step()
# Log changes from update
_, logp, _, v = actor_critic(obs, act)
pi_l_new = -(logp * adv).mean()
v_l_new = F.mse_loss(v, ret)
kl = (logp_old - logp).mean() # a sample estimate for KL-divergence
logger.store(LossPi=pi_loss,
LossV=v_l_old,
KL=kl,
Entropy=ent,
DeltaLossPi=(pi_l_new - pi_loss),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
actor_critic.eval()
for t in range(local_steps_per_epoch):
a, _, logp_t, v_t = actor_critic(torch.Tensor(o.reshape(1, -1)))
# save and log
buf.store(o, a.data.numpy(), r, v_t.item(), logp_t.data.numpy())
logger.store(VVals=v_t)
o, r, d, _ = env.step(a.data.numpy()[0])
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else actor_critic.value_function(
torch.Tensor(o.reshape(1, -1))).item()
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, actor_critic, None)
# Perform VPG update!
actor_critic.train()
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo(env_fn,
# by default, use the neural network mlp we define in core
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=80,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10):
"""
"Args:
env_fn: A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function with takes in placeholder symbols
for state, ``x_ph``, and action ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given states.
``logp`` (batch,) | Gives log probability according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================" -OpenAI
Okay, quick interruption to OpenAI documentation here.
actor_critic is the function which interfaces with tensorflow. It takes in
``x_ph`` (x placeholder), ie. a representation of the current state, and
``a_ph``, a representation of the some actions. (TODO: document
*what* these actions are).
actor_critic runs these inputs through the tensorflow graph and returns several
pieces of information that are relevant to PPO; these are described above.
Back to OpenAI:
"
ac_kwargs (dict): Any kwargs appropriate for actor_critic function
you provided to PPO.
seed (int): Seed for random number generators.
setps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value funciton per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1).
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function." - OpenAI
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# modify the seed based on the process so if
# we run this in multiple processes
# simultaneously we don't do the
# exact same thing
seed += 10000 * proc_id()
# set up our random stuff with this seed
tf.set_random_seed(seed)
np.random.seed(seed)
# create the environment
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# tell the policy (implemented in actor_critic function) what the action space is
ac_kwargs['action_space'] = env.action_space
# "Inputs to computation graph" -OpenAI
# create tensorflow placeholders for observations (x_ph), actions (a_ph),
# advantages (adv_ph), returns (ret_ph), log probabilities
# in the current state of the policy (logp_old_ph)
# (old since this is used compared to the newer version of the policy
# we are creating in the optimization step, comparing to this "old" version)
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space, env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# "Main outputs from computation graph" -OpenAI
# essentially here we fill in the tensorflow graph so we can compute
# the pi, logp, logp_pi, and v tensors based on the
# x_ph and a_ph we created above
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
# "Need all placeholders in *this* order later (to zip with data from buffer)" -OpenAI
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# "Every step, get: action, value, and logprob" -OpenAI
# we later feed this list into tf.session.run()
# to tell it to compute the value of pi, v, logp_pi
# using the tensorflow graph we have created
get_action_ops = [pi, v, logp_pi]
# Experience buffer
# number of steps per epoch per process
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count the number of parameters we are gonna be training,
# both for the policy and for the value function
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n'%var_counts)
# PPO objectives
# ratio is the ratio of two probabilities:
# pi(a|s) / pi_old(a|s)
# where pi(a|s) is the probability of performing action a
# given state s GIVEN THE POLICY WHOSE PARAMETERS WE ARE CHANGING
# DURING THE OPTIMIZATION STEP
# and pi_old(a|s) is the probability of the policy,
# with fixed mlp parameters after the last update,
# performing a given state s
# we essentially use math to find the gradient of pi(a|s) with respect
# to the parameters of the mlp, and this is the core of how we calculate
# the gradient of the objective function for gradient descent
ratio = tf.exp(logp - logp_old_ph) # "pi(a|s) / pi_old(a|s)"-OpenAI
# this min_adv, along with the tf.minimum call in the next line of code,
# implement the PPO-clip functionality
# NOTE: calling this `min_adv` is a bit confusing; if advantage is negative
# this is the min value we allow the gradient descent to consider as the advantage;
# but it is the MAX value if advantage is positive.
min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph, (1 - clip_ratio) * adv_ph)
# create the functions whose gradients we wish to use for gradient descent
# during optimization
# for our policy optimization, it is the PPO objective;
# for the value function it is simply an error-squared
# note that reduce_mean just calculates the mean of the values in the tensor;
# ie. this gives the expected value of the loss given the experimental values we have
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv))
v_loss = tf.reduce_mean((ret_ph - v) ** 2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(logp_old_ph - logp) # "a sample estimate for KL-divergence, easy to compute" -OpenAI
approx_ent = tf.reduce_mean(-logp) # "a sample estimate for entropy, also easy to compute" -OpenAI
clipped = tf.logical_or(ratio > (1 + clip_ratio), ratio < (1 - clip_ratio))
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32)) # what fraction of advantages are clipped
# Optimizers
# These use gradient descent with the gradient of the objective
# functions we defined above to improve parameters for pi and v
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
# initialize the tensorflow computation graph's parameters
# with values
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# "Sync params across processes" -OpenAI
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def update():
# create a dictionary of values, which specify to tensorflow what
# to input for the placeholders: tensors containing the data from
# the trajectory we have stored in buf
inputs = {k:v for k, v in zip(all_phs, buf.get())}
# calculate these for logging later
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent], feed_dict=inputs)
# Training
for i in range(train_pi_iters):
# run a training step for the policy, and estimate the kl-divergence
# (ie. how much the policy changed) on this step
_, kl = sess.run([train_pi, approx_kl], feed_dict=inputs)
kl = mpi_avg(kl)
# if the kl divergence is too high, stop training on this step
# TODO: understand better why it is important to do this
if kl > 1.5 * target_kl:
logger.log('Early stopping at step %d due to reaching max kl.'%i)
break
logger.store(StopIter=i)
# train our value function mlp
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# "Log changes from update" -OpenAI
# TODO: This could be made a bit more computationally efficient by not recalculating pi_l_old each loop
# after having calculated the same thing as pi_l_new the previous run through the loop!
# Plus, does it really make the most sense to output pi_l_old and v_l_old as LossPi and LossV
# instead of pi_l_new and v_l_new?
pi_l_new, v_l_new, kl, cf = sess.run([pi_loss, v_loss, approx_kl, clipfrac], feed_dict=inputs)
logger.store(LossPi=pi_l_old, LossV=v_l_old,
KL=kl, Entropy=ent, ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
# initialize the variables we use while training
# o = observation (env.reset() returns initial observation)
# r = reward = (starts as 0)
# d = done? (whether current episode in env is over)
# ep_ret = episode return
# ep_len = length of episode so far
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# "Main loop: collect experience in env and update/log each epoch"
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
# run the computation of the action, value function, and probability of the action
# using the most recent observation in the x_ph slot
a, v_t, logp_t = sess.run(get_action_ops, feed_dict={x_ph: o.reshape(1,-1)})
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
# take the action we computed and advance the environment
o, r, d, _ = env.step(a[0])
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t==local_steps_per_epoch - 1):
if not terminal:
print('Warning: trajectory cut off by epoch at %d steps'%ep_len)
# "if trajectory didn't reach terminal state, bootstrap value target" -OpenAI
# in other words, if the we are stopping this trajectory due to a termination
# signal from the env, last_val = the reward from the last step, r
# otherwise we stopped because we reached the max episode length or max local_steps_per_epoch,
# in which ase we set last_val = estimate of the value of current state based on v function
# we are training
last_val = r if d else sess.run(v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
# "only store EpRet / EpLen if trajectory finished" -OpenAI
if terminal:
logger.store(EpRet=ep_ret, EpLen=ep_len)
# reset our training variables and the training environment
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# every save_freq epochs,
# save the state of the environment
# also save the current state of our value function model
# and policy
# these are automatically saved by the save_state function
# since we have already called logger.setup_tf_saver
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# "Perform PPO update!"
update()
# "Log info about epoch"
logger.log_tabular('Epoch', epoch)
try:
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
except:
pass
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=80,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10):
"""
Proximal Policy Optimization (by clipping),
with early stopping based on approximate KL
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.策略给出的动作
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.策略给出的x状态下的a动作概率
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.动作被采样的概率
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)状态x下的V值
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to PPO. AC框架参数
seed (int): Seed for random number generators.#随机种子数0
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.每轮迭代次数4000
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.轮次50
gamma (float): Discount factor. (Always between 0 and 1.)折扣因子0.99
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.) Typically
denoted by :math:`\epsilon`. 剪切比率0.2
pi_lr (float): Learning rate for policy optimizer.策略学习率3e-4
vf_lr (float): Learning rate for value function optimizer.评价网络学习率1e-3
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)策略最大梯度下降步数80
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.评价最大梯度下降步数80
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.) TD（lambda）中的lambda =0.97
max_ep_len (int): Maximum length of trajectory / episode / rollout. 每次最长步长 1000
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)提前停车用，目标kl 0.01
logger_kwargs (dict): Keyword args for EpochLogger.日志参数
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.模型保存频率每10轮
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn() #创建环境
obs_dim = env.observation_space.shape #读取环境维度
act_dim = env.action_space.shape #动作维度
# Share information about action space with policy architecture 策略框架下共享动作有关信息
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph,
**ac_kwargs) #输入环境和动作，输出策略相关信息
# Need all placeholders in *this* order later (to zip with data from buffer)之后需要的数据：环境，动作，优势函数、奖励和上一步的策略
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob每步得到的信息：策略、V值和动作概率
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam) #经验池
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v']) #记录变量
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' %
var_counts) #日志输出策略和评价的变量数
# PPO objectives
ratio = tf.exp(logp -
logp_old_ph) # pi(a|s) / pi_old(a|s) 比率=pi（new）/pi（old）
min_adv = tf.where(
adv_ph > 0, (1 + clip_ratio) * adv_ph, (1 - clip_ratio) *
adv_ph) #tfwhere条件adv_ph,>0，表示优势增加，选(1+clip_ratio)*adv_ph，
#<0，表示优势减少，选(1-clip_ratio)*adv_ph。
pi_loss = -tf.reduce_mean(tf.minimum(
ratio * adv_ph, min_adv)) #策略loss=比率adv与min_adv的最小者，限制策略偏移
v_loss = tf.reduce_mean((ret_ph - v)**2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph - logp
) # a sample estimate for KL-divergence, easy to compute KL散度的样本估计，易于计算
approx_ent = tf.reduce_mean(
-logp
) # a sample estimate for entropy, also easy to compute 熵的样本估计，易于计算
clipped = tf.logical_or(
ratio > (1 + clip_ratio), ratio <
(1 - clip_ratio)) # 逻辑或运算，判定需要剪切，clipped = true/false
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32)) #clipped转换为浮点数
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def update():
inputs = {k: v
for k, v in zip(all_phs, buf.get())
} # zip（[x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]，
#[self.obs_buf, self.act_buf, self.adv_buf, self.ret_buf, self.logp_buf] ）
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent],
feed_dict=inputs) # 输入上述数据，计算俩loss和熵
# Training
for i in range(train_pi_iters): #策略迭代
_, kl = sess.run([train_pi, approx_kl], feed_dict=inputs) # 计算kl
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' % i)
break #提前停止策略训练
logger.store(StopIter=i)
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs) # 训练评价网络
# Log changes from update
pi_l_new, v_l_new, kl, cf = sess.run(
[pi_loss, v_loss, approx_kl, clipfrac],
feed_dict=inputs) #重新计算loss和kl，cf
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new -
v_l_old)) # 输出旧loss，kl，cf 和 delta loss
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0 #重置
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs): # 每一轮
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops,
feed_dict={x_ph:
o.reshape(1, -1)}) #根据环境给出动作
o2, r, d, _ = env.step(a[0]) #环境交互取得奖励
ep_ret += r #奖励累加
ep_len += 1 # 步长加一
# save and log
buf.store(o, a, r, v_t, logp_t) #存储（环境、动作、奖励、V值、概率）经验池
logger.store(VVals=v_t)
# Update obs (critical!)更新环境
o = o2
terminal = d or (ep_len == max_ep_len) #结束了或者到达最大步数了
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len) #轨迹被epoch在ep_len步切断
# if trajectory didn't reach terminal state, bootstrap value target 如果轨迹没有达到终端状态，引导值目标
last_val = 0 if d else sess.run(
v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo(env_fn,
actor_critic=MLPActorCritic,
ac_kwargs={},
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=80,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10):
"""
Proximal Policy Optimization (by clipping),
with early stopping based on approximate KL
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The constructor method for a PyTorch Module with a
``step`` method, an ``act`` method, a ``pi`` module, and a ``v``
module. The ``step`` method should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``a`` (batch, act_dim) | Numpy array of actions for each
| observation.
``v`` (batch,) | Numpy array of value estimates
| for the provided observations.
``logp_a`` (batch,) | Numpy array of log probs for the
| actions in ``a``.
=========== ================ ======================================
The ``act`` method behaves the same as ``step`` but only returns ``a``.
The ``pi`` module's forward call should accept a batch of
observations and optionally a batch of actions, and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` N/A | Torch Distribution object, containing
| a batch of distributions describing
| the policy for the provided observations.
``logp_a`` (batch,) | Optional (only returned if batch of
| actions is given). Tensor containing
| the log probability, according to
| the policy, of the provided actions.
| If actions not given, will contain
| ``None``.
=========== ================ ======================================
The ``v`` module's forward call should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``v`` (batch,) | Tensor containing the value estimates
| for the provided observations. (Critical:
| make sure to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object
you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.) Typically
denoted by :math:`\epsilon`.
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
torch.manual_seed(10)
np.random.seed(10)
random.seed(10)
# Set up logger and save configuration
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# Instantiate environment
env = env_fn()
# Create actor-critic module
ac = actor_critic(env.observation_space, env.action_space,
**ac_kwargs).cuda()
# Sync params across processes
sync_params(ac)
# Count variables
var_counts = tuple(core.count_vars(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# Set up experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOTrajectory(gamma, lam)
training_queue = TrainingQueue(200)
# Set up optimizers for policy and value function
pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)
# Prepare for interaction with environment
start_time = time.time()
o, ep_ret, ep_len = env.reset(), 0, 0
num_training = 15
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
training_data = training_queue.get_batch(num_training)
num_training = len(training_data['act'])
o = torch.tensor(o).float().unsqueeze(0)
if num_training > 0:
o = torch.cat([o, training_data['obs']])
o = o.cuda()
pi = ac.pi._distribution(o)
a = pi.sample()
if num_training > 0:
a[-num_training:] = training_data['act']
logp = ac.pi._log_prob_from_distribution(pi, a)
v = ac.v(o)
if num_training > 0:
run_update(logp[-num_training:], v[-num_training:],
training_data['ret'].cuda(),
training_data['adv'].cuda(),
training_data['logp'].cuda(), pi_optimizer,
vf_optimizer, clip_ratio, logger)
a = a[:len(a) - num_training].cpu().item()
o = o[:len(o) - num_training].cpu().numpy().squeeze()
v = v[:len(v) - num_training].cpu().item()
logp = logp[:len(logp) - num_training].cpu().item()
next_o, r, d, _ = env.step(a)
ep_ret += r
ep_len += 1
# save and log
buf.store(o, a, r, v, logp)
logger.store(VVals=v)
# Update obs (critical!)
o = next_o
num_training = 15
timeout = ep_len == max_ep_len
terminal = d or timeout
epoch_ended = t == local_steps_per_epoch - 1
if terminal:
if epoch_ended and not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len,
flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
if timeout or epoch_ended:
v = ac.v(
torch.as_tensor(o, dtype=torch.float32).unsqueeze(
0).cuda()).cpu().detach().item()
else:
v = 0
trajectory = buf.finish_path(v)
training_queue.put_batch(trajectory)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
# logger.log_tabular('LossPi', average_only=True)
# logger.log_tabular('LossV', average_only=True)
# logger.log_tabular('DeltaLossPi', average_only=True)
# logger.log_tabular('DeltaLossV', average_only=True)
# logger.log_tabular('Entropy', average_only=True)
# logger.log_tabular('KL', average_only=True)
# logger.log_tabular('ClipFrac', average_only=True)
# logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def vpg(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
pi_lr=3e-4,
vf_lr=1e-3,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10,
custom_h=None,
do_checkpoint_eval=False,
env_name=None):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to VPG.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# create logger for tensorboard
tb_logdir = "{}/tb_logs/".format(logger.output_dir)
tb_logger = Logger(log_dir=tb_logdir)
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
if custom_h is not None:
hidden_layers_str_list = custom_h.split('-')
hidden_layers_int_list = [int(h) for h in hidden_layers_str_list]
ac_kwargs['hidden_sizes'] = hidden_layers_int_list
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = VPGBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# VPG objectives
pi_loss = -tf.reduce_mean(logp * adv_ph)
v_loss = tf.reduce_mean((ret_ph - v)**2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-logp) # a sample estimate for entropy, also easy to compute
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
# create a tf session with GPU memory usage option to be allow_growth so that one program will not use up the
# whole GPU memory
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
# log tf graph
tf.summary.FileWriter(tb_logdir, sess.graph)
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
# for saving the best models and performances during train and evaluate
best_eval_AverageEpRet = 0.0
best_eval_StdEpRet = 1.0e20
def update():
inputs = {k: v for k, v in zip(all_phs, buf.get())}
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent],
feed_dict=inputs)
# Policy gradient step
sess.run(train_pi, feed_dict=inputs)
# Value function learning
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
pi_l_new, v_l_new, kl = sess.run([pi_loss, v_loss, approx_kl],
feed_dict=inputs)
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops,
feed_dict={x_ph: o.reshape(1, -1)})
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(
v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
# Save a new model every save_freq and at the last epoch. Do not overwrite the previous save.
logger.save_state({'env': env}, epoch)
# Evaluate and save best model
if do_checkpoint_eval and epoch > 0:
# below is a hack. best model related stuff is saved at itr 999999, therefore, simple_save999999.
# Doing this way, I can use test_policy and plot directly to test the best models.
# saved best models includes:
# 1) a copy of the env
# 2) the best rl model with parameters
# 3) a pickle file "best_eval_performance_n_structure" storing best_performance, best_structure and epoch
# note that 1) and 2) are spinningup defaults, and 3) is a custom save
best_eval_AverageEpRet, best_eval_StdEpRet = eval_and_save_best_model(
best_eval_AverageEpRet,
best_eval_StdEpRet,
# a new logger is created and passed in so that the new logger can leverage the directory
# structure without messing up the logger in the training loop
eval_logger=EpochLogger(
**dict(exp_name=logger_kwargs['exp_name'],
output_dir=os.path.join(logger.output_dir,
"simple_save999999"))),
train_logger=logger,
tb_logger=tb_logger,
epoch=epoch,
# the env_name is passed in so that to create an env when and where it is needed. This is to
# logx.save_state() error where an env pointer cannot be pickled
env_name=env_name,
get_action=lambda x: sess.run(
pi, feed_dict={x_ph: x[None, :]})[0])
# Perform VPG update!
update()
# # # Log into tensorboard
log_key_to_tb(tb_logger,
logger,
epoch,
key="EpRet",
with_min_and_max=True)
log_key_to_tb(tb_logger,
logger,
epoch,
key="EpLen",
with_min_and_max=False)
log_key_to_tb(tb_logger,
logger,
epoch,
key="VVals",
with_min_and_max=True)
log_key_to_tb(tb_logger,
logger,
epoch,
key="LossPi",
with_min_and_max=False)
log_key_to_tb(tb_logger,
logger,
epoch,
key="LossV",
with_min_and_max=False)
log_key_to_tb(tb_logger,
logger,
epoch,
key="DeltaLossPi",
with_min_and_max=False)
log_key_to_tb(tb_logger,
logger,
epoch,
key="DeltaLossV",
with_min_and_max=False)
log_key_to_tb(tb_logger,
logger,
epoch,
key="Entropy",
with_min_and_max=False)
log_key_to_tb(tb_logger,
logger,
epoch,
key="KL",
with_min_and_max=False)
tb_logger.log_scalar(tag="TotalEnvInteracts",
value=(epoch + 1) * steps_per_epoch,
step=epoch)
tb_logger.log_scalar(tag="Time",
value=time.time() - start_time,
step=epoch)
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo(env_fn,
expert=None,
policy_path=None,
actor_critic=core.mlp_actor_critic_m,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=5000,
epochs=10000,
dagger_epochs=500,
pretrain_epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=1e-4,
dagger_noise=0.01,
batch_size=64,
replay_size=int(5e3),
vf_lr=1e-4,
train_pi_iters=80,
train_v_iters=80,
lam=0.999,
max_ep_len=500,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10,
test_freq=10):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
policy_path (str): path of pretrained policy model
train from scratch if None
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
test_logger_kwargs = dict()
test_logger_kwargs['output_dir'] = osp.join(logger_kwargs['output_dir'],
"test")
test_logger_kwargs['exp_name'] = logger_kwargs['exp_name']
test_logger = EpochLogger(**test_logger_kwargs)
test_logger.save_config(locals())
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
act_high_limit = env.action_space.high
act_low_limit = env.action_space.low
sess = tf.Session()
if policy_path is None:
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
tfa_ph = core.placeholder(act_dim)
# Main outputs from computation graph
mu, pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
sess.run(tf.global_variables_initializer())
else:
# load pretrained model
# sess, x_ph, a_ph, mu, pi, logp, logp_pi, v = load_policy(policy_path, itr='last', deterministic=False, act_high=env.action_space.high)
# # get_action_2 = lambda x : sess.run(mu, feed_dict={x_ph: x[None,:]})[0]
# adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
model = restore_tf_graph(sess, osp.join(policy_path, 'simple_save'))
x_ph, a_ph, adv_ph, ret_ph, logp_old_ph = model['x_ph'], model[
'a_ph'], model['adv_ph'], model['ret_ph'], model['logp_old_ph']
mu, pi, logp, logp_pi, v = model['mu'], model['pi'], model[
'logp'], model['logp_pi'], model['v']
# tfa_ph = core.placeholder(act_dim)
tfa_ph = model['tfa_ph']
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
print("---------------", local_steps_per_epoch)
buf = PPOBuffer(obs_dim, act_dim, steps_per_epoch, gamma, lam)
# print(obs_dim)
# print(act_dim)
dagger_replay_buffer = DaggerReplayBuffer(obs_dim=obs_dim[0],
act_dim=act_dim[0],
size=replay_size)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# PPO objectives
if policy_path is None:
ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s)
min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph,
(1 - clip_ratio) * adv_ph)
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv))
v_loss = tf.reduce_mean((ret_ph - v)**2)
dagger_pi_loss = tf.reduce_mean(tf.square(mu - tfa_ph))
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-logp) # a sample estimate for entropy, also easy to compute
clipped = tf.logical_or(ratio > (1 + clip_ratio), ratio <
(1 - clip_ratio))
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32))
# Optimizers
dagger_pi_optimizer = tf.train.AdamOptimizer(learning_rate=pi_lr)
optimizer_pi = tf.train.AdamOptimizer(learning_rate=pi_lr)
optimizer_v = tf.train.AdamOptimizer(learning_rate=vf_lr)
train_dagger_pi_op = dagger_pi_optimizer.minimize(
dagger_pi_loss, name='train_dagger_pi_op')
train_pi = optimizer_pi.minimize(pi_loss, name='train_pi_op')
train_v = optimizer_v.minimize(v_loss, name='train_v_op')
sess.run(tf.variables_initializer(optimizer_pi.variables()))
sess.run(tf.variables_initializer(optimizer_v.variables()))
sess.run(tf.variables_initializer(dagger_pi_optimizer.variables()))
else:
graph = tf.get_default_graph()
dagger_pi_loss = model['dagger_pi_loss']
pi_loss = model['pi_loss']
v_loss = model['v_loss']
approx_ent = model['approx_ent']
approx_kl = model['approx_kl']
clipfrac = model['clipfrac']
train_dagger_pi_op = graph.get_operation_by_name('train_dagger_pi_op')
train_pi = graph.get_operation_by_name('train_pi_op')
train_v = graph.get_operation_by_name('train_v_op')
# sess = tf.Session()
# sess.run(tf.global_variables_initializer())
# Sync params across processes
# sess.run(sync_all_params())
tf.summary.FileWriter("log/", sess.graph)
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x_ph': x_ph, 'a_ph': a_ph, 'tfa_ph': tfa_ph, 'adv_ph': adv_ph, 'ret_ph': ret_ph, 'logp_old_ph': logp_old_ph}, \
outputs={'mu': mu, 'pi': pi, 'v': v, 'logp': logp, 'logp_pi': logp_pi, 'clipfrac': clipfrac, 'approx_kl': approx_kl, \
'pi_loss': pi_loss, 'v_loss': v_loss, 'dagger_pi_loss': dagger_pi_loss, 'approx_ent': approx_ent})
def update():
inputs = {k: v for k, v in zip(all_phs, buf.get())}
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent],
feed_dict=inputs)
# Training
for i in range(train_pi_iters):
_, kl = sess.run([train_pi, approx_kl], feed_dict=inputs)
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' % i)
break
logger.store(StopIter=i)
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
pi_l_new, v_l_new, kl, cf = sess.run(
[pi_loss, v_loss, approx_kl, clipfrac], feed_dict=inputs)
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
def choose_action(s, add_noise=False):
s = s[np.newaxis, :]
a = sess.run(mu, {x_ph: s})[0]
if add_noise:
noise = dagger_noise * act_high_limit * np.random.normal(
size=a.shape)
a = a + noise
return np.clip(a, act_low_limit, act_high_limit)
def test_agent(n=81, test_num=1):
n = env.unwrapped._set_test_mode(True)
con_flag = False
for j in range(n):
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
while not (d or (ep_len == max_ep_len)):
# Take deterministic actions at test time (noise_scale=0)
o, r, d, info = env.step(choose_action(np.array(o), 0))
ep_ret += r
ep_len += 1
if d:
test_logger.store(TestEpRet=ep_ret, TestEpLen=ep_len)
test_logger.store(arrive_des=info['arrive_des'])
test_logger.store(
arrive_des_appro=info['arrive_des_appro'])
if not info['out_of_range']:
test_logger.store(converge_dis=info['converge_dis'])
con_flag = True
test_logger.store(out_of_range=info['out_of_range'])
# print(info)
# test_logger.dump_tabular()
# time.sleep(10)
if not con_flag:
test_logger.store(converge_dis=10000)
env.unwrapped._set_test_mode(False)
def ref_test_agent(n=81, test_num=1):
n = env.unwrapped._set_test_mode(True)
con_flag = False
for j in range(n):
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
while not (d or (ep_len == max_ep_len)):
# Take deterministic actions at test time (noise_scale=0)
a = call_ref_controller(env, expert)
o, r, d, info = env.step(a)
ep_ret += r
ep_len += 1
if d:
test_logger.store(TestEpRet=ep_ret, TestEpLen=ep_len)
test_logger.store(arrive_des=info['arrive_des'])
test_logger.store(
arrive_des_appro=info['arrive_des_appro'])
if not info['out_of_range']:
test_logger.store(converge_dis=info['converge_dis'])
con_flag = True
test_logger.store(out_of_range=info['out_of_range'])
# print(info)
# test_logger.dump_tabular()
if not con_flag:
test_logger.store(converge_dis=10000)
env.unwrapped._set_test_mode(False)
ref_test_agent(test_num=-1)
test_logger.log_tabular('epoch', -1)
test_logger.log_tabular('TestEpRet', average_only=True)
test_logger.log_tabular('TestEpLen', average_only=True)
test_logger.log_tabular('arrive_des', average_only=True)
test_logger.log_tabular('arrive_des_appro', average_only=True)
test_logger.log_tabular('converge_dis', average_only=True)
test_logger.log_tabular('out_of_range', average_only=True)
test_logger.dump_tabular()
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
test_policy_epochs = 91
episode_steps = 500
total_env_t = 0
test_num = 0
print(colorize("begin dagger training", 'green', bold=True))
for epoch in range(1, dagger_epochs + 1, 1):
# test policy
if epoch > 0 and (epoch % save_freq == 0) or (epoch == epochs):
# Save model
logger.save_state({}, None)
# Test the performance of the deterministic version of the agent.
test_num += 1
test_agent(test_num=test_num)
test_logger.log_tabular('epoch', epoch)
test_logger.log_tabular('TestEpRet', average_only=True)
test_logger.log_tabular('TestEpLen', average_only=True)
test_logger.log_tabular('arrive_des', average_only=True)
test_logger.log_tabular('arrive_des_appro', average_only=True)
test_logger.log_tabular('converge_dis', average_only=True)
test_logger.log_tabular('out_of_range', average_only=True)
test_logger.dump_tabular()
# train policy
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
env.unwrapped._set_test_mode(False)
obs, acs, rewards = [], [], []
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(
get_action_ops, feed_dict={x_ph: np.array(o).reshape(1, -1)})
# a = get_action_2(np.array(o))
# save and log
obs.append(o)
ref_action = call_ref_controller(env, expert)
if (epoch < pretrain_epochs):
action = ref_action
else:
action = choose_action(np.array(o), True)
buf.store(o, action, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(action)
acs.append(ref_action)
rewards.append(r)
ep_ret += r
ep_len += 1
total_env_t += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(
v, feed_dict={x_ph: np.array(o).reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Perform dagger and partical PPO update!
inputs = {k: v for k, v in zip(all_phs, buf.get())}
# pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent], feed_dict=inputs)
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
max_step = len(np.array(rewards))
dagger_replay_buffer.stores(obs, acs, rewards)
for _ in range(int(local_steps_per_epoch / 10)):
batch = dagger_replay_buffer.sample_batch(batch_size)
feed_dict = {x_ph: batch['obs1'], tfa_ph: batch['acts']}
q_step_ops = [dagger_pi_loss, train_dagger_pi_op]
for j in range(10):
outs = sess.run(q_step_ops, feed_dict)
logger.store(LossPi=outs[0])
c_v_loss = sess.run(v_loss, feed_dict=inputs)
logger.store(LossV=c_v_loss,
KL=0,
Entropy=0,
ClipFrac=0,
DeltaLossPi=0,
DeltaLossV=0,
StopIter=0)
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
# Main loop: collect experience in env and update/log each epoch
print(colorize("begin ppo training", 'green', bold=True))
for epoch in range(1, epochs + 1, 1):
# test policy
if epoch > 0 and (epoch % save_freq == 0) or (epoch
== epochs) or epoch == 1:
# Save model
logger.save_state({}, None)
# Test the performance of the deterministic version of the agent.
test_num += 1
test_agent(test_num=test_num)
test_logger.log_tabular('epoch', epoch)
test_logger.log_tabular('TestEpRet', average_only=True)
test_logger.log_tabular('TestEpLen', average_only=True)
test_logger.log_tabular('arrive_des', average_only=True)
test_logger.log_tabular('arrive_des_appro', average_only=True)
test_logger.log_tabular('converge_dis', average_only=True)
test_logger.log_tabular('out_of_range', average_only=True)
test_logger.dump_tabular()
# train policy
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
env.unwrapped._set_test_mode(False)
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(
get_action_ops, feed_dict={x_ph: np.array(o).reshape(1, -1)})
# a = a[0]
# a = get_action_2(np.array(o))
# a = np.clip(a, act_low_limit, act_high_limit)
# if epoch < pretrain_epochs:
# a = env.action_space.sample()
# a = np.clip(a, act_low_limit, act_high_limit)
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(
v, feed_dict={x_ph: np.array(o).reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo(task,
actor_critic=model.ActorCritic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
lr=3e-4,
v_loss_coeff=0.5,
train_iters=80,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10,
wrapper_type="continuous_absolute",
log_wandb=False):
"""
Proximal Policy Optimization (by clipping),
with early stopping based on approximate KL
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The constructor method for a PyTorch Module with a
``step`` method, an ``act`` method, a ``pi`` module, and a ``v``
module. The ``step`` method should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``a`` (batch, act_dim) | Numpy array of actions for each
| observation.
``v`` (batch,) | Numpy array of value estimates
| for the provided observations.
``logp_a`` (batch,) | Numpy array of log probs for the
| actions in ``a``.
=========== ================ ======================================
The ``act`` method behaves the same as ``step`` but only returns ``a``.
The ``pi`` module's forward call should accept a batch of
observations and optionally a batch of actions, and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` N/A | Torch Distribution object, containing
| a batch of distributions describing
| the policy for the provided observations.
``logp_a`` (batch,) | Optional (only returned if batch of
| actions is given). Tensor containing
| the log probability, according to
| the policy, of the provided actions.
| If actions not given, will contain
| ``None``.
=========== ================ ======================================
The ``v`` module's forward call should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``v`` (batch,) | Tensor containing the value estimates
| for the provided observations. (Critical:
| make sure to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object
you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.) Typically
denoted by :math:`\epsilon`.
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
# Special function to avoid certain slowdowns from PyTorch + MPI combo.
setup_pytorch_for_mpi()
# Set up logger and save configuration
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# Random seed
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# Instantiate environment
env = dm_construction.get_environment(task, wrapper_type=wrapper_type)
obs_dim = env.observation_spec().shape
if wrapper_type == "continuous_absolute":
act_dim = 4 # for continuous absolute action space
else:
raise NotImplementedError
# Create actor-critic module
ac = actor_critic(env.observation_spec(), env.action_spec(), **ac_kwargs)
# Sync params across processes
sync_params(ac)
# Count variables
var_counts = count_vars(ac.ac)
logger.log(f"\nNumber of parameters: \t {var_counts}")
# Set up experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
def compute_loss(data):
obs, act, adv, logp_old, ret = data['obs'], data['act'], data[
'adv'], data['logp'], data['ret']
pi, v, logp = ac.ac(obs, act)
# value loss (just MSE)
loss_v = ((v - ret)**2).mean()
# policy loss
ratio = torch.exp(logp - logp_old)
clip_adv = torch.clamp(ratio, 1 - clip_ratio, 1 + clip_ratio) * adv
loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()
# useful extra info re: policy
approx_kl = (logp_old - logp).mean().item()
ent = pi.entropy().mean().item()
clipped = ratio.gt(1 + clip_ratio) | ratio.lt(1 - clip_ratio)
clipfrac = torch.as_tensor(clipped, dtype=torch.float32).mean().item()
pi_info = dict(kl=approx_kl, ent=ent, cf=clipfrac)
return loss_v, loss_pi, pi_info
# Set up optimizers for policy and value function
optimizer = Adam(ac.ac.parameters(), lr=lr)
# Set up model saving
logger.setup_pytorch_saver(ac)
def update():
data = buf.get()
v_l_old, pi_l_old, pi_info_old = compute_loss(data)
pi_l_old = pi_l_old.item()
vl_l_old = v_l_old.item()
# Train policy with multiple steps of gradient descent
for i in range(train_iters):
optimizer.zero_grad()
loss_v, loss_pi, pi_info = compute_loss(data)
kl = mpi_avg(pi_info['kl'])
if kl > 1.5 * target_kl:
logger.log(
f'Early stopping at step {i} due to reaching max kl.')
break
loss = loss_pi + loss_v * v_loss_coeff
loss.backward()
mpi_avg_grads(ac.ac) # average grads across MPI processes
optimizer.step()
logger.store(StopIter=i)
# Log changes from update
kl, ent, cf = pi_info['kl'], pi_info_old['ent'], pi_info['cf']
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(loss_pi.item() - pi_l_old),
DeltaLossV=(loss_v.item() - v_l_old))
# Prepare for interaction with environment
start_time = time.time()
timestep, ep_ret, ep_len = env.reset(difficulty=0), 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
encountered_terminal = False
for t in range(local_steps_per_epoch):
# assumes obs is an rgb array: rescale to [0, 1]
o = timestep.observation / 255.0
a, v, logp = ac.step(o)
next_timestep = env.step(ac.action_to_dict(a, rescale=True))
r = timestep.reward
d = next_timestep.last(
) # TODO: check if r, d are assoc w/ correct timestep
ep_ret += r
ep_len += 1
# save and log
buf.store(o, a, r, v, logp)
logger.store(VVals=v)
# TODO debugging
logger.store(AHor=a[0])
logger.store(AVer=a[1])
logger.store(ASel=a[3])
# Update obs (critical!)
timestep = next_timestep
timeout = ep_len == max_ep_len
terminal = d or timeout
epoch_ended = t == local_steps_per_epoch - 1
if terminal or epoch_ended:
if epoch_ended and not (terminal):
print(
f'Warning: trajectory cut off by epoch at {ep_len} steps.',
flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
if timeout or epoch_ended:
_, v, _ = ac.step(timestep.observation / 255.0)
else:
v = 0
buf.finish_path(v)
if terminal:
# only save EpRet / EpLen if trajectory finished.
logger.store(EpRet=ep_ret, EpLen=ep_len)
encountered_terminal = True
timestep, ep_ret, ep_len = env.reset(difficulty=0), 0, 0
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
if encountered_terminal:
# Note, if local_steps_per_epoch is too small so no terminal state
# has been encountered, then ep_ret and ep_len will not
# be stored before call to log_tabular, resulting in error.
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
# TODO debugging
logger.log_tabular('AHor', with_min_and_max=True)
logger.log_tabular('AVer', with_min_and_max=True)
logger.log_tabular('ASel', with_min_and_max=True)
# Save model
if (epoch % save_freq == 0 and epoch > 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
if proc_id() == 0 and log_wandb:
# Save the model parameters to wandb every save_freq epoch
# instead of waiting till the end
state = {
'epoch': epoch,
'ac_state_dict': ac.ac.state_dict(),
'optimizer': optimizer.state_dict(),
}
# output the model in the wandb.run.dir to avoid problems
# syncing the model in the cloud with wandb's files
state_fname = os.path.join(wandb.run.dir, "state_dict.pt")
torch.save(state, state_fname)
if proc_id() == 0 and log_wandb:
wandb.log(logger.log_current_row, step=epoch)
logger.dump_tabular()
def vpg(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
pi_lr=3e-4,
vf_lr=1e-3,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10,
explorer=None,
eps=0.05,
pretrain_epochs=0):
tf.reset_default_graph()
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = VPGBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# VPG objectives
pi_loss = -tf.reduce_mean(logp * adv_ph)
v_loss = tf.reduce_mean((ret_ph - v)**2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-logp) # a sample estimate for entropy, also easy to compute
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def update():
inputs = {k: v for k, v in zip(all_phs, buf.get())}
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent],
feed_dict=inputs)
# Policy gradient step
sess.run(train_pi, feed_dict=inputs)
# Value function learning
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
pi_l_new, v_l_new, kl = sess.run([pi_loss, v_loss, approx_kl],
feed_dict=inputs)
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
total_epochs = epochs + pretrain_epochs
# Main loop: collect experience in env and update/log each epoch
for epoch in range(total_epochs):
# TODO(abbyvs): deteriorate eps somehow. factor of .99?
# eps = eps*0.99
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops,
feed_dict={x_ph: o.reshape(1, -1)})
# explore if you are in a pretrain epoch or if eps-greedy
pre = epoch < pretrain_epochs
during = random.random() < eps
if pre or during:
if explorer is None:
raise ValueError('Trying to explore but explorer is None')
state = env.env.state_vector()
a = explorer.sample_action(state)
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(
v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform VPG update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def trpo(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=250,
gamma=0.99,
delta=0.01,
vf_lr=1e-3,
train_v_iters=80,
damping_coeff=0.1,
cg_iters=10,
backtrack_iters=10,
backtrack_coeff=0.8,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10,
algo='trpo'):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
============ ================ ========================================
Symbol Shape Description
============ ================ ========================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``info`` N/A | A dict of any intermediate quantities
| (from calculating the policy or log
| probabilities) which are needed for
| analytically computing KL divergence.
| (eg sufficient statistics of the
| distributions)
``info_phs`` N/A | A dict of placeholders for old values
| of the entries in ``info``.
``d_kl`` () | A symbol for computing the mean KL
| divergence between the current policy
| (``pi``) and the old policy (as
| specified by the inputs to
| ``info_phs``) over the batch of
| states given in ``x_ph``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
============ ================ ========================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to TRPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
delta (float): KL-divergence limit for TRPO / NPG update.
(Should be small for stability. Values like 0.01, 0.05.)
vf_lr (float): Learning rate for value function optimizer.
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
damping_coeff (float): Artifact for numerical stability, should be
smallish. Adjusts Hessian-vector product calculation:
.. math:: Hv \\rightarrow (\\alpha I + H)v
where :math:`\\alpha` is the damping coefficient.
Probably don't play with this hyperparameter.
cg_iters (int): Number of iterations of conjugate gradient to perform.
Increasing this will lead to a more accurate approximation
to :math:`H^{-1} g`, and possibly slightly-improved performance,
but at the cost of slowing things down.
Also probably don't play with this hyperparameter.
backtrack_iters (int): Maximum number of steps allowed in the
backtracking line search. Since the line search usually doesn't
backtrack, and usually only steps back once when it does, this
hyperparameter doesn't often matter.
backtrack_coeff (float): How far back to step during backtracking line
search. (Always between 0 and 1, usually above 0.5.)
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
algo: Either 'trpo' or 'npg': this code supports both, since they are
almost the same.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph, plus placeholders for old pdist (for KL)
pi, logp, logp_pi, info, info_phs, d_kl, v = actor_critic(
x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph
] + core.values_as_sorted_list(info_phs)
# Every step, get: action, value, logprob, & info for pdist (for computing kl div)
get_action_ops = [pi, v, logp_pi] + core.values_as_sorted_list(info)
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
info_shapes = {k: v.shape.as_list()[1:] for k, v in info_phs.items()}
buf = GAEBuffer(obs_dim, act_dim, local_steps_per_epoch, info_shapes,
gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# TRPO losses
ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s)
pi_loss = -tf.reduce_mean(ratio * adv_ph)
v_loss = tf.reduce_mean((ret_ph - v)**2)
# Optimizer for value function
train_vf = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
# Symbols needed for CG solver
pi_params = core.get_vars('pi')
gradient = core.flat_grad(pi_loss, pi_params)
v_ph, hvp = core.hessian_vector_product(d_kl, pi_params)
if damping_coeff > 0:
hvp += damping_coeff * v_ph
# Symbols for getting and setting params
get_pi_params = core.flat_concat(pi_params)
set_pi_params = core.assign_params_from_flat(v_ph, pi_params)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def cg(Ax, b):
"""
Conjugate gradient algorithm
(see https://en.wikipedia.org/wiki/Conjugate_gradient_method)
"""
x = np.zeros_like(b)
r = b.copy(
) # Note: should be 'b - Ax(x)', but for x=0, Ax(x)=0. Change if doing warm start.
p = r.copy()
r_dot_old = np.dot(r, r)
for _ in range(cg_iters):
z = Ax(p)
alpha = r_dot_old / (np.dot(p, z) + EPS)
x += alpha * p
r -= alpha * z
r_dot_new = np.dot(r, r)
p = r + (r_dot_new / r_dot_old) * p
r_dot_old = r_dot_new
return x
def update():
# Prepare hessian func, gradient eval
inputs = {k: v for k, v in zip(all_phs, buf.get())}
Hx = lambda x: mpi_avg(sess.run(hvp, feed_dict={**inputs, v_ph: x}))
g, pi_l_old, v_l_old = sess.run([gradient, pi_loss, v_loss],
feed_dict=inputs)
g, pi_l_old = mpi_avg(g), mpi_avg(pi_l_old)
# Core calculations for TRPO or NPG
x = cg(Hx, g)
alpha = np.sqrt(2 * delta / (np.dot(x, Hx(x)) + EPS))
old_params = sess.run(get_pi_params)
def set_and_eval(step):
sess.run(set_pi_params,
feed_dict={v_ph: old_params - alpha * x * step})
return mpi_avg(sess.run([d_kl, pi_loss], feed_dict=inputs))
if algo == 'npg':
# npg has no backtracking or hard kl constraint enforcement
kl, pi_l_new = set_and_eval(step=1.)
elif algo == 'trpo':
# trpo augments npg with backtracking line search, hard kl
for j in range(backtrack_iters):
kl, pi_l_new = set_and_eval(step=backtrack_coeff**j)
if kl <= delta and pi_l_new <= pi_l_old:
logger.log(
'Accepting new params at step %d of line search.' % j)
logger.store(BacktrackIters=j)
break
if j == backtrack_iters - 1:
logger.log('Line search failed! Keeping old params.')
logger.store(BacktrackIters=j)
kl, pi_l_new = set_and_eval(step=0.)
# Value function updates
for _ in range(train_v_iters):
sess.run(train_vf, feed_dict=inputs)
v_l_new = sess.run(v_loss, feed_dict=inputs)
# Log changes from update
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
agent_outs = sess.run(get_action_ops,
feed_dict={x_ph: o.reshape(1, -1)})
a, v_t, logp_t, info_t = agent_outs[0][0], agent_outs[
1], agent_outs[2], agent_outs[3:]
# save and log
buf.store(o, a, r, v_t, logp_t, info_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a)
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(
v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform TRPO or NPG update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('KL', average_only=True)
if algo == 'trpo':
logger.log_tabular('BacktrackIters', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def vpg(env_fn,
actor_critic=core.MLPActorCritic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
pi_lr=3e-4,
vf_lr=1e-3,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10):
"""
Vanilla Policy Gradient
(with GAE-Lambda for advantage estimation)
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The constructor method for a PyTorch Module with a
``step`` method, an ``act`` method, a ``pi`` module, and a ``v``
module. The ``step`` method should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``a`` (batch, act_dim) | Numpy array of actions for each
| observation.
``v`` (batch,) | Numpy array of value estimates
| for the provided observations.
``logp_a`` (batch,) | Numpy array of log probs for the
| actions in ``a``.
=========== ================ ======================================
The ``act`` method behaves the same as ``step`` but only returns ``a``.
The ``pi`` module's forward call should accept a batch of
observations and optionally a batch of actions, and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` N/A | Torch Distribution object, containing
| a batch of distributions describing
| the policy for the provided observations.
``logp_a`` (batch,) | Optional (only returned if batch of
| actions is given). Tensor containing
| the log probability, according to
| the policy, of the provided actions.
| If actions not given, will contain
| ``None``.
=========== ================ ======================================
The ``v`` module's forward call should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``v`` (batch,) | Tensor containing the value estimates
| for the provided observations. (Critical:
| make sure to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object
you provided to VPG.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
device = torch.device('cpu')
print(device)
# Special function to avoid certain slowdowns from PyTorch + MPI combo.
setup_pytorch_for_mpi()
# Set up logger and save configuration
logger = EpochLogger(**logger_kwargs)
print(logger_kwargs)
logger.save_config(locals())
# Random seed
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# Instantiate environment
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Create actor-critic module
ac = actor_critic(env.observation_space, env.action_space,
**ac_kwargs).to(device)
# Sync params across processes
sync_params(ac)
# Count variables
var_counts = tuple(core.count_vars(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# Set up experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = VPGBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Set up function for computing VPG policy loss
def compute_loss_pi(data):
obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data[
'logp']
obs, act, adv, logp_old = obs.to(device), act.to(device), adv.to(
device), logp_old.to(device)
# Policy loss
pi, logp = ac.pi(obs, act)
loss_pi = -(logp * adv).mean()
# Useful extra info
approx_kl = (logp_old - logp).mean().item()
ent = pi.entropy().mean().item()
pi_info = dict(kl=approx_kl, ent=ent)
return loss_pi, pi_info
# Set up function for computing value loss
def compute_loss_v(data):
obs, ret = data['obs'], data['ret']
return ((ac.v(obs) - ret)**2).mean()
# Set up optimizers for policy and value function
pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)
# Set up model saving
logger.setup_pytorch_saver(ac)
def update():
data = buf.get()
# Get loss and info values before update
pi_l_old, pi_info_old = compute_loss_pi(data)
pi_l_old = pi_l_old.item()
v_l_old = compute_loss_v(data).item()
# Train policy with a single step of gradient descent
pi_optimizer.zero_grad()
loss_pi, pi_info = compute_loss_pi(data)
loss_pi.backward()
mpi_avg_grads(ac.pi) # average grads across MPI processes
pi_optimizer.step()
# Value function learning
for i in range(train_v_iters):
vf_optimizer.zero_grad()
loss_v = compute_loss_v(data)
loss_v.backward()
mpi_avg_grads(ac.v) # average grads across MPI processes
vf_optimizer.step()
# Log changes from update
kl, ent = pi_info['kl'], pi_info_old['ent']
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
DeltaLossPi=(loss_pi.item() - pi_l_old),
DeltaLossV=(loss_v.item() - v_l_old))
# Prepare for interaction with environment
start_time = time.time()
o, ep_ret, ep_len = env.reset(), 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v, logp = ac.step(torch.as_tensor(o, dtype=torch.float32))
next_o, r, d, _ = env.step(a)
ep_ret += r
ep_len += 1
# save and log
buf.store(o, a, r, v, logp)
logger.store(VVals=v)
# Update obs (critical!)
o = next_o
timeout = ep_len == max_ep_len
terminal = d or timeout
epoch_ended = t == local_steps_per_epoch - 1
if terminal or epoch_ended:
if epoch_ended and not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len,
flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
if timeout or epoch_ended:
_, v, _ = ac.step(torch.as_tensor(o, dtype=torch.float32))
else:
v = 0
buf.finish_path(v)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform VPG update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo(env,
actor_critic=core.MLPActorCritic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=2048,
epochs=250,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=100,
train_v_iters=70,
lam=0.95,
max_ep_len=512,
target_kl=0.005,
logger_kwargs=dict(),
save_freq=5):
"""
Proximal Policy Optimization (by clipping),
with early stopping based on approximate KL
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The constructor method for a PyTorch Module with a
``step`` method, an ``act`` method, a ``pi`` module, and a ``v``
module. The ``step`` method should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``a`` (batch, act_dim) | Numpy array of actions for each
| observation.
``v`` (batch,) | Numpy array of value estimates
| for the provided observations.
``logp_a`` (batch,) | Numpy array of log probs for the
| actions in ``a``.
=========== ================ ======================================
The ``act`` method behaves the same as ``step`` but only returns ``a``.
The ``pi`` module's forward call should accept a batch of
observations and optionally a batch of actions, and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` N/A | Torch Distribution object, containing
| a batch of distributions describing
| the policy for the provided observations.
``logp_a`` (batch,) | Optional (only returned if batch of
| actions is given). Tensor containing
| the log probability, according to
| the policy, of the provided actions.
| If actions not given, will contain
| ``None``.
=========== ================ ======================================
The ``v`` module's forward call should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``v`` (batch,) | Tensor containing the value estimates
| for the provided observations. (Critical:
| make sure to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object
you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.) Typically
denoted by :math:`\epsilon`.
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
# Special function to avoid certain slowdowns from PyTorch + MPI combo.
setup_pytorch_for_mpi()
# Set up logger and save configuration
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# Random seed
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# Instantiate environment
env = env(
"PandaPegIn",
has_offscreen_renderer=True,
# has_renderer=True,
use_camera_obs=False,
control_freq=100,
)
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Create actor-critic module
ac = actor_critic(env.observation_space, env.action_space, **ac_kwargs)
#加载预训练模型
# fname = "data/ppo_peg_in_add_delta_pos_plus_plus/ppo_peg_in_add_delta_pos_plus_plus_s0/pyt_save/model24.pt"
# pre_model = torch.load(fname)
# ac.pi = pre_model.pi
# ac.v =pre_model.v
#使用TensorboardX
writer = logger.create_writer()
# Sync params across processes
sync_params(ac)
# Count variables
var_counts = tuple(core.count_vars(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# Set up experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Set up function for computing PPO policy loss
def compute_loss_pi(data):
obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data[
'logp']
# Policy loss
pi, logp = ac.pi(
obs, act
) #变化的是网络pi data['obs']，data['act']，data['adv']，data['logp']在回合内未变
ratio = torch.exp(logp - logp_old)
clip_adv = torch.clamp(ratio, 1 - clip_ratio, 1 + clip_ratio) * adv
loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()
# Useful extra info
approx_kl = (logp_old - logp).mean().item()
ent = pi.entropy().mean().item()
clipped = ratio.gt(1 + clip_ratio) | ratio.lt(1 - clip_ratio)
clipfrac = torch.as_tensor(clipped, dtype=torch.float32).mean().item()
pi_info = dict(kl=approx_kl, ent=ent, cf=clipfrac)
return loss_pi, pi_info #需要最小化loss_pi，pi_info包括kl散度、熵、越界程度（都针对单个回合）
# Set up function for computing value loss
def compute_loss_v(data):
obs, ret = data['obs'], data['ret']
return ((ac.v(obs) - ret)**2).mean()
# Set up optimizers for policy and value function
pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)
# Set up model saving
logger.setup_pytorch_saver(ac)
def update():
data = buf.get() #一次更新改变一次data
pi_l_old, pi_info_old = compute_loss_pi(data)
pi_l_old = pi_l_old.item()
v_l_old = compute_loss_v(data).item()
# Train policy with multiple steps of gradient descent
for i in range(train_pi_iters): #在kl散度不超标的前提下尽可能减小损失
pi_optimizer.zero_grad()
loss_pi, pi_info = compute_loss_pi(data)
kl = mpi_avg(pi_info['kl'])
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' % i)
break
loss_pi.backward()
mpi_avg_grads(ac.pi) # average grads across MPI processes
pi_optimizer.step()
logger.store(StopIter=i)
# Value function learning
for i in range(train_v_iters):
vf_optimizer.zero_grad()
loss_v = compute_loss_v(data)
loss_v.backward()
mpi_avg_grads(ac.v) # average grads across MPI processes
vf_optimizer.step()
# print(i,':',loss_v)
# print('='*20)
# Log changes from update
kl, ent, cf = pi_info['kl'], pi_info_old['ent'], pi_info['cf']
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(loss_pi.item() - pi_l_old),
DeltaLossV=(loss_v.item() - v_l_old))
# Prepare for interaction with environment
start_time = time.time()
o, ep_ret, ep_len = env.reset(), 0, 0
#运动到初始位置
pre_action = [0, 0, 0]
for i in range(4):
o, _, _, _ = env.step(pre_action)
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
print("epoch:", epoch)
for t in range(local_steps_per_epoch):
# if( t == steps_per_epoch/2 ):
# print("Half finished!")
#通过policy网络和值函数网络计算出:动作、值函数和采取这个动作的概率
a, v, logp = ac.step(torch.as_tensor(o, dtype=torch.float32))
next_o, r, d, _ = env.step(a)
ep_ret += r #单次游戏回报
ep_len += 1 #单次游戏时长
# save and log
buf.store(o, a, r, v, logp)
logger.store(VVals=v)
# Update obs (critical!)
o = next_o
timeout = ep_len == max_ep_len
terminal = d or timeout
epoch_ended = t == local_steps_per_epoch - 1
if terminal or epoch_ended: #game die; game超出时间; 回合结束
if epoch_ended and not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len,
flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
if timeout or epoch_ended:
_, v, _ = ac.step(torch.as_tensor(o, dtype=torch.float32))
else:
v = 0
buf.finish_path(v) #计算GAE和rewards-to-go
# print("steps:",t)
# print("done",d)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, epoch)
# Perform PPO update!
update()
#将数据写入TensorboardX
stats_to_write = logger.get_stats('EpRet')
writer.add_scalar('AverageEpRet',
stats_to_write[0],
global_step=(epoch + 1) * 2048)
# Log info about epoch 一个回合的数据
logger.log_tabular('Epoch', epoch) #第几个回合
logger.log_tabular('EpRet',
with_min_and_max=True) #回报的最大、最小、平均值，游戏结束时停留的状态的回报
logger.log_tabular('EpLen', average_only=True) #单次游戏长度的平均值
logger.log_tabular('VVals', with_min_and_max=True) #值函数的最大、最小、平均值
logger.log_tabular('TotalEnvInteracts',
(epoch + 1) * steps_per_epoch) #目前总步数
logger.log_tabular('LossPi', average_only=True) #回合开始时策略网络的损失
logger.log_tabular('LossV', average_only=True) #回合开始时值网络的损失
logger.log_tabular('DeltaLossPi', average_only=True) #策略网络回合结束损失-开始损失
logger.log_tabular('DeltaLossV', average_only=True) #值略网络回合结束损失-开始损失
logger.log_tabular('Entropy', average_only=True) #？
logger.log_tabular('KL', average_only=True) #散度值
logger.log_tabular('ClipFrac', average_only=True) #越界程度
logger.log_tabular('StopIter', average_only=True) #ppo策略网络迭代次数
logger.log_tabular('Time', time.time() - start_time) #回合时间
logger.dump_tabular()
# if __name__ == '__main__':
# import argparse
# parser = argparse.ArgumentParser()
# parser.add_argument('--env', type=str, default='HalfCheetah-v2')
# parser.add_argument('--hid', type=int, default=64)
# parser.add_argument('--l', type=int, default=2)
# parser.add_argument('--gamma', type=float, default=0.99)
# parser.add_argument('--seed', '-s', type=int, default=0)
# parser.add_argument('--cpu', type=int, default=1)
# parser.add_argument('--steps', type=int, default=4000)
# parser.add_argument('--epochs', type=int, default=50)
# parser.add_argument('--exp_name', type=str, default='ppo')
# args = parser.parse_args()
# mpi_fork(args.cpu) # run parallel code with mpi
# from spinup.utils.run_utils import setup_logger_kwargs
# logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed)
# ppo(lambda : gym.make(args.env), actor_critic=core.MLPActorCritic,
# ac_kwargs=dict(hidden_sizes=[args.hid]*args.l), gamma=args.gamma,
# seed=args.seed, steps_per_epoch=args.steps, epochs=args.epochs,
# logger_kwargs=logger_kwargs)
def ppo(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=33,
steps_per_epoch=4000,
epochs=50,
gamma=0.998,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=60,
train_v_iters=60,
lam=0.95,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10):
"""
Proximal Policy Optimization (by clipping),
with early stopping based on approximate KL
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.) Typically
denoted by :math:`\epsilon`.
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
## Logger setup
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
## Random seed setting
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
## Environment instantiation
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
#Policies vector (only one for this project)
policies = []
#TensorFlow session
sess = tf.Session()
#Build policy anc value networks
MAP = MAPolicy(scope='policy_0',
ob_space=env.observation_space,
ac_space=env.action_space,
network_spec=pi_specs,
normalize=True,
v_network_spec=v_specs)
policies = [MAP]
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Create aux placeholders for the computation graph
adv_ph, ret_ph, logp_old_ph = core.placeholders(1, 1, 1)
# Get main placeholders for the computation graph
map_phs_dict = MAP.phs
map_phs = [v for k, v in map_phs_dict.items()]
for k, v in map_phs_dict.items():
if v.name == None:
v.name = k
# Append aux and main placeholders
# Need placeholders in *this* order later (to zip with data from buffer)
new_phs = [adv_ph, ret_ph, logp_old_ph]
all_phs = np.append(map_phs, new_phs)
# Intantiate Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(
core.count_vars(scope) for scope in ['policy_net', 'vpred_net'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# PPO objectives
ratio = tf.exp(MAP.taken_action_logp -
logp_old_ph) # pi(a|s) / pi_old(a|s)
min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph,
(1 - clip_ratio) * adv_ph) # PPO-clip limits
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph,
min_adv)) # Policy loss function
v_loss = tf.reduce_mean(
(ret_ph - MAP.scaled_value_tensor)**2) # Value loss function
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph - MAP.taken_action_logp
) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-MAP.taken_action_logp
) # a sample estimate for entropy, also easy to compute
clipped = tf.logical_or(
ratio > (1 + clip_ratio), ratio < (1 - clip_ratio)
) # a logical value which states whether there was clipping
clipfrac = tf.reduce_mean(tf.cast(
clipped, tf.float32)) # a measure of clipping for posterior analysis
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(
pi_loss) #Policy network optimizer
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(
v_loss) #Value network optimizer
#initialize TensorFlow variabels
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Set up logger variables to be saved (it is necessary to save everything that is
# input/output to the networks so that the policy can be played afterwards during testing)
out_act_dict = MAP.sampled_action
out_state_dict = MAP.state_out
logger_outputs = {**out_act_dict, **out_state_dict}
for k, v in logger_outputs.items():
if 'lstm' in k:
logger_outputs[k + '_out'] = logger_outputs.pop(k)
logger_inputs = map_phs_dict
logger.setup_tf_saver(sess, inputs=logger_inputs, outputs=logger_outputs)
# ======================================================================== #
# ===================== Auxiliary Training Functions ===================== #
# ======================================================================== #
# Compute metrics for analysis during and after training
def compute_metrics(extra_dict={}):
loss_outs = {
'pi_loss': pi_loss,
'v_loss': v_loss,
'approx_ent': approx_ent,
'approx_kl': approx_kl,
'approx_cf': clipfrac,
'taken_action_logp': MAP.taken_action_logp,
'ratio': ratio,
'min_adv': min_adv
}
out_loss = policies[0].sess_run(buf.obs_buf,
sess_act=sess,
extra_feed_dict=extra_dict,
other_outputs=loss_outs,
replace=True)
return out_loss['pi_loss'], out_loss['v_loss'], out_loss[
'approx_ent'], out_loss['approx_kl'], out_loss['approx_cf']
# ======================================================================= #
# Run session on policy and value optimizers for training their respective networks
def train(net, extra_dict={}):
if net == 'pi':
train_outs = {'train_pi': train_pi, 'approx_kl': approx_kl}
elif net == 'v':
train_outs = {'train_v': train_v}
else:
print("Error: Network not defined")
return
out_train = policies[0].sess_run(buf.obs_buf,
sess_act=sess,
extra_feed_dict=extra_dict,
other_outputs=train_outs,
replace=True)
if net == 'pi':
return out_train['approx_kl']
# ======================================================================= #
# Perform training procedure
def update():
print("======= update!")
#get aux data from the buffer and match it with its respective placeholders
buf_data = buf.get(aux_vars_only=True)
aux_inputs = {k: v for k, v in zip(new_phs, buf_data)}
#for the training, the actions taken during the experience loop are also inputs to the network
extra_dict = {k: v for k, v in buf.act_buf.items() if k is not 'vpred'}
for k, v in extra_dict.items():
if k == 'action_movement':
extra_dict[k] = np.expand_dims(v, 1)
#actions and aux variables from the buffer are joined and passed to compute_metrics (observations are joined within the functions)
extra_dict.update(aux_inputs)
pi_l_old, v_l_old, ent, kl, cf = compute_metrics(extra_dict)
# Policy training loop
for i in range(train_pi_iters):
if i % 10 == 0:
print("training pi iter ", i)
kl = train('pi', extra_dict)
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' % i)
break
logger.store(StopIter=i)
print("")
# Value training loop
for j in range(train_v_iters):
if j % 10 == 0:
print("training v iter ", j)
train('v', extra_dict)
# Log changes from update with a new run on compute_metrics
pi_l_new, v_l_new, ent, kl, cf = compute_metrics(extra_dict)
#Store information
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
#Reset experience varibales
o, ep_ret, ep_len = env.reset(), 0, 0
#Reset policy
for policy in policies:
policy.reset()
print("======= update finished!")
# ======================================================================= #
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# ======================================================================= #
# ========================== Experience Loop ============================ #
# ======================================================================= #
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
print("epoch: ", epoch)
for t in range(local_steps_per_epoch):
# Pass observations through networs and get action + predicted value
if len(policies) == 1: #this project's case
a, info = policies[0].sess_run(o, sess_act=sess)
v_t = info['vpred']
logp_t = info['ac_logp']
else:
o = splitobs(o, keepdims=False)
ob_policy_idx = np.split(np.arange(len(o)), len(policies))
actions = []
for i, policy in enumerate(policies):
inp = itemgetter(*ob_policy_idx[i])(o)
inp = listdict2dictnp([inp] if ob_policy_idx[i].shape[0] ==
1 else inp)
ac, info = policy.act(inp)
actions.append(ac)
action = listdict2dictnp(actions, keepdims=True)
# Take a step in the environment
o2, r, d, env_info = env.step(a)
ep_ret += r
ep_len += 1
# If env.render is uncommented, the experience loop is displayed (visualized)
# in real time (much slower, but excelent debugging)
# env.render()
# save experience in buffer and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
# Update obs (critical!)
o = o2
# Treat the end of a trajectory
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1) or env_info.get(
'discard_episode', False):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
if d:
last_val = 0
else:
_, info = policies[0].sess_run(o, sess_act=sess)
last_val = info['vpred']
#Compute advantage estimates and rewards-to-go
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0
for policy in policies:
policy.reset()
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
print("Saved epoch: ", epoch)
logger.save_state({'env': env}, None)
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def ppo(env_fn, actor_critic=core.mlp_actor_critic, ac_kwargs=dict(), seed=0,
steps_per_epoch=4000, epochs=50, gamma=0.99, clip_ratio=0.2, pi_lr=3e-4,
vf_lr=1e-3, train_pi_iters=80, train_v_iters=80, lam=0.97, max_ep_len=1000,
target_kl=0.01, logger_kwargs=dict(), save_freq=10, custom_h=None, eval_episodes=50,
do_checkpoint_eval=False, env_name=None, eval_temp=1.0, train_starting_temp=1.0,
env_version=None, env_input=None, target_arcs=None):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# create logger for tensorboard
tb_logdir = "{}/tb_logs/".format(logger.output_dir)
tb_logger = Logger(log_dir=tb_logdir)
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
if custom_h is not None:
hidden_layers_str_list = custom_h.split('-')
hidden_layers_int_list = [int(h) for h in hidden_layers_str_list]
ac_kwargs['hidden_sizes'] = hidden_layers_int_list
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space, env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
temperature_ph = tf.placeholder(tf.float32, shape=(), name="init")
# Main outputs from computation graph
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, temperature_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph, temperature_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# PPO objectives
ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s)
min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph, (1 - clip_ratio) * adv_ph)
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv))
v_loss = tf.reduce_mean((ret_ph - v) ** 2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(logp_old_ph - logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(-logp) # a sample estimate for entropy, also easy to compute
clipped = tf.logical_or(ratio > (1 + clip_ratio), ratio < (1 - clip_ratio))
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32))
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
# create a tf session with GPU memory usage option to be allow_growth so that one program will not use up the
# whole GPU memory
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
# log tf graph
tf.summary.FileWriter(tb_logdir, sess.graph)
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph, 'temperature': temperature_ph}, outputs={'pi': pi, 'v': v})
def update():
inputs = {k: v for k, v in zip(all_phs, buf.get())}
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent], feed_dict=inputs)
# Training
for i in range(train_pi_iters):
_, kl = sess.run([train_pi, approx_kl], feed_dict=inputs)
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log('Early stopping at step %d due to reaching max kl.' % i)
break
logger.store(StopIter=i)
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
pi_l_new, v_l_new, kl, cf = sess.run([pi_loss, v_loss, approx_kl, clipfrac], feed_dict=inputs)
logger.store(LossPi=pi_l_old, LossV=v_l_old,
KL=kl, Entropy=ent, ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len, ep_dummy_action_count, ep_dummy_steps_normalized = env.reset(), 0, False, 0, 0, 0, []
# initialize variables for keeping track of BEST eval performance
best_eval_AverageEpRet = -0.05 # a negative value so that best model is saved at least once.
best_eval_StdEpRet = 1.0e30
# save is used to only allow saving BEST models after half of training epochs
save = True
# below are used for early-stop. We early stop if
# 1) a best model has been saved, and,
# 2) 50 epochs have passed without a new save
saved = False
early_stop_count_started = False
episode_count_after_saved = 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
current_temp = _get_current_temperature(epoch, epochs, train_starting_temp)
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops, feed_dict={x_ph: o.reshape(1, -1),
temperature_ph: current_temp})
# save and log
buf.store(o, a, r, v_t, logp_t, current_temp)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
ep_ret += r
ep_len += 1
if env_version >= 4 and env.action_is_dummy: # a is dummy action
ep_dummy_action_count += 1
ep_dummy_steps_normalized.append(ep_len / env.allowed_steps)
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' % ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(v, feed_dict={x_ph: o.reshape(1, -1),
temperature_ph: current_temp})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
if env_version >= 4:
logger.store(EpDummyCount=ep_dummy_action_count)
logger.store(EpTotalArcs=env.adjacency_matrix.sum())
if len(ep_dummy_steps_normalized) > 0:
ep_dummy_steps_normalized = np.asarray(ep_dummy_steps_normalized, dtype=np.float32).mean()
logger.store(EpDummyStepsNormalized=ep_dummy_steps_normalized)
o, r, d, ep_ret, ep_len, ep_dummy_action_count, ep_dummy_steps_normalized = env.reset(), 0, False, 0, 0, 0, []
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
# Save a new model every save_freq and at the last epoch. Do not overwrite the previous save.
# logger.save_state({'env_name': env_name}, epoch)
# # Save a new model every save_freq and at the last epoch. Only keep one copy - the current model
# logger.save_state({'env_name': env_name})
# Evaluate and save best model
if do_checkpoint_eval and epoch > 0:
# below is a hack. best model related stuff is saved at itr 999999, therefore, simple_save999999.
# Doing this way, I can use test_policy and plot directly to test the best models.
# saved best models includes:
# 1) a copy of the env_name
# 2) the best rl model with parameters
# 3) a pickle file "best_eval_performance_n_structure" storing best_performance, best_structure and epoch
# note that 1) and 2) are spinningup defaults, and 3) is a custom save
best_eval_AverageEpRet, best_eval_StdEpRet, saved = eval_and_save_best_model(
best_eval_AverageEpRet,
best_eval_StdEpRet,
# a new logger is created and passed in so that the new logger can leverage the directory
# structure without messing up the logger in the training loop
eval_logger=EpochLogger(**dict(
exp_name=logger_kwargs['exp_name'],
output_dir=os.path.join(logger.output_dir, "simple_save999999"))),
train_logger=logger,
tb_logger=tb_logger,
epoch=epoch,
# the env_name is passed in so that to create an env when and where it is needed. This is to
# logx.save_state() error where an env pointer cannot be pickled
env_name="F{}x{}T{}_SP{}_v{}".format(env.n_plant, env.n_product, env.target_arcs, env.n_sample,
env_version) if env_version >= 3 else env_name,
env_version=env_version,
env_input=env_input,
render=False, # change this to True if you want to visualize how arcs are added during evaluation
target_arcs=env.target_arcs,
get_action=lambda x: sess.run(pi, feed_dict={x_ph: x[None, :],
temperature_ph: eval_temp})[0],
# number of samples to draw when simulate demand
n_sample=5000,
num_episodes=eval_episodes,
save=save,
seed=seed
)
# Perform PPO update!
update()
# # # Log into tensorboard
log_key_to_tb(tb_logger, logger, epoch, key="EpRet", with_min_and_max=True)
log_key_to_tb(tb_logger, logger, epoch, key="EpLen", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="VVals", with_min_and_max=True)
log_key_to_tb(tb_logger, logger, epoch, key="LossPi", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="LossV", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="DeltaLossPi", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="DeltaLossV", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="Entropy", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="KL", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="ClipFrac", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="StopIter", with_min_and_max=False)
tb_logger.log_scalar(tag="TotalEnvInteracts", value=(epoch + 1) * steps_per_epoch, step=epoch)
tb_logger.log_scalar(tag="Time", value=time.time() - start_time, step=epoch)
tb_logger.log_scalar(tag="epoch_temp", value=current_temp, step=epoch)
if env_version >= 4:
log_key_to_tb(tb_logger, logger, epoch, key="EpDummyCount", with_min_and_max=False)
log_key_to_tb(tb_logger, logger, epoch, key="EpTotalArcs", with_min_and_max=False)
if len(logger.epoch_dict['EpDummyStepsNormalized']) > 0:
log_key_to_tb(tb_logger, logger, epoch, key="EpDummyStepsNormalized", with_min_and_max=False)
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.log_tabular('EpochTemp', current_temp)
if env_version >= 4:
logger.log_tabular('EpDummyCount', with_min_and_max=True)
if len(logger.epoch_dict['EpDummyStepsNormalized']) > 0:
logger.log_tabular('EpDummyStepsNormalized', average_only=True)
logger.log_tabular('EpTotalArcs', average_only=True)
logger.dump_tabular()
# check for early stop
if saved:
# start to count the episodes elapsed after a "saved" event
early_stop_count_started = True
# reset the count to 0
episode_count_after_saved = 0
else:
# check whether we should count this episode, i.e., whether early_stop_count_started == True
if early_stop_count_started:
episode_count_after_saved += 1
if episode_count_after_saved > 60:
logger.log('Early Stopped at epoch {}.'.format(epoch), color='cyan')
break
def __init__(self,
env_fn,
actor_critic=core.MLPActorCritic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=80,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=None,
save_freq=10,
train_graph_path='/home/visgean/',
train_graph_name='return.svg',
model=None):
self.actor_critic = actor_critic
self.ac_kwargs = ac_kwargs
self.seed = seed
self.steps_per_epoch = steps_per_epoch
self.epochs = epochs
self.gamma = gamma
self.clip_ratio = clip_ratio
self.pi_lr = pi_lr
self.vf_lr = vf_lr
self.train_pi_iters = train_pi_iters
self.train_v_iters = train_v_iters
self.lam = lam
self.max_ep_len = max_ep_len
self.target_kl = target_kl
self.logger_kwargs = logger_kwargs if logger_kwargs else {}
self.save_freq = save_freq
# Special function to avoid certain slowdowns from PyTorch + MPI combo.
setup_pytorch_for_mpi()
# Set up logger and save configuration
self.logger = EpochLoggerFixed(**self.logger_kwargs)
self.logger.save_config(locals())
# Random seed
self.seed += 10000 * proc_id()
torch.manual_seed(self.seed)
np.random.seed(self.seed)
# Instantiate environment
self.env = env_fn()
self.obs_dim = self.env.observation_space.shape
self.act_dim = self.env.action_space.shape
# Create actor-critic module
if model:
self.ac = model
else:
self.ac = actor_critic(self.env.observation_space,
self.env.action_space, **ac_kwargs)
# Sync params across processes
sync_params(self.ac)
# Count variables
self.var_counts = tuple(
core.count_vars(module) for module in [self.ac.pi, self.ac.v])
self.logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' %
self.var_counts)
# Set up experience buffer
self.local_steps_per_epoch = int(steps_per_epoch / num_procs())
self.buf = PPOBuffer(self.obs_dim, self.act_dim,
self.local_steps_per_epoch, gamma, lam)
# Set up optimizers for policy and value function
self.pi_optimizer = Adam(self.ac.pi.parameters(), lr=pi_lr)
self.vf_optimizer = Adam(self.ac.v.parameters(), lr=vf_lr)
# Set up model saving
self.logger.setup_pytorch_saver(self.ac)
# Prepare for interaction with environment
self.start_time = time.time()
self.obs = self.env.reset()
self.ep_ret = 0
self.ep_len = 0
self.test_returns = []
self.train_returns = []
self.max_return = 0
self.test_lengths = []
self.train_graph_path = train_graph_path + f'{proc_id()}_{train_graph_name}'
def vpg(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
pi_lr=3e-4,
vf_lr=1e-3,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10):
"""
Vanilla Policy Gradient
(with GAE-Lambda for advantage estimation)
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to VPG.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
tf.compat.v1.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = VPGBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# VPG objectives
pi_loss = -tf.reduce_mean(input_tensor=logp * adv_ph)
v_loss = tf.reduce_mean(input_tensor=(ret_ph - v)**2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
input_tensor=logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
input_tensor=-logp
) # a sample estimate for entropy, also easy to compute
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
sess = tf.compat.v1.Session()
sess.run(tf.compat.v1.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def update():
inputs = {k: v for k, v in zip(all_phs, buf.get())}
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent],
feed_dict=inputs)
# Policy gradient step
sess.run(train_pi, feed_dict=inputs)
# Value function learning
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
pi_l_new, v_l_new, kl = sess.run([pi_loss, v_loss, approx_kl],
feed_dict=inputs)
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, ep_ret, ep_len = env.reset(), 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops,
feed_dict={x_ph: o.reshape(1, -1)})
o2, r, d, _ = env.step(a[0])
ep_ret += r
ep_len += 1
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
# Update obs (critical!)
o = o2
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = 0 if d else sess.run(
v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, ep_ret, ep_len = env.reset(), 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform VPG update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def trpo(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
delta=0.01,
vf_lr=1e-3,
train_v_iters=80,
damping_coeff=0.1,
cg_iters=10,
backtrack_iters=10,
backtrack_coeff=0.8,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10,
algo='trpo'):
# Special function to avoid certain slow down form PYTORCH + MPI combo
setup_pytorch_for_mpi()
# Setup logger and save configuration
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# Create random seed
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# Instantiate Environment
# Get dimensions of action and observation space
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Create actor-critic module
ac = actor_critic(env.observation_space, env.action_space, **ac_kwargs)
# Sync parameters across processes
sync_params(ac)
# Count variables and add the information to the logger
var_counts = tuple(
core.count_variables(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# set up experiment buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = TRPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# TODO: set up a function for computing TRPO policy loss also get useful extra information
# Setup function for computing value loss
def compute_loss_v(data):
obs, ret = data['obs'], data['ret']
return ((ac.v(obs) - ret)**2).mean()
# Set up optimizers for policy and value funtions
pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)
# Set Up model saving
logger.setup_pytorch_saver(ac)
# TODO: Create the update function for the TRPO policy
def update():
return
# Prepare for the interaction with the environment
start_time = time.time()
o, ep_ret, ep_len = env.reset(), 0, 0
def ppo(env_fn,
actor_critic=core.MLPActorCritic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
beta=0.01,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=3e-4,
train_pi_iters=80,
train_v_iters=80,
lam=0.95,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10,
use_rnn=False,
reward_factor=1,
spectrum_repr=False):
"""
Proximal Policy Optimization (by clipping),
with early stopping based on approximate KL
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The constructor method for a PyTorch Module with a
``step`` method, an ``act`` method, a ``pi`` module, and a ``v``
module. The ``step`` method should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``a`` (batch, act_dim) | Numpy array of actions for each
| observation.
``v`` (batch,) | Numpy array of value estimates
| for the provided observations.
``logp_a`` (batch,) | Numpy array of log probs for the
| actions in ``a``.
=========== ================ ======================================
The ``act`` method behaves the same as ``step`` but only returns ``a``.
The ``pi`` module's forward call should accept a batch of
observations and optionally a batch of actions, and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` N/A | Torch Distribution object, containing
| a batch of distributions describing
| the policy for the provided observations.
``logp_a`` (batch,) | Optional (only returned if batch of
| actions is given). Tensor containing
| the log probability, according to
| the policy, of the provided actions.
| If actions not given, will contain
| ``None``.
=========== ================ ======================================
The ``v`` module's forward call should accept a batch of observations
and return:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``v`` (batch,) | Tensor containing the value estimates
| for the provided observations. (Critical:
| make sure to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object
you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.) Typically
denoted by :math:`\epsilon`.
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
# Special function to avoid certain slowdowns from PyTorch + MPI combo.
setup_pytorch_for_mpi()
# Set up logger and save configuration
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
# Random seed
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# Instantiate environment
env = env_fn()
ac = actor_critic(env.observation_space, env.action_space, **ac_kwargs)
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
if rank == 0:
print(ac)
# udpate env config
# env.scalar_thick = ac_kwargs['scalar_thick']
env.update_with_ac(**ac_kwargs)
# For Tuple spaces
obs_dim = ac.obs_dim
if isinstance(env.action_space, spaces.Tuple):
act_dim = core.tuple_space_dim(env.action_space, action=True)
else:
act_dim = env.action_space.shape
# Create actor-critic module
# print(ac)
# Sync params across processes
sync_params(ac)
# Count variables
var_counts = tuple(core.count_vars(module) for module in [ac.pi, ac.v])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# Set up experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim,
act_dim,
local_steps_per_epoch,
gamma,
lam,
cell_size=ac_kwargs['cell_size'])
# Set up function for computing PPO policy loss
def compute_loss_pi(data):
obs, act, adv, logp_old, hid = data['obs'], data['act'], data[
'adv'], data['logp'], data['hid']
# for i in range(len(obs)-1):
# if torch.eq(obs[i], torch.zeros(12)).sum()==12 and torch.eq(obs[i+1], torch.zeros(12)).sum()==12:
# print(obs[i], obs[i+1], act[i], act[i+1])
# Policy loss
pis = []
logp = 0
if len(ac.pi) > 1: # tuple actions
for i, actor_i in enumerate(ac.pi):
pi, logp_i = actor_i(obs, act[:, i][:, None])
logp += logp_i
pis.append(pi)
else:
pi, logp_i = ac.pi[0](obs, act)
logp += logp_i
pis.append(pi)
ratio = torch.exp(logp - logp_old)
clip_adv = torch.clamp(ratio, 1 - clip_ratio, 1 + clip_ratio) * adv
loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()
# Useful extra info
# sample estimation policy KL
approx_kl = (logp_old - logp).mean().item()
ent = sum([pi.entropy().mean().item() for pi in pis])
clipped = ratio.gt(1 + clip_ratio) | ratio.lt(1 - clip_ratio)
clipfrac = torch.as_tensor(clipped, dtype=torch.float32).mean().item()
pi_info = dict(kl=approx_kl, ent=ent, cf=clipfrac)
return loss_pi, pi_info
# Set up function for computing value loss
def compute_loss_v(data):
obs, ret = data['obs'], data['ret']
return 0.5 * ((ac.v(obs) - ret)**2).mean()
def compute_loss_pi_v_rnn(data):
obs, act, adv, logp_old, ret = data['obs'], data['act'], data[
'adv'], data['logp'], data['ret']
hid = torch.zeros(ac_kwargs['cell_size'])
v = []
logp = []
ent = []
num_traj = 0
#todo: test
for i in range(len(obs)):
v_i, logp_i, hid, ent_i = ac.evaluate(obs[i], act[i], hid)
if i < len(obs) - 1 and obs[i + 1].sum() == 0:
num_traj += 1
# print('Reinitialize #{}'.format(num_traj), flush=True)
hid = torch.zeros(ac_kwargs['cell_size'])
v.append(v_i)
logp.append(logp_i)
ent.append(ent_i)
logp = torch.cat(logp)
v = torch.cat(v)
ratio = torch.exp(logp - logp_old)
clip_adv = torch.clamp(ratio, 1 - clip_ratio, 1 + clip_ratio) * adv
loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()
# print(logp_old - logp)
approx_kl = (logp_old - logp).mean().item()
ent = torch.stack(ent).mean()
clipped = ratio.gt(1 + clip_ratio) | ratio.lt(1 - clip_ratio)
clipfrac = torch.as_tensor(clipped, dtype=torch.float32).mean().item()
pi_info = dict(kl=approx_kl, ent=ent, cf=clipfrac)
loss_v = 0.5 * ((v - ret)**2).mean()
# import pdb; pdb.set_trace()
loss_pi = loss_pi - beta * ent
logger.store(RetBuf=ret.clone().detach().numpy())
# import pdb; pdb.set_trace()
return loss_pi, pi_info, loss_v
# Set up optimizers for policy and value function
pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)
vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)
if use_rnn:
optimizer = Adam(ac.parameters(), lr=pi_lr)
# Set up model saving
logger.setup_pytorch_saver(ac)
def update():
data = buf.get()
# import pdb; pdb.set_trace()
if not use_rnn:
pi_l_old, pi_info_old = compute_loss_pi(data)
v_l_old = compute_loss_v(data).item()
# Train policy with multiple steps of gradient descent
for i in range(train_pi_iters):
pi_optimizer.zero_grad()
loss_pi, pi_info = compute_loss_pi(data)
kl = mpi_avg(pi_info['kl'])
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' %
i)
break
loss_pi.backward()
mpi_avg_grads(ac.pi) # average grads across MPI processes
pi_optimizer.step()
logger.store(StopIter=i)
# Value function learning
for i in range(train_v_iters):
vf_optimizer.zero_grad()
if not use_rnn:
loss_v = compute_loss_v(data)
loss_v.backward()
mpi_avg_grads(ac.v) # average grads across MPI processes
vf_optimizer.step()
else:
pi_l_old, pi_info_old, v_l_old = compute_loss_pi_v_rnn(data)
pi_l_old = pi_l_old.item()
for i in range(train_pi_iters):
optimizer.zero_grad()
loss_pi, pi_info, loss_v = compute_loss_pi_v_rnn(data)
kl = mpi_avg(pi_info['kl'])
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' %
i)
break
loss = loss_pi + loss_v
loss.backward()
mpi_avg_grads(ac)
optimizer.step()
logger.store(StopIter=i)
# import pdb; pdb.set_trace()
# Log changes from update
kl, ent, cf = pi_info['kl'], pi_info_old['ent'], pi_info['cf']
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(loss_pi.item() - pi_l_old),
DeltaLossV=(loss_v.item() - v_l_old))
# Prepare for interaction with environment
start_time = time.time()
obs, ep_ret, ep_len = env.reset(), 0, 0
# import pdb; pdb.set_trace()
# if ac_kwargs['scalar_thick']:
# thick= obs[env.num_materials:env.num_materials+env.num_thicknesses].argmax() / env.num_thicknesses
# obs = np.concatenate((obs[:env.num_materials+1], np.array([thick])))
# if ac_kwargs['scalar_thick']:
# thick= obs[env.num_materials:env.num_materials+env.num_thicknesses].argmax() / env.num_thicknesses
# obs = np.concatenate((obs[:env.num_materials+1], np.array([thick])))
hid = np.zeros(
ac_kwargs['cell_size']) if ac_kwargs['cell_size'] else np.zeros(1)
# import pdb; pdb.set_trace()
design_tracker = DesignTracker(epochs, **logger_kwargs)
total_env_time = 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
epoch_start_time = time.time()
for t in range(local_steps_per_epoch):
#TODO: only evaluate
act, v, logp, hid = ac.step(
torch.as_tensor(obs, dtype=torch.float32),
torch.as_tensor(hid, dtype=torch.float32))
# nv_start = time.time()
next_obs, r, d, _ = env.step(act)
# env_end = time.time()
# env_time = env_end - env_start
# total_env_time += env_time
r = r * reward_factor # scale the rewards, possibly match the reward scale of atari
ep_ret += r
if not d:
ep_len += 1
# save and log
if use_rnn:
buf.store(obs, act, r, v, logp, hid)
else:
buf.store(obs, act, r, v, logp)
logger.store(VVals=v)
# Update obs (critical!)
obs = next_obs
timeout = ep_len == max_ep_len
terminal = d or timeout
epoch_ended = t == local_steps_per_epoch - 1
if terminal or epoch_ended:
# print(t)
# if epoch_ended and not(terminal):
# print('Warning: trajectory cut off by epoch at %d steps.'
# % ep_len, flush=True)
# if trajectory didn't reach terminal state, bootstrap value target
# if timeout or epoch_ended:
if not terminal:
_, v, _, _ = ac.step(
torch.as_tensor(obs, dtype=torch.float32),
torch.as_tensor(hid, dtype=torch.float32))
else:
v = 0
buf.finish_path(v)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
if hasattr(env, 'layers') and hasattr(env, 'thicknesses'):
design_tracker.store(env.layers, env.thicknesses,
ep_ret, epoch)
if rank == 0:
print(env.layers, env.thicknesses)
obs, ep_ret, ep_len = env.reset(), 0, 0
# reinitilize hidden state
hid = np.zeros(ac_kwargs['cell_size'])
if hasattr(env, "layers"):
logger.store(Act=act[1])
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
design_tracker.save_state()
# Perform PPO update!
update()
elapsed = time.time() - start_time
epoch_time = time.time() - epoch_start_time
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
if hasattr(env, 'layers'):
logger.log_tabular('Act', with_min_and_max=True)
logger.log_tabular('RetBuf', with_min_and_max=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', elapsed)
logger.log_tabular('FPS', int(steps_per_epoch / epoch_time))
logger.dump_tabular()
def ppo(
env_fn,
actor_critic=core.ActorCritic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=80,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10,
use_gpu=False,
gpu_parallel=False,
):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: The agent's main model which is composed of
the policy and value function model, where the policy takes
some state, ``x`` and action ``a``, and value function takes
the state ``x``. The model returns a tuple of:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a``
| in states ``x``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x``. (Critical: make sure
| to flatten this via .squeeze()!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
class you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Main model
actor_critic = actor_critic(in_features=obs_dim[0], **ac_kwargs)
# actor_critic = torch.nn.DataParallel(actor_critic).to(device)
# gpu是否使用
# device = torch.device("cpu" if USE_DEVICE=="cpu" else "cuda")
if torch.cuda.is_available():
device = torch.device("cuda" if use_gpu else "cpu")
if gpu_parallel:
actor_critic = torch.nn.DataParallel(actor_critic)
else:
use_gpu = False
use_parallel = False
device = torch.device("cpu")
actor_critic = actor_critic.to(device)
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(
core.count_vars(module)
for module in [actor_critic.policy, actor_critic.value_function])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# Optimizers
train_pi = torch.optim.Adam(actor_critic.policy.parameters(), lr=pi_lr)
train_v = torch.optim.Adam(actor_critic.value_function.parameters(),
lr=vf_lr)
# Sync params across processes
sync_all_params(actor_critic.parameters())
def update():
temp_get = buf.get()
obs, act, adv, ret, logp_old = [
torch.Tensor(x).to(device) for x in temp_get
]
# Training policy
_, logp, _ = actor_critic.policy(obs, act)
ratio = (logp - logp_old).exp()
min_adv = torch.where(adv > 0, (1 + clip_ratio) * adv,
(1 - clip_ratio) * adv)
pi_l_old = -(torch.min(ratio * adv, min_adv)).mean()
ent = (-logp).mean() # a sample estimate for entropy
for i in range(train_pi_iters):
# Output from policy function graph
_, logp, _ = actor_critic.policy(obs, act)
# PPO policy objective
ratio = (logp - logp_old).exp()
min_adv = torch.where(adv > 0, (1 + clip_ratio) * adv,
(1 - clip_ratio) * adv)
pi_loss = -(torch.min(ratio * adv, min_adv)).mean()
# Policy gradient step
train_pi.zero_grad()
pi_loss.backward()
average_gradients(train_pi.param_groups)
train_pi.step()
_, logp, _ = actor_critic.policy(obs, act)
kl = (logp_old - logp).mean()
kl = mpi_avg(kl.item())
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' % i)
break
logger.store(StopIter=i)
# Training value function
v = actor_critic.value_function(obs)
v_l_old = F.mse_loss(v, ret)
for _ in range(train_v_iters):
# Output from value function graph
v = actor_critic.value_function(obs)
# PPO value function objective
v_loss = F.mse_loss(v, ret)
# Value function gradient step
train_v.zero_grad()
v_loss.backward()
average_gradients(train_v.param_groups)
train_v.step()
# Log changes from update
_, logp, _, v = actor_critic(obs, act)
ratio = (logp - logp_old).exp()
min_adv = torch.where(adv > 0, (1 + clip_ratio) * adv,
(1 - clip_ratio) * adv)
pi_l_new = -(torch.min(ratio * adv, min_adv)).mean()
v_l_new = F.mse_loss(v, ret)
kl = (logp_old - logp).mean() # a sample estimate for KL-divergence
clipped = (ratio > (1 + clip_ratio)) | (ratio < (1 - clip_ratio))
cf = (clipped.float()).mean()
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
actor_critic.eval()
for t in range(local_steps_per_epoch):
a, _, logp_t, v_t = actor_critic(
torch.Tensor(o.reshape(1, -1)).to(device))
# save and log
buf.store(o,
a.cpu().detach().numpy(), r, v_t.item(),
logp_t.cpu().detach().numpy())
logger.store(VVals=v_t)
o, r, d, _ = env.step(a.cpu().detach().numpy()[0])
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else actor_critic.value_function(
torch.Tensor(o.reshape(1, -1)).to(device)).item()
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, actor_critic, None)
# Perform PPO update!
actor_critic.train()
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def gail(env_fn,traj_dir, actor_critic=core.mlp_actor_critic_add, ac_kwargs=dict(),d_hidden_size =64,d_batch_size = 64,seed=0,
steps_per_epoch=4000, epochs=50, gamma=0.99, clip_ratio=0.2, pi_lr=3e-4,
vf_lr=1e-3, train_pi_iters=40, train_v_iters=40, lam=0.97, max_ep_len=4000,beta =1e-4,
target_kl=0.01, logger_kwargs=dict(), save_freq=100,
r_env_ratio=0,gail_ratio =1, d_itr =20, reward_type = 'negative',
pretrain_bc_itr =0):
"""
additional args
d_hidden_size : hidden layer size of Discriminator
d_batch_size : Discriminator's batch size
r_env_ratio,gail_ratio : the weight of rewards from envirionment and gail .Total reward = gail_ratio *rew_gail+r_env_ratio* rew_from_environment
d_itr : The number of iteration of update discriminater
reward_type : GAIL reward has three type ['negative','positive', 'AIRL']
trj_num :the number of trajectory for
pretrain_bc_itr: the number of iteration of pretraining by behavior cloeing
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
D=Discriminator(env,hidden_size = d_hidden_size,reward_type =reward_type)
e_obs = np.loadtxt(traj_dir + '/observations.csv',delimiter=',')
e_act = np.loadtxt(traj_dir + '/actions.csv',delimiter= ',')#Demo treajectory
Sibuffer =SIBuffer(obs_dim, act_dim, e_obs,e_act,trj_num= 0, max_size =None)#!sibuf
assert e_obs.shape[1:] == obs_dim
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space, env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi,pi_std, entropy, v = actor_critic(x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
#buf_gail = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)#add buffer with TRgail rewards
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n'%var_counts)
# PPO objectives
ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s)
min_adv = tf.where(adv_ph>0, (1+clip_ratio)*adv_ph, (1-clip_ratio)*adv_ph)
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv))- beta*entropy
v_loss = tf.reduce_mean((ret_ph - v)**2)#ret_phには累積報酬のバッファが入る
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(logp_old_ph - logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(-logp) # a sample estimate for entropy, also easy to compute
clipped = tf.logical_or(ratio > (1+clip_ratio), ratio < (1-clip_ratio))
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32))
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
sess = tf.Session()
BC = BehavioralCloning(sess,pi,logp,x_ph,a_ph)
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Sync params across processes
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def update():
inputs = {k:v for k,v in zip(all_phs, buf.get())}#all_phsは各バッファーに対応するプレースホルダー辞書
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent], feed_dict=inputs)
# Training#ここも変える必要あり? おそらく変えなくて良い
for i in range(train_pi_iters):
_, kl = sess.run([train_pi, approx_kl], feed_dict=inputs)
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:#更新時のklが想定の1.5倍大きいとログをだしてtrainループを着る
logger.log('Early stopping at step %d due to reaching max kl.'%i)
break
logger.store(StopIter=i)
for _ in range(train_v_iters):#ｖの更新
sess.run(train_v, feed_dict=inputs)
# Log changes from update（新しいロスの計算）
pi_l_new, v_l_new, kl, cf = sess.run([pi_loss, v_loss, approx_kl, clipfrac], feed_dict=inputs)
std, std_ent = sess.run([pi_std,entropy],feed_dict = inputs)
logger.store(LossPi=pi_l_old, LossV=v_l_old,
KL=kl, Entropy=std_ent, ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),#更新での改善量
DeltaLossV=(v_l_new - v_l_old),
Std = std)
start_time = time.time()
o, r, d, ep_ret_task,ep_ret_gail, ep_len = env.reset(), 0, False, 0,0 , 0
if pretrain_bc_itr>0:
BC.learn(Sibuffer.expert_obs,Sibuffer.expert_act ,max_itr =pretrain_bc_itr)
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops, feed_dict={x_ph: o.reshape(1,-1)})
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
buf.store_rew(r)
'''
if t <150:
env.render()
time.sleep(0.03)
'''
ep_ret_task += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t==local_steps_per_epoch-1):
if d:# if trajectory didn't reach terminal state, bootstrap value target
last_val = r
else:
last_val = sess.run(v, feed_dict={x_ph: o.reshape(1,-1)})#v_last=...だったけどこれで良さげ
buf.store_rew(last_val)#if its terminal ,nothing change and if its maxitr last_val is use
buf.finish_path()
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret_task, EpLen=ep_len)#,EpRet_Sum =ep_ret_sum,EpRet_Gail =ep_ret_gail)
o, r, d, ep_ret_task,ep_ret_sum,ep_ret_gail, ep_len = env.reset(), 0, False, 0, 0, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs-1):
logger.save_state({'env': env}, epoch)
agent_obs , agent_act = buf.obs_buf, buf.act_buf
d_batch_size = d_batch_size#or len(agent_obs)//d_itr #update discreminator
for _t in range(d_itr):
e_obs_batch ,e_act_batch =Sibuffer.get_random_batch(d_batch_size)
a_obs_batch =sample_batch(agent_obs,batch_size = d_batch_size)
a_act_batch= sample_batch(agent_act,batch_size = d_batch_size)
D.train(sess, e_obs_batch,e_act_batch , a_obs_batch,a_act_batch )
js_d = D.get_js_div(sess,Sibuffer.main_obs_buf,Sibuffer.main_act_buf,agent_obs,agent_act)
#---------------get_gail_reward------------------------------
rew_gail=D.get_reward(sess,agent_obs, agent_act).ravel()
buf.rew_buf = gail_ratio *rew_gail+r_env_ratio*buf.rew_buf
for path_slice in buf.slicelist[:-1]:
ep_ret_gail = rew_gail[path_slice].sum()
ep_ret_sum = buf.rew_buf[path_slice].sum()
logger.store(EpRet_Sum=ep_ret_sum,EpRet_Gail=ep_ret_gail)
buf.culculate_adv_buf()
# -------------Perform PPO update!--------------------
update()
logger.store(JS=js_d)
# Log info about epoch
#if epoch%10 == 0:#logger print each 10 epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpRet_Sum', average_only=True)
logger.log_tabular('EpRet_Gail', average_only=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch+1)*steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time()-start_time)
logger.log_tabular('Std', average_only=True)
logger.log_tabular('JS', average_only=True)
#logger.log_tabular('JS_Ratio', average_only=True)
logger.dump_tabular()
def ppo(env_fn, actor_critic=core.ActorCritic, ac_kwargs=dict(), seed=0,
steps_per_epoch=4000, epochs=50, gamma=0.99, clip_ratio=0.2, pi_lr=3e-4,
vf_lr=1e-3, train_pi_iters=80, train_v_iters=80, lam=0.97, max_ep_len=1000,
target_kl=0.01, logger_kwargs=dict(), save_freq=10):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A reference to ActorCritic class which after instantiation
takes state, ``x``, and action, ``a``, and returns:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a``
| in states ``x``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
torch.manual_seed(seed)
np.random.seed(seed)
# https://pytorch.org/docs/master/notes/randomness.html#cudnn
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Actor Critic model instance
actor_critic = actor_critic(obs_dim, **ac_kwargs)
actor_critic.to(device) # load to cpu/gpu
# Count variables
var_counts = tuple(core.count_vars(model) for model in [actor_critic.policy, actor_critic.value])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n'%var_counts)
# Optimizers
train_pi = optim.Adam(actor_critic.policy.parameters(), lr=pi_lr)
train_v = optim.Adam(actor_critic.value.parameters(), lr=vf_lr)
# Sync params across processes
# sync_all_params() # TODO MPI pytorch
def update():
actor_critic.train()
obs, act, adv, ret, logp_old = map(lambda x: Tensor(x).to(device), buf.get())
_ , logp, _, val = actor_critic(obs, act)
# PPO objectives
ratio = (logp - logp_old).exp() # pi(a|s) / pi_old(a|s)
min_adv = torch.where(adv>0, (1+clip_ratio)*adv, (1-clip_ratio)*adv)
pi_l_old = -(torch.min(ratio * adv, min_adv)).mean() # pi_loss
v_l_old = ((ret - val)**2).min() #v_loss
ent = (-logp).mean() # approx_ent
# Training
for i in range(train_pi_iters):
_ , logp, _, val = actor_critic(obs, act)
ratio = (logp - logp_old).exp() # pi(a|s) / pi_old(a|s)
min_adv = torch.where(adv>0, (1+clip_ratio)*adv, (1-clip_ratio)*adv)
pi_loss = -(torch.min(ratio * adv, min_adv)).mean()
# PG Optimizer step
train_pi.zero_grad()
pi_loss.backward()
train_pi.step()
kl = (logp_old - logp).mean() # approx_kl
# kl = mpi_avg(kl) # TODO MPI Pytorch
if kl > 1.5 * target_kl:
logger.log('Early stopping at step %d due to reaching max kl.'%i)
break
logger.store(StopIter=i)
# Value function learning
for _ in range(train_v_iters):
val = actor_critic.value(obs)
v_loss = (ret - val).pow(2).mean()
train_v.zero_grad()
v_loss.backward()
train_v.step()
actor_critic.eval()
# Log changes from update
_ , logp, _, val = actor_critic(obs, act)
ratio = (logp - logp_old).exp() # pi(a|s) / pi_old(a|s)
min_adv = torch.where(adv>0, (1+clip_ratio)*adv, (1-clip_ratio)*adv)
pi_l_new = -(torch.min(ratio * adv, min_adv)).mean() # pi_loss
v_l_new = ((ret - val)**2).min() #v_loss
# Info (useful to watch during learning)
# a sample estimate for KL-divergence, easy to compute
kl = (logp_old - logp).mean() # approx_kl
clipped = (ratio > (1+clip_ratio)) | (ratio < (1-clip_ratio))
cf = clipped.float().mean() # clipfrac
logger.store(LossPi=pi_l_old, LossV=v_l_old,
KL=kl, Entropy=ent, ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, _, logp_pi_t, v_t = actor_critic(Tensor(o.reshape(1,-1)).to(device))
# save and log
buf.store(o, a.cpu().numpy(), r, v_t.item(), logp_pi_t.cpu().detach().numpy())
logger.store(VVals=v_t)
o, r, d, _ = env.step(a.cpu().numpy())
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t==local_steps_per_epoch-1):
if not(terminal):
print('Warning: trajectory cut off by epoch at %d steps.'%ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else actor_critic(Tensor(o.reshape(1,-1)).to(device))[-1].item()
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs-1):
logger.save_state({'env': env}, actor_critic, None)
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch+1)*steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time()-start_time)
logger.dump_tabular()
def ppo(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
clip_ratio=0.2,
pi_lr=3e-4,
vf_lr=1e-3,
train_pi_iters=80,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
target_kl=0.01,
logger_kwargs=dict(),
save_freq=10):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to PPO.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
clip_ratio (float): Hyperparameter for clipping in the policy objective.
Roughly: how far can the new policy go from the old policy while
still profiting (improving the objective function)? The new policy
can still go farther than the clip_ratio says, but it doesn't help
on the objective anymore. (Usually small, 0.1 to 0.3.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_pi_iters (int): Maximum number of gradient descent steps to take
on policy loss per epoch. (Early stopping may cause optimizer
to take fewer than this.)
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
target_kl (float): Roughly what KL divergence we think is appropriate
between new and old policies after an update. This will get used
for early stopping. (Usually small, 0.01 or 0.05.)
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# PPO objectives
ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s)
min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph,
(1 - clip_ratio) * adv_ph)
pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv))
v_loss = tf.reduce_mean((ret_ph - v)**2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-logp) # a sample estimate for entropy, also easy to compute
clipped = tf.logical_or(ratio > (1 + clip_ratio), ratio < (1 - clip_ratio))
clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32))
# Optimizers
train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def update():
inputs = {k: v for k, v in zip(all_phs, buf.get())}
pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent],
feed_dict=inputs)
# Training
for i in range(train_pi_iters):
_, kl = sess.run([train_pi, approx_kl], feed_dict=inputs)
kl = mpi_avg(kl)
if kl > 1.5 * target_kl:
logger.log(
'Early stopping at step %d due to reaching max kl.' % i)
break
logger.store(StopIter=i)
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
pi_l_new, v_l_new, kl, cf = sess.run(
[pi_loss, v_loss, approx_kl, clipfrac], feed_dict=inputs)
logger.store(LossPi=pi_l_old,
LossV=v_l_old,
KL=kl,
Entropy=ent,
ClipFrac=cf,
DeltaLossPi=(pi_l_new - pi_l_old),
DeltaLossV=(v_l_new - v_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops,
feed_dict={x_ph: o.reshape(1, -1)})
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(
v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform PPO update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', with_min_and_max=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('VVals', with_min_and_max=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('LossV', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('DeltaLossV', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('KL', average_only=True)
logger.log_tabular('ClipFrac', average_only=True)
logger.log_tabular('StopIter', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
def pg_linesearch(env_fn,
actor_critic=core.mlp_actor_critic,
ac_kwargs=dict(),
seed=0,
steps_per_epoch=4000,
epochs=50,
gamma=0.99,
pi_lr=3e-4,
backtrack_coeff=0.8,
delta=0.01,
backtrack_iters=1000,
vf_lr=1e-3,
train_v_iters=80,
lam=0.97,
max_ep_len=1000,
logger_kwargs=dict(),
save_freq=10):
"""
Args:
env_fn : A function which creates a copy of the environment.
The environment must satisfy the OpenAI Gym API.
actor_critic: A function which takes in placeholder symbols
for state, ``x_ph``, and action, ``a_ph``, and returns the main
outputs from the agent's Tensorflow computation graph:
=========== ================ ======================================
Symbol Shape Description
=========== ================ ======================================
``pi`` (batch, act_dim) | Samples actions from policy given
| states.
``logp`` (batch,) | Gives log probability, according to
| the policy, of taking actions ``a_ph``
| in states ``x_ph``.
``logp_pi`` (batch,) | Gives log probability, according to
| the policy, of the action sampled by
| ``pi``.
``v`` (batch,) | Gives the value estimate for states
| in ``x_ph``. (Critical: make sure
| to flatten this!)
=========== ================ ======================================
ac_kwargs (dict): Any kwargs appropriate for the actor_critic
function you provided to VPG.
seed (int): Seed for random number generators.
steps_per_epoch (int): Number of steps of interaction (state-action pairs)
for the agent and the environment in each epoch.
epochs (int): Number of epochs of interaction (equivalent to
number of policy updates) to perform.
gamma (float): Discount factor. (Always between 0 and 1.)
pi_lr (float): Learning rate for policy optimizer.
vf_lr (float): Learning rate for value function optimizer.
train_v_iters (int): Number of gradient descent steps to take on
value function per epoch.
lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
close to 1.)
max_ep_len (int): Maximum length of trajectory / episode / rollout.
logger_kwargs (dict): Keyword args for EpochLogger.
save_freq (int): How often (in terms of gap between epochs) to save
the current policy and value function.
"""
logger = EpochLogger(**logger_kwargs)
logger.save_config(locals())
seed += 10000 * proc_id()
tf.set_random_seed(seed)
np.random.seed(seed)
env = env_fn()
obs_dim = env.observation_space.shape
act_dim = env.action_space.shape
# Share information about action space with policy architecture
ac_kwargs['action_space'] = env.action_space
# Inputs to computation graph
x_ph, a_ph = core.placeholders_from_spaces(env.observation_space,
env.action_space)
adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
# Main outputs from computation graph
pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
# Need all placeholders in *this* order later (to zip with data from buffer)
all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
# Every step, get: action, value, and logprob
get_action_ops = [pi, v, logp_pi]
# Experience buffer
local_steps_per_epoch = int(steps_per_epoch / num_procs())
buf = PGBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)
# Count variables
var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v'])
logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts)
# VPG objectives
pi_loss = -tf.reduce_mean(logp * adv_ph)
v_loss = tf.reduce_mean((ret_ph - v)**2)
# Info (useful to watch during learning)
approx_kl = tf.reduce_mean(
logp_old_ph -
logp) # a sample estimate for KL-divergence, easy to compute
approx_ent = tf.reduce_mean(
-logp) # a sample estimate for entropy, also easy to compute
# Optimizers
#train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
# Symbols needed for CG solver
pi_params = trpo_core.get_vars('pi')
gradient = trpo_core.flat_grad(pi_loss, pi_params)
#v_ph, hvp = trpo_core.hessian_vector_product(d_kl, pi_params)
v_ph = tf.placeholder(tf.float32, shape=gradient.shape)
##TODO: more analysis on damping Coeff
#if damping_coeff > 0:
#hvp += damping_coeff * v_ph
# Symbols for getting and setting params
get_pi_params = trpo_core.flat_concat(pi_params)
set_pi_params = trpo_core.assign_params_from_flat(v_ph, pi_params)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Sync params across processes
sess.run(sync_all_params())
# Setup model saving
logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v})
def cg(Ax, b):
"""
Conjugate gradient algorithm
(see https://en.wikipedia.org/wiki/Conjugate_gradient_method)
"""
##TODO: Next Step is to try the hessian
x = np.zeros_like(b)
r = b.copy(
) # Note: should be 'b - Ax(x)', but for x=0, Ax(x)=0. Change if doing warm start.
p = r.copy()
r_dot_old = np.dot(r, r)
cg_iters = 20
for _ in range(cg_iters):
z = Ax(p)
alpha = r_dot_old / (np.dot(p, z) + EPS)
x += alpha * p
r -= alpha * z
r_dot_new = np.dot(r, r)
p = r + (r_dot_new / r_dot_old) * p
r_dot_old = r_dot_new
return x
def update():
inputs = {k: v for k, v in zip(all_phs, buf.get())}
#TODO: Next step is to calculate the hessian using safe distance
#Hx = lambda x : mpi_avg(sess.run(hvp, feed_dict={**inputs, v_ph: x}))
g, pi_l_old, v_l_old, ent = sess.run(
[gradient, pi_loss, v_loss, approx_ent], feed_dict=inputs)
g, pi_l_old = mpi_avg(g), mpi_avg(pi_l_old)
#x = cg(Hx, g)
#x = optimize.fmin_cg(pi_l_old, x0, fprime=g)
x = g
old_params = sess.run(get_pi_params)
old_penalty = env.penalty(env.s)
alpha = np.sqrt(2 * delta / (np.dot(x, g) + EPS))
# backtracking line search, hard constraint check on env penalty
for j in range(backtrack_iters):
step = backtrack_coeff**j
sess.run(set_pi_params,
feed_dict={v_ph: old_params - alpha * x * step})
pi_l_new = sess.run([pi_loss], feed_dict=inputs)
penalty = env.penalty(env.s)
#print("Old Penalty {}, Penalty {}".format(old_penalty,penalty))
if penalty == 0 or penalty < old_penalty:
#if pi_l_new <= pi_l_old:
logger.log('Accepting new params at step %d of line search.' %
j)
logger.store(BacktrackIters=j)
logger.store(penalty=penalty, old_penalty=old_penalty)
break
if j == backtrack_iters - 1:
logger.log('Line search failed! Keeping old params.')
logger.store(BacktrackIters=j)
logger.store(penalty=penalty, old_penalty=old_penalty)
# Policy gradient step
#sess.run(train_pi, feed_dict=inputs)
# Value function learning
for _ in range(train_v_iters):
sess.run(train_v, feed_dict=inputs)
# Log changes from update
#pi_l_new, v_l_new, kl = sess.run([pi_loss, v_loss, approx_kl], feed_dict={v_ph: old_params - alpha * x * step})
logger.store(LossPi=pi_l_old,
Entropy=ent,
DeltaLossPi=(pi_l_new - pi_l_old))
start_time = time.time()
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Main loop: collect experience in env and update/log each epoch
for epoch in range(epochs):
for t in range(local_steps_per_epoch):
a, v_t, logp_t = sess.run(get_action_ops,
feed_dict={x_ph: o.reshape(1, -1)})
# save and log
buf.store(o, a, r, v_t, logp_t)
logger.store(VVals=v_t)
o, r, d, _ = env.step(a[0])
ep_ret += r
ep_len += 1
terminal = d or (ep_len == max_ep_len)
if terminal or (t == local_steps_per_epoch - 1):
if not (terminal):
print('Warning: trajectory cut off by epoch at %d steps.' %
ep_len)
# if trajectory didn't reach terminal state, bootstrap value target
last_val = r if d else sess.run(
v, feed_dict={x_ph: o.reshape(1, -1)})
buf.finish_path(last_val)
if terminal:
# only save EpRet / EpLen if trajectory finished
logger.store(EpRet=ep_ret, EpLen=ep_len)
o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0
# Save model
if (epoch % save_freq == 0) or (epoch == epochs - 1):
logger.save_state({'env': env}, None)
# Perform PG update!
update()
# Log info about epoch
logger.log_tabular('Epoch', epoch)
logger.log_tabular('EpRet', average_only=True)
logger.log_tabular('penalty', average_only=True)
logger.log_tabular('old_penalty', average_only=True)
logger.log_tabular('EpLen', average_only=True)
logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch)
logger.log_tabular('LossPi', average_only=True)
logger.log_tabular('DeltaLossPi', average_only=True)
logger.log_tabular('Entropy', average_only=True)
logger.log_tabular('Time', time.time() - start_time)
logger.dump_tabular()
