def _test_abc_batch(self,
steps=100000,
require_success=True,
gpu=-1,
load_model=False,
num_envs=4):
if self.recurrent and gpu >= 0:
self.skipTest(
'NStepLSTM does not support double backprop with GPU.')
if self.recurrent and chainer.__version__ == '7.0.0b3':
self.skipTest(
'chainer==7.0.0b3 has a bug in double backrop of LSTM.'
' See https://github.com/chainer/chainer/pull/8037')
env, _ = self.make_vec_env_and_successful_return(test=False,
num_envs=num_envs)
test_env, successful_return = self.make_vec_env_and_successful_return(
test=True, num_envs=num_envs)
agent = self.make_agent(env, gpu)
max_episode_len = None if self.episodic else 2
if load_model:
print('Load agent from', self.agent_dirname)
agent.load(self.agent_dirname)
# Train
train_agent_batch_with_evaluation(
agent=agent,
env=env,
steps=steps,
outdir=self.tmpdir,
eval_interval=200,
eval_n_steps=None,
eval_n_episodes=40,
successful_score=successful_return,
eval_env=test_env,
log_interval=100,
max_episode_len=max_episode_len,
)
env.close()
# Test
n_test_runs = 10
eval_returns = batch_run_evaluation_episodes(
test_env,
agent,
n_steps=None,
n_episodes=n_test_runs,
max_episode_len=max_episode_len,
)
test_env.close()
if require_success:
n_succeeded = np.sum(np.asarray(eval_returns) >= successful_return)
self.assertEqual(n_succeeded, n_test_runs)
# Save
agent.save(self.agent_dirname)
def _test_abc_batch(self,
steps=100000,
require_success=True,
gpu=-1,
load_model=False,
num_envs=4):
env, _ = self.make_vec_env_and_successful_return(test=False,
num_envs=num_envs)
test_env, successful_return = self.make_vec_env_and_successful_return(
test=True, num_envs=num_envs)
agent = self.make_agent(env, gpu)
max_episode_len = None if self.episodic else 2
if load_model:
print('Load agent from', self.agent_dirname)
agent.load(self.agent_dirname)
# Train
train_agent_batch_with_evaluation(
agent=agent,
env=env,
steps=steps,
outdir=self.tmpdir,
eval_interval=200,
eval_n_steps=None,
eval_n_episodes=40,
successful_score=successful_return,
eval_env=test_env,
log_interval=100,
max_episode_len=max_episode_len,
)
env.close()
# Test
n_test_runs = 10
eval_returns = batch_run_evaluation_episodes(
test_env,
agent,
n_steps=None,
n_episodes=n_test_runs,
max_episode_len=max_episode_len,
)
test_env.close()
if require_success:
n_succeeded = np.sum(np.asarray(eval_returns) >= successful_return)
self.assertEqual(n_succeeded, n_test_runs)
# Save
agent.save(self.agent_dirname)
def _test_batch_training(self,
gpu,
steps=5000,
load_model=False,
require_success=True):
random_seed.set_random_seed(1)
logging.basicConfig(level=logging.DEBUG)
env, _ = self.make_vec_env_and_successful_return(test=False)
test_env, successful_return = self.make_vec_env_and_successful_return(
test=True)
agent = self.make_agent(env, gpu)
if load_model:
print('Load agent from', self.agent_dirname)
agent.load(self.agent_dirname)
agent.replay_buffer.load(self.rbuf_filename)
# Train
train_agent_batch_with_evaluation(
agent=agent,
env=env,
steps=steps,
outdir=self.tmpdir,
eval_interval=200,
eval_n_steps=None,
eval_n_episodes=5,
successful_score=1,
eval_env=test_env,
)
env.close()
# Test
n_test_runs = 5
eval_returns = batch_run_evaluation_episodes(
test_env,
agent,
n_steps=None,
n_episodes=n_test_runs,
)
test_env.close()
n_succeeded = np.sum(np.asarray(eval_returns) >= successful_return)
if require_success:
self.assertEqual(n_succeeded, n_test_runs)
# Save
agent.save(self.agent_dirname)
agent.replay_buffer.save(self.rbuf_filename)
def main():
import logging
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=0,
help='GPU to use, set to -1 if no GPU.')
parser.add_argument('--env', type=str, default='Hopper-v2',
help='OpenAI Gym MuJoCo env to perform algorithm on.')
parser.add_argument('--num-envs', type=int, default=1,
help='Number of envs run in parallel.')
parser.add_argument('--seed', type=int, default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--outdir', type=str, default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--steps', type=int, default=2 * 10 ** 6,
help='Total number of timesteps to train the agent.')
parser.add_argument('--eval-interval', type=int, default=100000,
help='Interval in timesteps between evaluations.')
parser.add_argument('--eval-n-runs', type=int, default=100,
help='Number of episodes run for each evaluation.')
parser.add_argument('--render', action='store_true',
help='Render env states in a GUI window.')
parser.add_argument('--demo', action='store_true',
help='Just run evaluation, not training.')
parser.add_argument('--load', type=str, default='',
help='Directory to load agent from.')
parser.add_argument('--logger-level', type=int, default=logging.INFO,
help='Level of the root logger.')
parser.add_argument('--monitor', action='store_true',
help='Wrap env with gym.wrappers.Monitor.')
parser.add_argument('--log-interval', type=int, default=1000,
help='Interval in timesteps between outputting log'
' messages during training')
parser.add_argument('--update-interval', type=int, default=2048,
help='Interval in timesteps between model updates.')
parser.add_argument('--epochs', type=int, default=10,
help='Number of epochs to update model for per PPO'
' iteration.')
parser.add_argument('--batch-size', type=int, default=64,
help='Minibatch size')
args = parser.parse_args()
logging.basicConfig(level=args.logger_level)
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu,))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2 ** 32
args.outdir = experiments.prepare_output_dir(args, args.outdir)
def make_env(process_idx, test):
env = gym.make(args.env)
# Use different random seeds for train and test envs
process_seed = int(process_seeds[process_idx])
env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
env.seed(env_seed)
# Cast observations to float32 because our model uses float32
env = chainerrl.wrappers.CastObservationToFloat32(env)
if args.monitor:
env = gym.wrappers.Monitor(env, args.outdir)
if args.render:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv(
[functools.partial(make_env, idx, test)
for idx, env in enumerate(range(args.num_envs))])
# Only for getting timesteps, and obs-action spaces
sample_env = gym.make(args.env)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
obs_space = sample_env.observation_space
action_space = sample_env.action_space
print('Observation space:', obs_space)
print('Action space:', action_space)
assert isinstance(action_space, gym.spaces.Box)
# Normalize observations based on their empirical mean and variance
obs_normalizer = chainerrl.links.EmpiricalNormalization(
obs_space.low.size, clip_threshold=5)
# While the original paper initialized weights by normal distribution,
# we use orthogonal initialization as the latest openai/baselines does.
winit = chainerrl.initializers.Orthogonal(1.)
winit_last = chainerrl.initializers.Orthogonal(1e-2)
action_size = action_space.low.size
policy = chainer.Sequential(
L.Linear(None, 64, initialW=winit),
F.tanh,
L.Linear(None, 64, initialW=winit),
F.tanh,
L.Linear(None, action_size, initialW=winit_last),
chainerrl.policies.GaussianHeadWithStateIndependentCovariance(
action_size=action_size,
var_type='diagonal',
var_func=lambda x: F.exp(2 * x), # Parameterize log std
var_param_init=0, # log std = 0 => std = 1
),
)
vf = chainer.Sequential(
L.Linear(None, 64, initialW=winit),
F.tanh,
L.Linear(None, 64, initialW=winit),
F.tanh,
L.Linear(None, 1, initialW=winit),
)
# Combine a policy and a value function into a single model
model = chainerrl.links.Branched(policy, vf)
opt = chainer.optimizers.Adam(3e-4, eps=1e-5)
opt.setup(model)
agent = PPO(
model,
opt,
obs_normalizer=obs_normalizer,
gpu=args.gpu,
update_interval=args.update_interval,
minibatch_size=args.batch_size,
epochs=args.epochs,
clip_eps_vf=None,
entropy_coef=0,
standardize_advantages=True,
gamma=0.995,
lambd=0.97,
)
if args.load:
agent.load(args.load)
if args.demo:
env = make_batch_env(True)
eval_stats = experiments.eval_performance(
env=env,
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(False),
eval_env=make_batch_env(True),
outdir=args.outdir,
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
log_interval=args.log_interval,
max_episode_len=timestep_limit,
save_best_so_far_agent=False,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--outdir', type=str, default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--env', type=str,
default='RoboschoolAtlasForwardWalk-v1',
help='OpenAI Gym env to perform algorithm on.')
parser.add_argument('--num-envs', type=int, default=4,
help='Number of envs run in parallel.')
parser.add_argument('--seed', type=int, default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--gpu', type=int, default=0,
help='GPU to use, set to -1 if no GPU.')
parser.add_argument('--load', type=str, default='',
help='Directory to load agent from.')
parser.add_argument('--steps', type=int, default=10 ** 7,
help='Total number of timesteps to train the agent.')
parser.add_argument('--eval-n-runs', type=int, default=20,
help='Number of episodes run for each evaluation.')
parser.add_argument('--eval-interval', type=int, default=100000,
help='Interval in timesteps between evaluations.')
parser.add_argument('--replay-start-size', type=int, default=10000,
help='Minimum replay buffer size before ' +
'performing gradient updates.')
parser.add_argument('--update-interval', type=int, default=1,
help='Interval in timesteps between model updates.')
parser.add_argument('--batch-size', type=int, default=256,
help='Minibatch size')
parser.add_argument('--render', action='store_true',
help='Render env states in a GUI window.')
parser.add_argument('--demo', action='store_true',
help='Just run evaluation, not training.')
parser.add_argument('--monitor', action='store_true',
help='Wrap env with Monitor to write videos.')
parser.add_argument('--log-interval', type=int, default=1000,
help='Interval in timesteps between outputting log'
' messages during training')
parser.add_argument('--logger-level', type=int, default=logging.INFO,
help='Level of the root logger.')
parser.add_argument('--n-hidden-channels', type=int, default=1024,
help='Number of hidden channels of NN models.')
parser.add_argument('--discount', type=float, default=0.98,
help='Discount factor.')
parser.add_argument('--n-step-return', type=int, default=3,
help='N-step return.')
parser.add_argument('--lr', type=float, default=3e-4,
help='Learning rate.')
parser.add_argument('--adam-eps', type=float, default=1e-1,
help='Adam eps.')
args = parser.parse_args()
logging.basicConfig(level=args.logger_level)
args.outdir = experiments.prepare_output_dir(
args, args.outdir, argv=sys.argv)
print('Output files are saved in {}'.format(args.outdir))
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu,))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2 ** 32
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv(
[functools.partial(make_env, args, process_seeds[idx], test)
for idx, env in enumerate(range(args.num_envs))])
sample_env = make_env(args, process_seeds[0], test=False)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
obs_space = sample_env.observation_space
action_space = sample_env.action_space
print('Observation space:', obs_space)
print('Action space:', action_space)
del sample_env
action_size = action_space.low.size
winit = chainer.initializers.GlorotUniform()
winit_policy_output = chainer.initializers.GlorotUniform()
def squashed_diagonal_gaussian_head(x):
assert x.shape[-1] == action_size * 2
mean, log_scale = F.split_axis(x, 2, axis=1)
log_scale = F.clip(log_scale, -20., 2.)
var = F.exp(log_scale * 2)
return chainerrl.distribution.SquashedGaussianDistribution(
mean, var=var)
policy = chainer.Sequential(
L.Linear(None, args.n_hidden_channels, initialW=winit),
F.relu,
L.Linear(None, args.n_hidden_channels, initialW=winit),
F.relu,
L.Linear(None, action_size * 2, initialW=winit_policy_output),
squashed_diagonal_gaussian_head,
)
policy_optimizer = optimizers.Adam(
args.lr, eps=args.adam_eps).setup(policy)
def make_q_func_with_optimizer():
q_func = chainer.Sequential(
concat_obs_and_action,
L.Linear(None, args.n_hidden_channels, initialW=winit),
F.relu,
L.Linear(None, args.n_hidden_channels, initialW=winit),
F.relu,
L.Linear(None, 1, initialW=winit),
)
q_func_optimizer = optimizers.Adam(
args.lr, eps=args.adam_eps).setup(q_func)
return q_func, q_func_optimizer
q_func1, q_func1_optimizer = make_q_func_with_optimizer()
q_func2, q_func2_optimizer = make_q_func_with_optimizer()
# Draw the computational graph and save it in the output directory.
fake_obs = chainer.Variable(
policy.xp.zeros_like(obs_space.low, dtype=np.float32)[None],
name='observation')
fake_action = chainer.Variable(
policy.xp.zeros_like(action_space.low, dtype=np.float32)[None],
name='action')
chainerrl.misc.draw_computational_graph(
[policy(fake_obs)], os.path.join(args.outdir, 'policy'))
chainerrl.misc.draw_computational_graph(
[q_func1(fake_obs, fake_action)], os.path.join(args.outdir, 'q_func1'))
chainerrl.misc.draw_computational_graph(
[q_func2(fake_obs, fake_action)], os.path.join(args.outdir, 'q_func2'))
rbuf = replay_buffer.ReplayBuffer(10 ** 6, num_steps=args.n_step_return)
def burnin_action_func():
"""Select random actions until model is updated one or more times."""
return np.random.uniform(
action_space.low, action_space.high).astype(np.float32)
# Hyperparameters in http://arxiv.org/abs/1802.09477
agent = chainerrl.agents.SoftActorCritic(
policy,
q_func1,
q_func2,
policy_optimizer,
q_func1_optimizer,
q_func2_optimizer,
rbuf,
gamma=args.discount,
update_interval=args.update_interval,
replay_start_size=args.replay_start_size,
gpu=args.gpu,
minibatch_size=args.batch_size,
burnin_action_func=burnin_action_func,
entropy_target=-action_size,
temperature_optimizer=chainer.optimizers.Adam(
args.lr, eps=args.adam_eps),
)
if len(args.load) > 0:
agent.load(args.load)
if args.demo:
eval_env = make_env(args, seed=0, test=True)
eval_stats = experiments.eval_performance(
env=eval_env,
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit,
)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(test=False),
eval_env=make_batch_env(test=True),
outdir=args.outdir,
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
log_interval=args.log_interval,
max_episode_len=timestep_limit,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--outdir', type=str, default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--seed', type=int, default=0,
help='Random seed [0, 2 ** 31)')
parser.add_argument('--gpu', type=int, default=0,
help='GPU to use, set to -1 if no GPU.')
parser.add_argument('--demo', action='store_true', default=False,
help='Evaluate the agent without training.')
parser.add_argument('--load', type=str, default=None,
help='Load a saved agent from a given directory.')
parser.add_argument('--final-exploration-steps',
type=int, default=5 * 10 ** 5,
help='Timesteps after which we stop'
' annealing exploration rate')
parser.add_argument('--final-epsilon', type=float, default=0.2,
help='Final value of epsilon during training.')
parser.add_argument('--steps', type=int, default=2 * 10 ** 6,
help='Total number of timesteps to train the agent.')
parser.add_argument('--replay-start-size', type=int, default=5 * 10 ** 4,
help='Minimum replay buffer size before'
' performing gradient updates.')
parser.add_argument('--target-update-interval',
type=int, default=1 * 10 ** 4,
help='Frequency (in timesteps) at which'
' the target network is updated.')
parser.add_argument('--eval-interval', type=int, default=10 ** 5,
help='Frequency (in timesteps) of evaluation phase.')
parser.add_argument('--update-interval', type=int, default=1,
help='Frequency (in timesteps) of network updates.')
parser.add_argument('--eval-n-runs', type=int, default=100,
help='Number of episodes used for evaluation.')
parser.add_argument('--logging-level', type=int, default=20,
help='Logging level. 10:DEBUG, 20:INFO etc.')
parser.add_argument('--render', action='store_true', default=False,
help='Render env states in a GUI window.')
parser.add_argument('--lr', type=float, default=6.25e-5,
help='Learning rate')
parser.add_argument('--num-envs', type=int, default=1,
help='Number of envs run in parallel.')
parser.add_argument('--batch-size', type=int, default=32,
help='Batch size used for training.')
parser.add_argument('--record', action='store_true', default=False,
help='Record videos of evaluation envs.'
' --render should also be specified.')
parser.add_argument('--gamma', type=float, default=0.99,
help='Discount factor.')
args = parser.parse_args()
import logging
logging.basicConfig(level=args.logging_level)
# Set a random seed used in ChainerRL.
misc.set_random_seed(args.seed, gpus=(args.gpu,))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2 ** 32
args.outdir = experiments.prepare_output_dir(args, args.outdir)
print('Output files are saved in {}'.format(args.outdir))
max_episode_steps = 8
def make_env(idx, test):
from pybullet_envs.bullet.kuka_diverse_object_gym_env import KukaDiverseObjectEnv # NOQA
# Use different random seeds for train and test envs
process_seed = int(process_seeds[idx])
env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
# Set a random seed for this subprocess
misc.set_random_seed(env_seed)
env = KukaDiverseObjectEnv(
isDiscrete=True,
renders=args.render and (args.demo or not test),
height=84,
width=84,
maxSteps=max_episode_steps,
isTest=test,
)
# (84, 84, 3) -> (3, 84, 84)
env = TransposeObservation(env, (2, 0, 1))
env = ObserveElapsedSteps(env, max_episode_steps)
# KukaDiverseObjectEnv internally asserts int actions and does not
# accept python-future's newint.
env = CastAction(env, __builtins__.int)
env.seed(int(env_seed))
if test and args.record:
assert args.render,\
'To use --record, --render needs be specified.'
video_dir = os.path.join(args.outdir, 'video_{}'.format(idx))
os.mkdir(video_dir)
env = RecordMovie(env, video_dir)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv(
[functools.partial(make_env, idx, test)
for idx in range(args.num_envs)])
eval_env = make_batch_env(test=True)
n_actions = eval_env.action_space.n
q_func = GraspingQFunction(n_actions, max_episode_steps)
# Draw the computational graph and save it in the output directory.
fake_obs = (
np.zeros((3, 84, 84), dtype=np.float32)[None],
np.zeros((), dtype=np.int32)[None],
)
chainerrl.misc.draw_computational_graph(
[q_func(fake_obs)],
os.path.join(args.outdir, 'model'))
# Use the hyper parameters of the Nature paper
opt = optimizers.RMSpropGraves(
lr=args.lr, alpha=0.95, momentum=0.0, eps=1e-2)
opt.setup(q_func)
# Anneal beta from beta0 to 1 throughout training
betasteps = args.steps / args.update_interval
rbuf = replay_buffer.PrioritizedReplayBuffer(
10 ** 6, alpha=0.6, beta0=0.4, betasteps=betasteps)
explorer = explorers.LinearDecayEpsilonGreedy(
1.0, args.final_epsilon,
args.final_exploration_steps,
lambda: np.random.randint(n_actions))
def phi(x):
# Feature extractor
image, elapsed_steps = x
# Normalize RGB values: [0, 255] -> [0, 1]
norm_image = np.asarray(image, dtype=np.float32) / 255
return norm_image, elapsed_steps
agent = chainerrl.agents.DoubleDQN(
q_func,
opt,
rbuf,
gpu=args.gpu,
gamma=args.gamma,
explorer=explorer,
minibatch_size=args.batch_size,
replay_start_size=args.replay_start_size,
target_update_interval=args.target_update_interval,
update_interval=args.update_interval,
batch_accumulator='sum',
phi=phi,
)
if args.load:
agent.load(args.load)
if args.demo:
eval_stats = experiments.eval_performance(
env=eval_env,
agent=agent,
n_runs=args.eval_n_runs)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(test=False),
eval_env=eval_env,
steps=args.steps,
eval_n_runs=args.eval_n_runs,
eval_interval=args.eval_interval,
outdir=args.outdir,
save_best_so_far_agent=False,
log_interval=1000,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--env', type=str, default='BreakoutNoFrameskip-v4')
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 31)')
parser.add_argument('--outdir', type=str, default='results')
parser.add_argument(
'--max-frames',
type=int,
default=30 * 60 * 60, # 30 minutes with 60 fps
help='Maximum number of frames for each episode.')
parser.add_argument('--steps', type=int, default=8 * 10**7)
parser.add_argument('--update-steps', type=int, default=5)
parser.add_argument('--lr', type=float, default=7e-4)
parser.add_argument('--gamma',
type=float,
default=0.99,
help='discount factor')
parser.add_argument('--rmsprop-epsilon', type=float, default=1e-5)
parser.add_argument('--use-gae',
action='store_true',
default=False,
help='use generalized advantage estimation')
parser.add_argument('--tau',
type=float,
default=0.95,
help='gae parameter')
parser.add_argument('--alpha',
type=float,
default=0.99,
help='RMSprop optimizer alpha')
parser.add_argument('--eval-interval', type=int, default=10**6)
parser.add_argument('--eval-n-runs', type=int, default=10)
parser.add_argument('--weight-decay', type=float, default=0.0)
parser.add_argument('--demo', action='store_true', default=False)
parser.add_argument('--load', type=str, default='')
parser.add_argument('--max-grad-norm',
type=float,
default=40,
help='value loss coefficient')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--num-envs', type=int, default=1)
parser.add_argument('--logging-level',
type=int,
default=20,
help='Logging level. 10:DEBUG, 20:INFO etc.')
parser.add_argument('--monitor',
action='store_true',
default=False,
help='Monitor env. Videos and additional information'
' are saved as output files.')
parser.add_argument('--render',
action='store_true',
default=False,
help='Render env states in a GUI window.')
parser.set_defaults(use_lstm=False)
args = parser.parse_args()
logging.basicConfig(level=args.logging_level)
# Set a random seed used in ChainerRL.
# If you use more than one processes, the results will be no longer
# deterministic even with the same random seed.
misc.set_random_seed(args.seed)
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2**31
args.outdir = experiments.prepare_output_dir(args, args.outdir)
print('Output files are saved in {}'.format(args.outdir))
def make_env(process_idx, test):
# Use different random seeds for train and test envs
process_seed = process_seeds[process_idx]
env_seed = 2**31 - 1 - process_seed if test else process_seed
env = atari_wrappers.wrap_deepmind(atari_wrappers.make_atari(
args.env, max_frames=args.max_frames),
episode_life=not test,
clip_rewards=not test)
env.seed(int(env_seed))
if args.monitor:
env = gym.wrappers.Monitor(
env, args.outdir, mode='evaluation' if test else 'training')
if args.render:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv([
functools.partial(make_env, idx, test)
for idx, env in enumerate(range(args.num_envs))
])
sample_env = make_env(0, test=False)
n_actions = sample_env.action_space.n
model = A2CFF(n_actions)
optimizer = rmsprop_async.RMSpropAsync(lr=args.lr,
eps=args.rmsprop_epsilon,
alpha=args.alpha)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.GradientClipping(args.max_grad_norm))
if args.weight_decay > 0:
optimizer.add_hook(NonbiasWeightDecay(args.weight_decay))
agent = a2c.A2C(
model,
optimizer,
gamma=args.gamma,
gpu=args.gpu,
num_processes=args.num_envs,
update_steps=args.update_steps,
phi=phi,
use_gae=args.use_gae,
tau=args.tau,
)
if args.load:
agent.load(args.load)
if args.demo:
eval_stats = experiments.eval_performance(
env=make_batch_env(test=True),
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs)
print('n_runs: {} mean: {} median: {} stdev: {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(test=False),
eval_env=make_batch_env(test=True),
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
outdir=args.outdir,
save_best_so_far_agent=False,
log_interval=1000,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--env',
type=str,
default='BreakoutNoFrameskip-v4',
help='Gym Env ID.')
parser.add_argument('--gpu',
type=int,
default=0,
help='GPU device ID. Set to -1 to use CPUs only.')
parser.add_argument('--num-envs',
type=int,
default=8,
help='Number of env instances run in parallel.')
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--outdir',
type=str,
default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--steps',
type=int,
default=10**7,
help='Total time steps for training.')
parser.add_argument(
'--max-frames',
type=int,
default=30 * 60 * 60, # 30 minutes with 60 fps
help='Maximum number of frames for each episode.')
parser.add_argument('--lr',
type=float,
default=2.5e-4,
help='Learning rate.')
parser.add_argument('--eval-interval',
type=int,
default=100000,
help='Interval (in timesteps) between evaluation'
' phases.')
parser.add_argument('--eval-n-runs',
type=int,
default=10,
help='Number of episodes ran in an evaluation phase.')
parser.add_argument('--demo',
action='store_true',
default=False,
help='Run demo episodes, not training.')
parser.add_argument('--load',
type=str,
default='',
help='Directory path to load a saved agent data from'
' if it is a non-empty string.')
parser.add_argument('--logging-level',
type=int,
default=20,
help='Logging level. 10:DEBUG, 20:INFO etc.')
parser.add_argument('--render',
action='store_true',
default=False,
help='Render env states in a GUI window.')
parser.add_argument('--monitor',
action='store_true',
default=False,
help='Monitor env. Videos and additional information'
' are saved as output files.')
parser.add_argument('--update-interval',
type=int,
default=128 * 8,
help='Interval (in timesteps) between PPO iterations.')
parser.add_argument('--batchsize',
type=int,
default=32 * 8,
help='Size of minibatch (in timesteps).')
parser.add_argument('--epochs',
type=int,
default=4,
help='Number of epochs used for each PPO iteration.')
parser.add_argument('--log-interval',
type=int,
default=10000,
help='Interval (in timesteps) of printing logs.')
parser.add_argument('--recurrent',
action='store_true',
default=False,
help='Use a recurrent model. See the code for the'
' model definition.')
parser.add_argument('--flicker',
action='store_true',
default=False,
help='Use so-called flickering Atari, where each'
' screen is blacked out with probability 0.5.')
parser.add_argument('--no-frame-stack',
action='store_true',
default=False,
help='Disable frame stacking so that the agent can'
' only see the current screen.')
parser.add_argument('--checkpoint-frequency',
type=int,
default=None,
help='Frequency at which agents are stored.')
args = parser.parse_args()
import logging
logging.basicConfig(level=args.logging_level)
# Set a random seed used in ChainerRL.
misc.set_random_seed(args.seed, gpus=(args.gpu, ))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2**32
args.outdir = experiments.prepare_output_dir(args, args.outdir)
print('Output files are saved in {}'.format(args.outdir))
def make_env(idx, test):
# Use different random seeds for train and test envs
process_seed = int(process_seeds[idx])
env_seed = 2**32 - 1 - process_seed if test else process_seed
env = atari_wrappers.wrap_deepmind(
atari_wrappers.make_atari(args.env, max_frames=args.max_frames),
episode_life=not test,
clip_rewards=not test,
flicker=args.flicker,
frame_stack=not args.no_frame_stack,
)
env.seed(env_seed)
if args.monitor:
env = chainerrl.wrappers.Monitor(
env, args.outdir, mode='evaluation' if test else 'training')
if args.render:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv([
(lambda: make_env(idx, test))
for idx, env in enumerate(range(args.num_envs))
])
sample_env = make_env(0, test=False)
print('Observation space', sample_env.observation_space)
print('Action space', sample_env.action_space)
n_actions = sample_env.action_space.n
winit_last = chainer.initializers.LeCunNormal(1e-2)
if args.recurrent:
model = chainerrl.links.StatelessRecurrentSequential(
L.Convolution2D(None, 32, 8, stride=4), F.relu,
L.Convolution2D(None, 64, 4, stride=2), F.relu,
L.Convolution2D(None, 64, 3, stride=1), F.relu,
L.Linear(None, 512), F.relu, L.NStepGRU(1, 512, 512, 0),
chainerrl.links.Branched(
chainer.Sequential(
L.Linear(None, n_actions, initialW=winit_last),
chainerrl.distribution.SoftmaxDistribution,
),
L.Linear(None, 1),
))
else:
model = chainer.Sequential(
L.Convolution2D(None, 32, 8, stride=4), F.relu,
L.Convolution2D(None, 64, 4, stride=2), F.relu,
L.Convolution2D(None, 64, 3, stride=1), F.relu,
L.Linear(None, 512), F.relu,
chainerrl.links.Branched(
chainer.Sequential(
L.Linear(None, n_actions, initialW=winit_last),
chainerrl.distribution.SoftmaxDistribution,
),
L.Linear(None, 1),
))
# Draw the computational graph and save it in the output directory.
fake_obss = np.zeros(sample_env.observation_space.shape,
dtype=np.float32)[None]
if args.recurrent:
fake_out, _ = model(fake_obss, None)
else:
fake_out = model(fake_obss)
chainerrl.misc.draw_computational_graph([fake_out],
os.path.join(args.outdir, 'model'))
opt = chainer.optimizers.Adam(alpha=args.lr, eps=1e-5)
opt.setup(model)
opt.add_hook(chainer.optimizer.GradientClipping(0.5))
def phi(x):
# Feature extractor
return np.asarray(x, dtype=np.float32) / 255
agent = PPO(
model,
opt,
gpu=args.gpu,
phi=phi,
update_interval=args.update_interval,
minibatch_size=args.batchsize,
epochs=args.epochs,
clip_eps=0.1,
clip_eps_vf=None,
standardize_advantages=True,
entropy_coef=1e-2,
recurrent=args.recurrent,
)
if args.load:
agent.load(args.load)
if args.demo:
eval_stats = experiments.eval_performance(
env=make_batch_env(test=True),
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs)
print('n_runs: {} mean: {} median: {} stdev: {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
step_hooks = []
# Linearly decay the learning rate to zero
def lr_setter(env, agent, value):
agent.optimizer.alpha = value
step_hooks.append(
experiments.LinearInterpolationHook(args.steps, args.lr, 0,
lr_setter))
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(False),
eval_env=make_batch_env(True),
outdir=args.outdir,
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
checkpoint_freq=args.checkpoint_frequency,
eval_interval=args.eval_interval,
log_interval=args.log_interval,
save_best_so_far_agent=False,
step_hooks=step_hooks,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--outdir', type=str, default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--env', type=str, default='Humanoid-v2',
help='OpenAI Gym MuJoCo env to perform algorithm on.')
parser.add_argument('--num-envs', type=int, default=1,
help='Number of envs run in parallel.')
parser.add_argument('--seed', type=int, default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--gpu', type=int, default=-1,
help='GPU to use, set to -1 if no GPU.')
parser.add_argument('--load', type=str, default='',
help='Directory to load agent from.')
parser.add_argument('--steps', type=int, default=10 ** 6,
help='Total number of timesteps to train the agent.')
parser.add_argument('--eval-n-runs', type=int, default=10,
help='Number of episodes run for each evaluation.')
parser.add_argument('--eval-interval', type=int, default=5000,
help='Interval in timesteps between evaluations.')
parser.add_argument('--replay-start-size', type=int, default=10000,
help='Minimum replay buffer size before ' +
'performing gradient updates.')
parser.add_argument('--batch-size', type=int, default=256,
help='Minibatch size')
parser.add_argument('--render', action='store_true',
help='Render env states in a GUI window.')
parser.add_argument('--demo', action='store_true',
help='Just run evaluation, not training.')
parser.add_argument('--monitor', action='store_true',
help='Wrap env with gym.wrappers.Monitor.')
parser.add_argument('--log-interval', type=int, default=1000,
help='Interval in timesteps between outputting log'
' messages during training')
parser.add_argument('--logger-level', type=int, default=logging.INFO,
help='Level of the root logger.')
parser.add_argument('--policy-output-scale', type=float, default=1.,
help='Weight initialization scale of policy output.')
parser.add_argument('--debug', action='store_true',
help='Debug mode.')
args = parser.parse_args()
logging.basicConfig(level=args.logger_level)
if args.debug:
chainer.set_debug(True)
args.outdir = experiments.prepare_output_dir(
args, args.outdir, argv=sys.argv)
print('Output files are saved in {}'.format(args.outdir))
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu,))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2 ** 32
# def make_env(process_idx, test):
# env = gym.make(args.env)
# # Unwrap TimiLimit wrapper
# assert isinstance(env, gym.wrappers.TimeLimit)
# env = env.env
# # Use different random seeds for train and test envs
# process_seed = int(process_seeds[process_idx])
# env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
# env.seed(env_seed)
# # Cast observations to float32 because our model uses float32
# env = chainerrl.wrappers.CastObservationToFloat32(env)
# # Normalize action space to [-1, 1]^n
# env = chainerrl.wrappers.NormalizeActionSpace(env)
# if args.monitor:
# env = gym.wrappers.Monitor(env, args.outdir)
# if args.render:
# env = chainerrl.wrappers.Render(env)
# return env
def make_env(test):
env = gym.make(
"DaktyPushingSimulationEnv-v0",
level=5,
simulation_backend="mujoco",
control_frequency_in_hertz=100,
state_space_components_to_be_used=None,
alternate_env_object=None,
discretization_factor_torque_control_space=None,
model_as_function_for_pixel_to_latent_space_parsing=(None, None)
)
print('\n############\n', env, '\n############\n')
env.unwrapped.finger.set_resolution_quality('low')
print('\n############\n', env, '\n############\n')
env = gym.wrappers.TimeLimit(env)
print('\n############\n', env, '\n############\n')
# Unwrap TimeLimit wrapper
assert isinstance(env, gym.wrappers.TimeLimit)
env = env.env
# Use different random seeds for train and test envs
env_seed = 2 ** 32 - 1 - args.seed if test else args.seed
env.seed(env_seed)
# Cast observations to float32 because our model uses float32
env = chainerrl.wrappers.CastObservationToFloat32(env)
if args.monitor:
env = chainerrl.wrappers.Monitor(env, args.outdir)
if args.render and not test:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv(
[functools.partial(make_env, idx, test)
for idx, env in enumerate(range(args.num_envs))])
sample_env = make_env(test=False)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
obs_space = sample_env.observation_space
action_space = sample_env.action_space
print('Observation space:', obs_space)
print('Action space:', action_space)
action_size = action_space.low.size
winit = chainer.initializers.GlorotUniform()
winit_policy_output = chainer.initializers.GlorotUniform(
args.policy_output_scale)
def squashed_diagonal_gaussian_head(x):
assert x.shape[-1] == action_size * 2
mean, log_scale = F.split_axis(x, 2, axis=1)
log_scale = F.clip(log_scale, -20., 2.)
var = F.exp(log_scale * 2)
return chainerrl.distribution.SquashedGaussianDistribution(
mean, var=var)
policy = chainer.Sequential(
L.Linear(None, 256, initialW=winit),
F.relu,
L.Linear(None, 256, initialW=winit),
F.relu,
L.Linear(None, action_size * 2, initialW=winit_policy_output),
squashed_diagonal_gaussian_head,
)
policy_optimizer = optimizers.Adam(3e-4).setup(policy)
def make_q_func_with_optimizer():
q_func = chainer.Sequential(
concat_obs_and_action,
L.Linear(None, 256, initialW=winit),
F.relu,
L.Linear(None, 256, initialW=winit),
F.relu,
L.Linear(None, 1, initialW=winit),
)
q_func_optimizer = optimizers.Adam(3e-4).setup(q_func)
return q_func, q_func_optimizer
q_func1, q_func1_optimizer = make_q_func_with_optimizer()
q_func2, q_func2_optimizer = make_q_func_with_optimizer()
# Draw the computational graph and save it in the output directory.
fake_obs = chainer.Variable(
policy.xp.zeros_like(obs_space.low, dtype=np.float32)[None],
name='observation')
fake_action = chainer.Variable(
policy.xp.zeros_like(action_space.low, dtype=np.float32)[None],
name='action')
chainerrl.misc.draw_computational_graph(
[policy(fake_obs)], os.path.join(args.outdir, 'policy'))
chainerrl.misc.draw_computational_graph(
[q_func1(fake_obs, fake_action)], os.path.join(args.outdir, 'q_func1'))
chainerrl.misc.draw_computational_graph(
[q_func2(fake_obs, fake_action)], os.path.join(args.outdir, 'q_func2'))
rbuf = replay_buffer.ReplayBuffer(10 ** 6)
def burnin_action_func():
"""Select random actions until model is updated one or more times."""
return np.random.uniform(
action_space.low, action_space.high).astype(np.float32)
# Hyperparameters in http://arxiv.org/abs/1802.09477
agent = chainerrl.agents.SoftActorCritic(
policy,
q_func1,
q_func2,
policy_optimizer,
q_func1_optimizer,
q_func2_optimizer,
rbuf,
gamma=0.99,
replay_start_size=args.replay_start_size,
gpu=args.gpu,
minibatch_size=args.batch_size,
burnin_action_func=burnin_action_func,
entropy_target=-action_size,
temperature_optimizer=chainer.optimizers.Adam(3e-4),
)
if len(args.load) > 0:
agent.load(args.load)
if args.demo:
eval_stats = experiments.eval_performance(
env=make_env(test=True),
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit,
)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_env(test=False),
eval_env=make_env(test=True),
outdir=args.outdir,
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
log_interval=args.log_interval,
max_episode_len=timestep_limit,
)
def main():
import logging
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--env', type=str, default='Hopper-v2')
parser.add_argument('--num-envs', type=int, default=1)
parser.add_argument('--arch',
type=str,
default='FFGaussian',
choices=('FFSoftmax', 'FFMellowmax', 'FFGaussian'))
parser.add_argument('--bound-mean', action='store_true')
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--outdir',
type=str,
default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--steps', type=int, default=10**6)
parser.add_argument('--eval-interval', type=int, default=10000)
parser.add_argument('--eval-n-runs', type=int, default=10)
parser.add_argument('--reward-scale-factor', type=float, default=1e-2)
parser.add_argument('--standardize-advantages', action='store_true')
parser.add_argument('--render', action='store_true', default=False)
parser.add_argument('--lr', type=float, default=3e-4)
parser.add_argument('--weight-decay', type=float, default=0.0)
parser.add_argument('--load', type=str, default='')
parser.add_argument('--logger-level', type=int, default=logging.DEBUG)
parser.add_argument('--monitor', action='store_true')
parser.add_argument('--window-size', type=int, default=100)
parser.add_argument('--update-interval', type=int, default=2048)
parser.add_argument('--log-interval', type=int, default=1000)
parser.add_argument('--batchsize', type=int, default=64)
parser.add_argument('--epochs', type=int, default=10)
parser.add_argument('--entropy-coef', type=float, default=0.0)
args = parser.parse_args()
#logging.basicConfig(level=args.logger_level)
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu, ))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2**32
args.outdir = experiments.prepare_output_dir(args, args.outdir)
def make_env(process_idx, test):
env = gym.make(args.env)
# Use different random seeds for train and test envs
process_seed = int(process_seeds[process_idx])
env_seed = 2**32 - 1 - process_seed if test else process_seed
env.seed(env_seed)
# Cast observations to float32 because our model uses float32
env = chainerrl.wrappers.CastObservationToFloat32(env)
if args.monitor:
env = gym.wrappers.Monitor(env, args.outdir)
if not test:
# Scale rewards (and thus returns) to a reasonable range so that
# training is easier
env = chainerrl.wrappers.ScaleReward(env, args.reward_scale_factor)
if args.render:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv([
(lambda: make_env(idx, test))
for idx, env in enumerate(range(args.num_envs))
])
# Only for getting timesteps, and obs-action spaces
sample_env = gym.make(args.env)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
obs_space = sample_env.observation_space
action_space = sample_env.action_space
# Normalize observations based on their empirical mean and variance
obs_normalizer = chainerrl.links.EmpiricalNormalization(obs_space.low.size,
clip_threshold=5)
# Switch policy types accordingly to action space types
if args.arch == 'FFSoftmax':
model = A3CFFSoftmax(obs_space.low.size, action_space.n)
elif args.arch == 'FFMellowmax':
model = A3CFFMellowmax(obs_space.low.size, action_space.n)
elif args.arch == 'FFGaussian':
model = A3CFFGaussian(obs_space.low.size,
action_space,
bound_mean=args.bound_mean)
opt = chainer.optimizers.Adam(alpha=args.lr, eps=1e-5)
opt.setup(model)
if args.weight_decay > 0:
opt.add_hook(NonbiasWeightDecay(args.weight_decay))
agent = PPO(
model,
opt,
obs_normalizer=obs_normalizer,
gpu=args.gpu,
update_interval=args.ppo_update_interval,
minibatch_size=args.batchsize,
epochs=args.epochs,
clip_eps_vf=None,
entropy_coef=args.entropy_coef,
standardize_advantages=args.standardize_advantages,
)
if args.load:
agent.load(args.load)
# Linearly decay the learning rate to zero
def lr_setter(env, agent, value):
agent.optimizer.alpha = value
lr_decay_hook = experiments.LinearInterpolationHook(
args.steps, args.lr, 0, lr_setter)
# Linearly decay the clipping parameter to zero
def clip_eps_setter(env, agent, value):
agent.clip_eps = value
clip_eps_decay_hook = experiments.LinearInterpolationHook(
args.steps, 0.2, 0, clip_eps_setter)
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(False),
eval_env=make_batch_env(True),
outdir=args.outdir,
steps=args.steps,
eval_n_runs=args.eval_n_runs,
eval_interval=args.eval_interval,
log_interval=args.log_interval,
return_window_size=args.window_size,
max_episode_len=timestep_limit,
save_best_so_far_agent=False,
step_hooks=[
lr_decay_hook,
clip_eps_decay_hook,
],
)
def main():
import logging
logging.basicConfig(level=logging.DEBUG)
parser = argparse.ArgumentParser()
parser.add_argument('--outdir',
type=str,
default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--env', type=str, default='Humanoid-v2')
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--final-exploration-steps', type=int, default=10**6)
parser.add_argument('--actor-lr', type=float, default=1e-4)
parser.add_argument('--critic-lr', type=float, default=1e-3)
parser.add_argument('--load', type=str, default='')
parser.add_argument('--steps', type=int, default=10**7)
parser.add_argument('--n-hidden-channels', type=int, default=300)
parser.add_argument('--n-hidden-layers', type=int, default=3)
parser.add_argument('--replay-start-size', type=int, default=5000)
parser.add_argument('--n-update-times', type=int, default=1)
parser.add_argument('--target-update-interval', type=int, default=1)
parser.add_argument('--target-update-method',
type=str,
default='soft',
choices=['hard', 'soft'])
parser.add_argument('--soft-update-tau', type=float, default=1e-2)
parser.add_argument('--update-interval', type=int, default=4)
parser.add_argument('--eval-n-runs', type=int, default=100)
parser.add_argument('--eval-interval', type=int, default=10**5)
parser.add_argument('--gamma', type=float, default=0.995)
parser.add_argument('--minibatch-size', type=int, default=200)
parser.add_argument('--render', action='store_true')
parser.add_argument('--demo', action='store_true')
parser.add_argument('--use-bn', action='store_true', default=False)
parser.add_argument('--monitor', action='store_true')
parser.add_argument('--reward-scale-factor', type=float, default=1e-2)
parser.add_argument('--num-envs', type=int, default=1)
args = parser.parse_args()
args.outdir = experiments.prepare_output_dir(args,
args.outdir,
argv=sys.argv)
print('Output files are saved in {}'.format(args.outdir))
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu, ))
def clip_action_filter(a):
return np.clip(a, action_space.low, action_space.high)
def reward_filter(r):
return r * args.reward_scale_factor
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2**32
def make_env(idx, test):
env = gym.make(args.env)
# Use different random seeds for train and test envs
process_seed = int(process_seeds[idx])
env_seed = 2**32 - 1 - process_seed if test else process_seed
env.seed(env_seed)
# Cast observations to float32 because our model uses float32
env = chainerrl.wrappers.CastObservationToFloat32(env)
if args.monitor:
env = gym.wrappers.Monitor(env, args.outdir)
if isinstance(env.action_space, spaces.Box):
misc.env_modifiers.make_action_filtered(env, clip_action_filter)
if not test:
# Scale rewards (and thus returns) to a reasonable range so that
# training is easier
env = chainerrl.wrappers.ScaleReward(env, args.reward_scale_factor)
if args.render and not test:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv([
functools.partial(make_env, idx, test)
for idx, env in enumerate(range(args.num_envs))
])
sample_env = make_env(0, test=False)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
obs_size = np.asarray(sample_env.observation_space.shape).prod()
action_space = sample_env.action_space
action_size = np.asarray(action_space.shape).prod()
if args.use_bn:
q_func = q_functions.FCBNLateActionSAQFunction(
obs_size,
action_size,
n_hidden_channels=args.n_hidden_channels,
n_hidden_layers=args.n_hidden_layers,
normalize_input=True)
pi = policy.FCBNDeterministicPolicy(
obs_size,
action_size=action_size,
n_hidden_channels=args.n_hidden_channels,
n_hidden_layers=args.n_hidden_layers,
min_action=action_space.low,
max_action=action_space.high,
bound_action=True,
normalize_input=True)
else:
q_func = q_functions.FCSAQFunction(
obs_size,
action_size,
n_hidden_channels=args.n_hidden_channels,
n_hidden_layers=args.n_hidden_layers)
pi = policy.FCDeterministicPolicy(
obs_size,
action_size=action_size,
n_hidden_channels=args.n_hidden_channels,
n_hidden_layers=args.n_hidden_layers,
min_action=action_space.low,
max_action=action_space.high,
bound_action=True)
model = DDPGModel(q_func=q_func, policy=pi)
opt_a = optimizers.Adam(alpha=args.actor_lr)
opt_c = optimizers.Adam(alpha=args.critic_lr)
opt_a.setup(model['policy'])
opt_c.setup(model['q_function'])
opt_a.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_a')
opt_c.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_c')
rbuf = replay_buffer.ReplayBuffer(5 * 10**5)
def random_action():
a = action_space.sample()
if isinstance(a, np.ndarray):
a = a.astype(np.float32)
return a
ou_sigma = (action_space.high - action_space.low) * 0.2
explorer = explorers.AdditiveOU(sigma=ou_sigma)
agent = DDPG(model,
opt_a,
opt_c,
rbuf,
gamma=args.gamma,
explorer=explorer,
replay_start_size=args.replay_start_size,
target_update_method=args.target_update_method,
target_update_interval=args.target_update_interval,
update_interval=args.update_interval,
soft_update_tau=args.soft_update_tau,
n_times_update=args.n_update_times,
gpu=args.gpu,
minibatch_size=args.minibatch_size)
if len(args.load) > 0:
agent.load(args.load)
if args.demo:
eval_stats = experiments.eval_performance(
env=make_batch_env(test=True),
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(test=False),
steps=args.steps,
eval_env=make_batch_env(test=True),
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
outdir=args.outdir,
max_episode_len=timestep_limit)
def main():
import logging
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--env', type=str, default='Hopper-v2')
parser.add_argument('--num-envs', type=int, default=1)
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--outdir',
type=str,
default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--steps', type=int, default=10**6)
parser.add_argument('--eval-interval', type=int, default=10000)
parser.add_argument('--eval-n-runs', type=int, default=10)
parser.add_argument('--reward-scale-factor', type=float, default=1e-2)
parser.add_argument('--standardize-advantages', action='store_true')
parser.add_argument('--render', action='store_true', default=False)
parser.add_argument('--lr', type=float, default=3e-4)
parser.add_argument('--weight-decay', type=float, default=0.0)
parser.add_argument('--demo', action='store_true', default=False)
parser.add_argument('--load', type=str, default='')
parser.add_argument('--logger-level', type=int, default=logging.DEBUG)
parser.add_argument('--monitor', action='store_true')
parser.add_argument('--window-size', type=int, default=100)
parser.add_argument('--update-interval', type=int, default=2048)
parser.add_argument('--log-interval', type=int, default=1000)
parser.add_argument('--batchsize', type=int, default=64)
parser.add_argument('--epochs', type=int, default=10)
parser.add_argument('--entropy-coef', type=float, default=0.0)
args = parser.parse_args()
logging.basicConfig(level=args.logger_level)
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu, ))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2**32
args.outdir = experiments.prepare_output_dir(args, args.outdir)
def make_env(process_idx, test):
env = gym.make(args.env)
# Use different random seeds for train and test envs
process_seed = int(process_seeds[process_idx])
env_seed = 2**32 - 1 - process_seed if test else process_seed
env.seed(env_seed)
# Cast observations to float32 because our model uses float32
env = chainerrl.wrappers.CastObservationToFloat32(env)
if args.monitor:
env = chainerrl.wrappers.Monitor(env, args.outdir)
if not test:
# Scale rewards (and thus returns) to a reasonable range so that
# training is easier
env = chainerrl.wrappers.ScaleReward(env, args.reward_scale_factor)
if args.render:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv([
functools.partial(make_env, idx, test)
for idx, env in enumerate(range(args.num_envs))
])
# Only for getting timesteps, and obs-action spaces
sample_env = gym.make(args.env)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
obs_space = sample_env.observation_space
action_space = sample_env.action_space
# Normalize observations based on their empirical mean and variance
obs_normalizer = chainerrl.links.EmpiricalNormalization(obs_space.low.size,
clip_threshold=5)
winit_last = chainer.initializers.LeCunNormal(1e-2)
# Switch policy types accordingly to action space types
if isinstance(action_space, gym.spaces.Discrete):
n_actions = action_space.n
policy = chainer.Sequential(
L.Linear(None, 64),
F.tanh,
L.Linear(None, 64),
F.tanh,
L.Linear(None, n_actions, initialW=winit_last),
chainerrl.distribution.SoftmaxDistribution,
)
elif isinstance(action_space, gym.spaces.Box):
action_size = action_space.low.size
policy = chainer.Sequential(
L.Linear(None, 64),
F.tanh,
L.Linear(None, 64),
F.tanh,
L.Linear(None, action_size, initialW=winit_last),
chainerrl.policies.GaussianHeadWithStateIndependentCovariance(
action_size=action_size,
var_type='diagonal',
var_func=lambda x: F.exp(2 * x), # Parameterize log std
var_param_init=0, # log std = 0 => std = 1
),
)
else:
print("""\
This example only supports gym.spaces.Box or gym.spaces.Discrete action spaces."""
) # NOQA
return
vf = chainer.Sequential(
L.Linear(None, 64),
F.tanh,
L.Linear(None, 64),
F.tanh,
L.Linear(None, 1),
)
# Combine a policy and a value function into a single model
model = chainerrl.links.Branched(policy, vf)
opt = chainer.optimizers.Adam(alpha=args.lr, eps=1e-5)
opt.setup(model)
if args.weight_decay > 0:
opt.add_hook(NonbiasWeightDecay(args.weight_decay))
agent = PPO(
model,
opt,
obs_normalizer=obs_normalizer,
gpu=args.gpu,
update_interval=args.update_interval,
minibatch_size=args.batchsize,
epochs=args.epochs,
clip_eps_vf=None,
entropy_coef=args.entropy_coef,
standardize_advantages=args.standardize_advantages,
)
if args.load:
agent.load(args.load)
if args.demo:
env = make_batch_env(True)
eval_stats = experiments.eval_performance(
env=env,
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
# Linearly decay the learning rate to zero
def lr_setter(env, agent, value):
agent.optimizer.alpha = value
lr_decay_hook = experiments.LinearInterpolationHook(
args.steps, args.lr, 0, lr_setter)
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(False),
eval_env=make_batch_env(True),
outdir=args.outdir,
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
log_interval=args.log_interval,
return_window_size=args.window_size,
max_episode_len=timestep_limit,
save_best_so_far_agent=False,
step_hooks=[
lr_decay_hook,
],
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--env', type=str, default='CarIntersect-v3')
parser.add_argument('--outdir',
type=str,
default='train/results',
help='Directory path to save output files.')
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 31)')
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--demo', action='store_true', default=False)
parser.add_argument('--load', action='store_true', default=None)
parser.add_argument('--train', action='store_true', default=None)
parser.add_argument('--eval-epsilon', type=float, default=0.0)
parser.add_argument('--noisy-net-sigma', type=float, default=0.5)
parser.add_argument('--steps', type=int, default=2 * 10**6)
parser.add_argument('--replay-start-size', type=int, default=2 * 10**4)
parser.add_argument('--eval-n-episodes', type=int, default=5)
parser.add_argument('--eval-interval', type=int, default=10**4)
parser.add_argument('--logging-level',
type=int,
default=20,
help='Logging level. 10:DEBUG, 20:INFO etc.')
parser.add_argument('--monitor',
action='store_true',
default=False,
help='Monitor env.')
parser.add_argument('--num-envs', type=int, default=40)
parser.add_argument('--final-epsilon', type=float, default=0.01)
parser.add_argument('--final-exploration-frames',
type=int,
default=2 * 10**4)
args = parser.parse_args()
import logging
logging.basicConfig(level=args.logging_level)
misc.set_random_seed(args.seed, gpus=(args.gpu, ))
args.outdir = experiments.prepare_output_dir(args, args.outdir)
print('Output files are saved in {}'.format(args.outdir))
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
def make_car_env_discrete(max_frames=30 * 30,
env_seed=42,
random_suffix=None):
print('CarIntersect-v3')
env = gym.make('CarIntersect-v3')
env = chainerrl.wrappers.ContinuingTimeLimit(
env, max_episode_steps=max_frames)
env = MaxAndSkipEnv(env, skip=4)
env = DiscreteWrapper(env)
print('save_wrapper')
env = SaveWrapper(env, random_suffix=random_suffix)
env = WarpFrame(env)
env.seed(env_seed)
return env
def make_batch_env(test):
vec_env = chainerrl.envs.MultiprocessVectorEnv([
functools.partial(make_car_env_discrete)
for _, _ in enumerate(range(args.num_envs))
])
vec_env = chainerrl.wrappers.VectorFrameStack(vec_env, 4)
# print(vec_env.observation_space)
return vec_env
env = make_batch_env(test=False)
n_actions = env.action_space.n
n_atoms = 51
v_max = 10
v_min = -10
q_func = DistributionalDuelingDQN(n_actions,
n_atoms,
v_min,
v_max,
n_input_channels=12)
# Noisy nets
links.to_factorized_noisy(q_func, sigma_scale=args.noisy_net_sigma)
# Turn off explorer
explorer = explorers.LinearDecayEpsilonGreedy(
0.3, args.final_epsilon, args.final_exploration_frames,
lambda: np.random.randint(n_actions))
# Draw the computational graph and save it in the output directory.
# chainerrl.misc.draw_computational_graph(
# [q_func(np.zeros((4, 84, 84), dtype=np.float32)[None])],
# os.path.join(args.outdir, 'model'))
# Use the same hyper parameters as https://arxiv.org/abs/1707.06887
opt = chainer.optimizers.Adam(0.00025, eps=1.5 * 10**-4)
opt.setup(q_func)
# Prioritized Replay
# Anneal beta from beta0 to 1 throughout training
update_interval = 4
betasteps = args.steps / update_interval
rbuf = replay_buffer.PrioritizedReplayBuffer(10**5,
alpha=0.5,
beta0=0.4,
betasteps=betasteps,
num_steps=10)
def phi(x):
# Feature extractor
return np.asarray(x, dtype=np.float32) / 255
Agent = agents.CategoricalDoubleDQN
print(args.replay_start_size)
agent = Agent(
q_func,
opt,
rbuf,
gpu=args.gpu,
gamma=0.99,
explorer=explorer,
minibatch_size=64,
replay_start_size=args.replay_start_size,
target_update_interval=3 * 10**3,
update_interval=update_interval,
batch_accumulator='mean',
phi=phi,
)
if args.load is True:
print('evaluation started')
dir_of_best_network = os.path.join("train/", "best")
agent.load(dir_of_best_network)
stats = experiments.evaluator.eval_performance(env=env,
agent=agent,
n_steps=None,
n_episodes=10,
logger=None)
print(stats)
if args.train or not args.load:
print('training started')
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=env,
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_episodes,
eval_interval=args.eval_interval,
outdir=args.outdir,
save_best_so_far_agent=True,
log_interval=1000,
)
def main():
import logging
logging.basicConfig(level=logging.DEBUG)
parser = argparse.ArgumentParser()
parser.add_argument('--outdir', type=str, default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--env', type=str, default='FetchPickAndPlace-v1')
parser.add_argument('--seed', type=int, default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--final-exploration-steps',
type=int, default=10 ** 6)
parser.add_argument('--actor-lr', type=float, default=1e-3)
parser.add_argument('--critic-lr', type=float, default=1e-3)
parser.add_argument('--load', type=str, default='')
parser.add_argument('--steps', type=int, default=200 * 50 * 16 * 50)
parser.add_argument('--n-hidden-channels', type=int, default=64)
parser.add_argument('--n-hidden-layers', type=int, default=3)
parser.add_argument('--replay-start-size', type=int, default=10000)
parser.add_argument('--n-update-times', type=int, default=40)
parser.add_argument('--target-update-interval',
type=int, default=16 * 50)
parser.add_argument('--target-update-method',
type=str, default='soft', choices=['hard', 'soft'])
parser.add_argument('--soft-update-tau', type=float, default=1 - 0.95)
parser.add_argument('--update-interval', type=int, default=16 * 50)
parser.add_argument('--eval-n-runs', type=int, default=30)
parser.add_argument('--eval-interval', type=int, default=50 * 16 * 50)
parser.add_argument('--gamma', type=float, default=0.98)
parser.add_argument('--minibatch-size', type=int, default=128)
parser.add_argument('--render', action='store_true')
parser.add_argument('--demo', action='store_true')
parser.add_argument('--monitor', action='store_true')
parser.add_argument('--epsilon', type=float, default=0.05)
parser.add_argument('--noise-std', type=float, default=0.05)
parser.add_argument('--clip-threshold', type=float, default=5.0)
parser.add_argument('--num-envs', type=int, default=1)
args = parser.parse_args()
args.outdir = experiments.prepare_output_dir(
args, args.outdir, argv=sys.argv)
print('Output files are saved in {}'.format(args.outdir))
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu,))
def clip_action_filter(a):
return np.clip(a, action_space.low, action_space.high)
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2 ** 32
def make_env(idx, test):
env = gym.make(args.env)
# Use different random seeds for train and test envs
process_seed = int(process_seeds[idx])
env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
env.seed(env_seed)
if args.monitor:
env = gym.wrappers.Monitor(env, args.outdir)
if isinstance(env.action_space, spaces.Box):
misc.env_modifiers.make_action_filtered(env, clip_action_filter)
if args.render and not test:
env = chainerrl.wrappers.Render(env)
if test:
env = HEREnvWrapper(env, args.outdir)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv(
[(lambda: make_env(idx, test))
for idx, env in enumerate(range(args.num_envs))])
sample_env = make_env(0, test=False)
def reward_function(state, action, goal):
return sample_env.compute_reward(achieved_goal=state['achieved_goal'],
desired_goal=goal,
info=None)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
space_dict = sample_env.observation_space.spaces
observation_space = space_dict['observation']
goal_space = space_dict['desired_goal']
obs_size = np.asarray(observation_space.shape).prod()
goal_size = np.asarray(goal_space.shape).prod()
action_space = sample_env.action_space
action_size = np.asarray(action_space.shape).prod()
q_func = q_functions.FCSAQFunction(
obs_size + goal_size, action_size,
n_hidden_channels=args.n_hidden_channels,
n_hidden_layers=args.n_hidden_layers)
pi = policy.FCDeterministicPolicy(
obs_size + goal_size, action_size=action_size,
n_hidden_channels=args.n_hidden_channels,
n_hidden_layers=args.n_hidden_layers,
min_action=action_space.low, max_action=action_space.high,
bound_action=True)
model = DDPGModel(q_func=q_func, policy=pi)
opt_a = optimizers.Adam(alpha=args.actor_lr)
opt_c = optimizers.Adam(alpha=args.critic_lr)
opt_a.setup(model['policy'])
opt_c.setup(model['q_function'])
opt_a.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_a')
opt_c.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_c')
rbuf = replay_buffer.HindsightReplayBuffer(reward_function,
10 ** 6,
future_k=4)
def phi(dict_state):
return np.concatenate(
(dict_state['observation'].astype(np.float32, copy=False),
dict_state['desired_goal'].astype(np.float32, copy=False)), 0)
# Normalize observations based on their empirical mean and variance
obs_normalizer = chainerrl.links.EmpiricalNormalization(
obs_size + goal_size, clip_threshold=args.clip_threshold)
explorer = HERExplorer(args.noise_std,
args.epsilon,
action_space)
agent = DDPG(model, opt_a, opt_c, rbuf,
obs_normalizer=obs_normalizer,
gamma=args.gamma,
explorer=explorer,
replay_start_size=args.replay_start_size,
phi=phi,
target_update_method=args.target_update_method,
target_update_interval=args.target_update_interval,
update_interval=args.update_interval,
soft_update_tau=args.soft_update_tau,
n_times_update=args.n_update_times,
gpu=args.gpu,
minibatch_size=args.minibatch_size,
clip_critic_tgt=(-1.0/(1.0-args.gamma), 0.0))
if len(args.load) > 0:
agent.load(args.load)
if args.demo:
eval_stats = experiments.eval_performance(
env=make_batch_env(test=True),
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent, env=make_batch_env(test=False), steps=args.steps,
eval_env=make_batch_env(test=True), eval_n_steps=None,
eval_n_episodes=args.eval_n_runs, eval_interval=args.eval_interval,
outdir=args.outdir,
max_episode_len=timestep_limit)
def main():
import logging
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=-1)
parser.add_argument('--env', type=str, default='Hopper-v2')
parser.add_argument('--num-envs', type=int, default=1)
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--outdir',
type=str,
default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--steps', type=int, default=10**6)
parser.add_argument('--eval-interval', type=int, default=10000)
parser.add_argument('--eval-n-runs', type=int, default=10)
parser.add_argument('--reward-scale-factor', type=float, default=1e-2)
parser.add_argument('--standardize-advantages', action='store_true')
parser.add_argument('--render', action='store_true', default=False)
parser.add_argument('--lr', type=float, default=3e-4)
parser.add_argument('--weight-decay', type=float, default=0.0)
parser.add_argument('--demo', action='store_true', default=False)
parser.add_argument('--load', type=str, default='')
parser.add_argument('--logger-level', type=int, default=logging.DEBUG)
parser.add_argument('--monitor', action='store_true')
parser.add_argument('--window-size', type=int, default=100)
parser.add_argument('--update-interval', type=int, default=2048)
parser.add_argument('--log-interval', type=int, default=1000)
parser.add_argument('--batchsize', type=int, default=64)
parser.add_argument('--epochs', type=int, default=10)
parser.add_argument('--entropy-coef', type=float, default=0.0)
args = parser.parse_args()
logging.basicConfig(level=args.logger_level)
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu, ))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2**32
args.outdir = experiments.prepare_output_dir(args, args.outdir)
# def make_env(process_idx, test):
# env = gym.make(args.env)
# # Use different random seeds for train and test envs
# process_seed = int(process_seeds[process_idx])
# env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
# env.seed(env_seed)
# # Cast observations to float32 because our model uses float32
# env = chainerrl.wrappers.CastObservationToFloat32(env)
# if args.monitor:
# env = chainerrl.wrappers.Monitor(env, args.outdir)
# if not test:
# # Scale rewards (and thus returns) to a reasonable range so that
# # training is easier
# env = chainerrl.wrappers.ScaleReward(env, args.reward_scale_factor)
# if args.render:
# env = chainerrl.wrappers.Render(env)
# return env
def make_env(test):
env = gym.make(
"DaktyPushingSimulationEnv-v0",
level=5,
simulation_backend="mujoco",
control_frequency_in_hertz=100,
state_space_components_to_be_used=None,
alternate_env_object=None,
discretization_factor_torque_control_space=None,
model_as_function_for_pixel_to_latent_space_parsing=(None, None))
# print('\n############\n', env, '\n############\n')
env.unwrapped.finger.set_resolution_quality('low')
# print('\n############\n', env, '\n############\n')
env = gym.wrappers.TimeLimit(env)
# print('\n############\n', env, '\n############\n')
# Unwrap TimeLimit wrapper
assert isinstance(env, gym.wrappers.TimeLimit)
env = env.env
# Use different random seeds for train and test envs
# env_seed = 2 ** 32 - 1 - args.seed if test else args.seed
# env.seed(env_seed)
process_seed = 420
env_seed = 2**32 - 1 - process_seed if test else process_seed
env.seed(env_seed)
# Cast observations to float32 because our model uses float32
env = chainerrl.wrappers.CastObservationToFloat32(env)
if args.monitor:
env = chainerrl.wrappers.Monitor(env, args.outdir)
if args.render and not test:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv([
functools.partial(make_env, idx, test)
for idx, env in enumerate(range(args.num_envs))
])
# Only for getting timesteps, and obs-action spaces
sample_env = make_env(0)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
obs_space = sample_env.observation_space
action_space = sample_env.action_space
print('\n\n------------------- obs_space: ', obs_space.shape, '\n\n\n')
# Normalize observations based on their empirical mean and variance
obs_normalizer = chainerrl.links.EmpiricalNormalization(obs_space.low.size,
clip_threshold=5)
winit_last = chainer.initializers.LeCunNormal(1e-2)
action_size = action_space.low.size
policy = chainer.Sequential(
L.Linear(None, 64),
F.tanh,
L.Linear(None, 64),
F.tanh,
L.Linear(None, action_size, initialW=winit_last),
chainerrl.policies.GaussianHeadWithStateIndependentCovariance(
action_size=action_size,
var_type='diagonal',
var_func=lambda x: F.exp(2 * x), # Parameterize log std
var_param_init=0, # log std = 0 => std = 1
))
vf = chainer.Sequential(
concat_obs_and_action,
L.Linear(None, 64),
F.tanh,
L.Linear(None, 64),
F.tanh,
L.Linear(None, 1),
)
# Combine a policy and a value function into a single model
model = chainerrl.links.Branched(policy, vf)
opt = chainer.optimizers.Adam(alpha=args.lr, eps=1e-5)
opt.setup(model)
if args.weight_decay > 0:
opt.add_hook(NonbiasWeightDecay(args.weight_decay))
agent = PPO(
model,
opt,
obs_normalizer=obs_normalizer,
gpu=args.gpu,
update_interval=args.update_interval,
minibatch_size=args.batchsize,
epochs=args.epochs,
clip_eps_vf=None,
entropy_coef=args.entropy_coef,
standardize_advantages=args.standardize_advantages,
)
if args.load:
agent.load(args.load)
if args.demo:
env = make_env(True)
eval_stats = experiments.eval_performance(
env=env,
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
env = make_env(False)
n_episodes = 10000
# pbar = tqdm(total=n_episodes)
max_episode_len = 1000
for i in range(1, n_episodes + 1):
# pbar.update(1)
obs = env.reset()
# print('obs inital..............', obs.shape)
reward = 0
done = False
R = 0 # return (sum of rewards)
t = 0 # time step
# pbar = tqdm(total=max_episode_len)
while not done and t < max_episode_len:
# pbar.update(1)
# Uncomment to watch the behaviour
# env.render()
action = agent.act_and_train(obs, reward)
# print('action..................', action)
obs, reward, done, _ = env.step(action)
# print('obs.....................', obs)
# print('reward..................', reward)
R += reward
t += 1
if i % 10 == 0:
print('episode:', i, 'R:', R, 'statistics:',
agent.get_statistics())
agent.stop_episode_and_train(obs, reward, done)
print('Finished.')
# Linearly decay the learning rate to zero
def lr_setter(env, agent, value):
agent.optimizer.alpha = value
lr_decay_hook = experiments.LinearInterpolationHook(
args.steps, args.lr, 0, lr_setter)
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_env(False),
eval_env=make_env(True),
outdir=args.outdir,
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
log_interval=args.log_interval,
return_window_size=args.window_size,
max_episode_len=timestep_limit,
save_best_so_far_agent=False,
step_hooks=[
lr_decay_hook,
],
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--env', type=str, default='BreakoutNoFrameskip-v4')
parser.add_argument('--outdir',
type=str,
default='results',
help='Directory path to save output files.'
' If it does not exist, it will be created.')
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 31)')
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--demo', action='store_true', default=False)
parser.add_argument('--load', type=str, default=None)
parser.add_argument('--final-exploration-frames', type=int, default=10**6)
parser.add_argument('--final-epsilon', type=float, default=0.01)
parser.add_argument('--eval-epsilon', type=float, default=0.001)
parser.add_argument('--noisy-net-sigma', type=float, default=None)
parser.add_argument('--arch',
type=str,
default='doubledqn',
choices=['nature', 'nips', 'dueling', 'doubledqn'])
parser.add_argument('--steps', type=int, default=5 * 10**7)
parser.add_argument(
'--max-frames',
type=int,
default=30 * 60 * 60, # 30 minutes with 60 fps
help='Maximum number of frames for each episode.')
parser.add_argument('--replay-start-size', type=int, default=5 * 10**4)
parser.add_argument('--target-update-interval',
type=int,
default=3 * 10**4)
parser.add_argument('--eval-interval', type=int, default=10**5)
parser.add_argument('--update-interval', type=int, default=4)
parser.add_argument('--eval-n-runs', type=int, default=10)
parser.add_argument('--no-clip-delta',
dest='clip_delta',
action='store_false')
parser.set_defaults(clip_delta=True)
parser.add_argument('--agent',
type=str,
default='DoubleDQN',
choices=['DQN', 'DoubleDQN', 'PAL'])
parser.add_argument('--logging-level',
type=int,
default=20,
help='Logging level. 10:DEBUG, 20:INFO etc.')
parser.add_argument('--render',
action='store_true',
default=False,
help='Render env states in a GUI window.')
parser.add_argument('--monitor',
action='store_true',
default=False,
help='Monitor env. Videos and additional information'
' are saved as output files.')
parser.add_argument('--lr',
type=float,
default=2.5e-4,
help='Learning rate')
parser.add_argument('--prioritized',
action='store_true',
default=False,
help='Use prioritized experience replay.')
parser.add_argument('--num-envs', type=int, default=1)
args = parser.parse_args()
import logging
logging.basicConfig(level=args.logging_level)
# Set a random seed used in ChainerRL.
misc.set_random_seed(args.seed, gpus=(args.gpu, ))
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2**32
args.outdir = experiments.prepare_output_dir(args, args.outdir)
print('Output files are saved in {}'.format(args.outdir))
def make_env(idx, test):
# Use different random seeds for train and test envs
process_seed = int(process_seeds[idx])
env_seed = 2**32 - 1 - process_seed if test else process_seed
env = atari_wrappers.wrap_deepmind(
atari_wrappers.make_atari(args.env, max_frames=args.max_frames),
episode_life=not test,
clip_rewards=not test,
frame_stack=False,
)
if test:
# Randomize actions like epsilon-greedy in evaluation as well
env = chainerrl.wrappers.RandomizeAction(env, args.eval_epsilon)
env.seed(env_seed)
if args.monitor:
env = gym.wrappers.Monitor(
env, args.outdir, mode='evaluation' if test else 'training')
if args.render:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
vec_env = chainerrl.envs.MultiprocessVectorEnv([
functools.partial(make_env, idx, test)
for idx, env in enumerate(range(args.num_envs))
])
vec_env = chainerrl.wrappers.VectorFrameStack(vec_env, 4)
return vec_env
sample_env = make_env(0, test=False)
n_actions = sample_env.action_space.n
q_func = parse_arch(args.arch, n_actions)
if args.noisy_net_sigma is not None:
links.to_factorized_noisy(q_func)
# Turn off explorer
explorer = explorers.Greedy()
# Draw the computational graph and save it in the output directory.
chainerrl.misc.draw_computational_graph(
[q_func(np.zeros((4, 84, 84), dtype=np.float32)[None])],
os.path.join(args.outdir, 'model'))
# Use the same hyper parameters as the Nature paper's
opt = optimizers.RMSpropGraves(lr=args.lr,
alpha=0.95,
momentum=0.0,
eps=1e-2)
opt.setup(q_func)
# Select a replay buffer to use
if args.prioritized:
# Anneal beta from beta0 to 1 throughout training
betasteps = args.steps / args.update_interval
rbuf = replay_buffer.PrioritizedReplayBuffer(10**6,
alpha=0.6,
beta0=0.4,
betasteps=betasteps)
else:
rbuf = replay_buffer.ReplayBuffer(10**6)
explorer = explorers.LinearDecayEpsilonGreedy(
1.0, args.final_epsilon, args.final_exploration_frames,
lambda: np.random.randint(n_actions))
def phi(x):
# Feature extractor
return np.asarray(x, dtype=np.float32) / 255
Agent = parse_agent(args.agent)
agent = Agent(q_func,
opt,
rbuf,
gpu=args.gpu,
gamma=0.99,
explorer=explorer,
replay_start_size=args.replay_start_size,
target_update_interval=args.target_update_interval,
clip_delta=args.clip_delta,
update_interval=args.update_interval,
batch_accumulator='sum',
phi=phi)
if args.load:
agent.load(args.load)
if args.demo:
eval_stats = experiments.eval_performance(
env=make_batch_env(test=True),
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(test=False),
eval_env=make_batch_env(test=True),
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
outdir=args.outdir,
save_best_so_far_agent=False,
log_interval=1000,
)
def main():
import logging
parser = argparse.ArgumentParser()
parser.add_argument('--env', type=str, default='Pendulum-v0')
parser.add_argument('--arch',
type=str,
default='Gaussian',
choices=('FFSoftmax', 'FFMellowmax', 'Gaussian'))
parser.add_argument('--seed',
type=int,
default=0,
help='Random seed [0, 2 ** 32)')
parser.add_argument('--outdir', type=str, default=None)
parser.add_argument('--profile', action='store_true')
parser.add_argument('--steps', type=int, default=8 * 10**7)
parser.add_argument('--update-steps', type=int, default=5)
parser.add_argument('--log-interval', type=int, default=1000)
parser.add_argument('--eval-interval', type=int, default=10**5)
parser.add_argument('--eval-n-runs', type=int, default=10)
parser.add_argument('--reward-scale-factor', type=float, default=1e-2)
parser.add_argument('--rmsprop-epsilon', type=float, default=1e-5)
parser.add_argument('--render', action='store_true', default=False)
parser.add_argument('--gamma',
type=float,
default=0.99,
help='discount factor')
parser.add_argument('--use-gae',
action='store_true',
default=False,
help='use generalized advantage estimation')
parser.add_argument('--tau',
type=float,
default=0.95,
help='gae parameter')
parser.add_argument('--lr', type=float, default=7e-4)
parser.add_argument('--weight-decay', type=float, default=0.0)
parser.add_argument('--demo', action='store_true', default=False)
parser.add_argument('--load', type=str, default='')
parser.add_argument('--logger-level', type=int, default=logging.DEBUG)
parser.add_argument('--monitor', action='store_true')
parser.add_argument('--max-grad-norm',
type=float,
default=0.5,
help='value loss coefficient')
parser.add_argument('--alpha',
type=float,
default=0.99,
help='RMSprop optimizer alpha')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--num-envs', type=int, default=1)
args = parser.parse_args()
logging.basicConfig(level=args.logger_level)
# Set a random seed used in ChainerRL.
# If you use more than one processes, the results will be no longer
# deterministic even with the same random seed.
misc.set_random_seed(args.seed)
# Set different random seeds for different subprocesses.
# If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
# If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
assert process_seeds.max() < 2**32
args.outdir = experiments.prepare_output_dir(args, args.outdir)
def make_env(process_idx, test):
env = gym.make(args.env)
# Use different random seeds for train and test envs
process_seed = int(process_seeds[process_idx])
env_seed = 2**32 - 1 - process_seed if test else process_seed
env.seed(env_seed)
# Cast observations to float32 because our model uses float32
env = chainerrl.wrappers.CastObservationToFloat32(env)
if args.monitor and process_idx == 0:
env = chainerrl.wrappers.Monitor(env, args.outdir)
# Scale rewards observed by agents
if not test:
misc.env_modifiers.make_reward_filtered(
env, lambda x: x * args.reward_scale_factor)
if args.render and process_idx == 0 and not test:
env = chainerrl.wrappers.Render(env)
return env
def make_batch_env(test):
return chainerrl.envs.MultiprocessVectorEnv([
functools.partial(make_env, idx, test)
for idx, env in enumerate(range(args.num_envs))
])
sample_env = make_env(process_idx=0, test=False)
timestep_limit = sample_env.spec.tags.get(
'wrapper_config.TimeLimit.max_episode_steps')
obs_space = sample_env.observation_space
action_space = sample_env.action_space
# Switch policy types accordingly to action space types
if args.arch == 'Gaussian':
model = A2CGaussian(obs_space.low.size, action_space.low.size)
elif args.arch == 'FFSoftmax':
model = A2CFFSoftmax(obs_space.low.size, action_space.n)
elif args.arch == 'FFMellowmax':
model = A2CFFMellowmax(obs_space.low.size, action_space.n)
optimizer = chainer.optimizers.RMSprop(args.lr,
eps=args.rmsprop_epsilon,
alpha=args.alpha)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.GradientClipping(args.max_grad_norm))
if args.weight_decay > 0:
optimizer.add_hook(NonbiasWeightDecay(args.weight_decay))
agent = a2c.A2C(model,
optimizer,
gamma=args.gamma,
gpu=args.gpu,
num_processes=args.num_envs,
update_steps=args.update_steps,
use_gae=args.use_gae,
tau=args.tau)
if args.load:
agent.load(args.load)
if args.demo:
env = make_env(0, True)
eval_stats = experiments.eval_performance(
env=env,
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=make_batch_env(test=False),
eval_env=make_batch_env(test=True),
steps=args.steps,
log_interval=args.log_interval,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
outdir=args.outdir,
)
def main(args, train_env):
logging.basicConfig(level=args.logger_level)
# Set a random seed used in ChainerRL
misc.set_random_seed(args.seed, gpus=(args.gpu, ))
if not (args.demo and args.load):
args.outdir = experiments.prepare_output_dir(args, args.outdir)
temp = args.outdir.split('/')[-1]
dst = args.outdir.strip(temp)
def make_env(test):
env = gym.make(args.env)
if test:
episode_length = args.eval_episode_length
else:
episode_length = args.episode_length
env.initialize_environment(
case=args.state_rep,
n_historical_events=args.n_historical_events,
episode_length=episode_length,
n_experts=args.n_experts,
n_demos_per_expert=1,
n_expert_time_steps=args.length_expert_TS,
seed_agent=args.seed_agent,
seed_expert=args.seed_expert,
adam_days=args.adam_days)
# Use different random seeds for train and test envs
env_seed = 2**32 - 1 - args.seed if test else args.seed
env.seed(env_seed)
# Cast observations to float32 because our model uses float32
env = chainerrl.wrappers.CastObservationToFloat32(env)
if args.monitor:
env = gym.wrappers.Monitor(env, args.outdir)
if not test:
# Scale rewards (and thus returns) to a reasonable range so that
# training is easier
env = chainerrl.wrappers.ScaleReward(env, args.reward_scale_factor)
if args.render:
env = chainerrl.wrappers.Render(env)
return env
sample_env = gym.make(args.env)
sample_env.initialize_environment(
case=args.state_rep,
n_historical_events=args.n_historical_events,
episode_length=args.episode_length,
n_experts=args.n_experts,
n_demos_per_expert=1,
n_expert_time_steps=args.length_expert_TS,
seed_agent=args.seed_agent,
seed_expert=args.seed_expert,
adam_days=args.adam_days)
demonstrations = sample_env.generate_expert_trajectories(out_dir=dst,
eval=False)
timestep_limit = None #sample_env.spec.tags.get('wrapper_config.TimeLimit.max_episode_steps') # This value is None
# Generate expert data for evaluation
temp_env = gym.make(args.env)
temp_env.initialize_environment(
case=args.state_rep,
n_historical_events=args.n_historical_events,
episode_length=
0, # This parameter does not really matter since we create this env only for generating samples
n_experts=args.n_experts,
n_demos_per_expert=1, # We do not perform any clustering right now
# n_demos_per_expert=args.n_demos_per_expert, # How large should the expert cluster be?
n_expert_time_steps=args.
eval_episode_length, # How long should each expert trajectory be?
seed_expert=args.seed_expert,
adam_days=args.adam_days)
temp_env.generate_expert_trajectories(out_dir=dst, eval=True)
obs_space = sample_env.observation_space
action_space = sample_env.action_space
# Normalize observations based on their empirical mean and variance
if args.state_rep == 1:
obs_dim = obs_space.low.size
elif args.state_rep == 2 or args.state_rep == 21 or args.state_rep == 22 or args.state_rep == 24 or args.state_rep == 4 or args.state_rep == 221 or args.state_rep == 222 \
or args.state_rep == 71 or args.state_rep == 17 or args.state_rep == 81:
obs_dim = obs_space.n
elif args.state_rep == 3 or args.state_rep == 11 or args.state_rep == 23 or args.state_rep == 31 or args.state_rep == 7:
obs_dim = obs_space.nvec.size
else:
raise NotImplementedError
if args.normalize_obs:
obs_normalizer = chainerrl.links.EmpiricalNormalization(
obs_dim,
clip_threshold=5) # shape: Shape of input values except batch axis
else:
obs_normalizer = None
# Switch policy types accordingly to action space types
if args.arch == 'FFSoftmax':
model = A3CFFSoftmax(obs_dim,
action_space.n,
hidden_sizes=args.G_layers)
elif args.arch == 'FFMellowmax':
model = A3CFFMellowmax(obs_space.low.size, action_space.n)
elif args.arch == 'FFGaussian':
model = A3CFFGaussian(obs_space.low.size,
action_space,
bound_mean=args.bound_mean)
opt = chainer.optimizers.Adam(alpha=args.lr, eps=10e-1)
opt.setup(model)
if args.show_D_dummy: # Let discriminator see dummy
input_dim_D = obs_dim + 1
elif not args.show_D_dummy: # Do not let discriminator see dummy
if args.state_rep == 21 or args.state_rep == 17:
input_dim_D = obs_dim + 1
else:
input_dim_D = obs_dim + 1 - args.n_experts
if args.weight_decay > 0:
opt.add_hook(NonbiasWeightDecay(args.weight_decay))
if args.algo == 'ppo':
agent = PPO(
model,
opt,
obs_normalizer=obs_normalizer,
gpu=args.gpu,
update_interval=args.update_interval,
minibatch_size=args.batchsize,
epochs=args.epochs,
clip_eps_vf=None,
entropy_coef=args.entropy_coef,
standardize_advantages=args.standardize_advantages,
)
elif args.algo == 'gail':
from customer_behaviour.algorithms.irl.gail import GAIL as G
from customer_behaviour.algorithms.irl.gail import Discriminator as D
demonstrations = np.load(dst + '/expert_trajectories.npz')
D = D(gpu=args.gpu,
input_dim=input_dim_D,
hidden_sizes=args.D_layers,
loss_type=args.loss_type)
agent = G(env=sample_env,
demonstrations=demonstrations,
discriminator=D,
model=model,
optimizer=opt,
obs_normalizer=obs_normalizer,
gpu=args.gpu,
update_interval=args.update_interval,
minibatch_size=args.batchsize,
epochs=args.epochs,
clip_eps_vf=None,
entropy_coef=args.entropy_coef,
standardize_advantages=args.standardize_advantages,
args=args)
elif args.algo == 'airl':
from customer_behaviour.algorithms.irl.airl import AIRL as G
from customer_behaviour.algorithms.irl.airl import Discriminator as D
# obs_normalizer = None
demonstrations = np.load(dst + '/expert_trajectories.npz')
D = D(gpu=args.gpu,
input_dim=input_dim_D - 1,
hidden_sizes=args.D_layers) # AIRL only inputs state to D
agent = G(env=sample_env,
demonstrations=demonstrations,
discriminator=D,
model=model,
optimizer=opt,
obs_normalizer=obs_normalizer,
gpu=args.gpu,
update_interval=args.update_interval,
minibatch_size=args.batchsize,
epochs=args.epochs,
clip_eps_vf=None,
entropy_coef=args.entropy_coef,
standardize_advantages=args.standardize_advantages,
noise=args.noise,
n_experts=args.n_experts,
episode_length=args.episode_length,
adam_days=args.adam_days,
dummy_D=args.show_D_dummy)
elif args.algo == 'mmct-gail':
from customer_behaviour.algorithms.irl.gail.mmct_gail import MMCTGAIL as G
from customer_behaviour.algorithms.irl.gail import Discriminator as D
demonstrations = np.load(dst + '/expert_trajectories.npz')
D = D(gpu=args.gpu,
input_dim=input_dim_D,
hidden_sizes=args.D_layers,
loss_type=args.loss_type)
agent = G(env=sample_env,
demonstrations=demonstrations,
discriminator=D,
model=model,
optimizer=opt,
obs_normalizer=obs_normalizer,
gpu=args.gpu,
update_interval=args.update_interval,
minibatch_size=args.batchsize,
epochs=args.epochs,
clip_eps_vf=None,
entropy_coef=args.entropy_coef,
standardize_advantages=args.standardize_advantages,
args=args)
if args.load:
# By default, not in here
agent.load(args.load)
if args.demo:
# By default, not in here
env = make_env(True)
eval_stats = experiments.eval_performance(
env=env,
agent=agent,
n_steps=None,
n_episodes=args.eval_n_runs,
max_episode_len=timestep_limit)
print('n_runs: {} mean: {} median: {} stdev {}'.format(
args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
eval_stats['stdev']))
outdir = args.load if args.load else args.outdir
save_agent_demo(make_env(False), agent, outdir)
else:
# Linearly decay the learning rate to zero
def lr_setter(env, agent, value):
agent.optimizer.alpha = value
lr_decay_hook = experiments.LinearInterpolationHook(
args.steps, args.lr, 0, lr_setter)
# Linearly decay the clipping parameter to zero
def clip_eps_setter(env, agent, value):
agent.clip_eps = max(value, 1e-8)
clip_eps_decay_hook = experiments.LinearInterpolationHook(
args.steps, 0.2, 0, clip_eps_setter)
if train_env is None:
experiments.train_agent_with_evaluation(
agent=agent,
env=make_env(
False
), # Environment train the agent against (False -> scaled rewards)
eval_env=make_env(True), # Environment used for evaluation
outdir=args.outdir,
steps=args.
steps, # Total number of timesteps for training (args.n_training_episodes*args.episode_length)
eval_n_steps=
None, # Number of timesteps at each evaluation phase
eval_n_episodes=args.
eval_n_runs, # Number of episodes at each evaluation phase (default: 10)
eval_interval=args.
eval_interval, # Interval of evaluation (defualt: 10000 steps (?))
train_max_episode_len=
timestep_limit, # Maximum episode length during training (is None)
save_best_so_far_agent=False,
step_hooks=[
lr_decay_hook,
clip_eps_decay_hook,
],
checkpoint_freq=args.eval_interval)
else:
experiments.train_agent_batch_with_evaluation(
agent=agent,
env=train_env,
steps=args.steps,
eval_n_steps=None,
eval_n_episodes=args.eval_n_runs,
eval_interval=args.eval_interval,
outdir=args.outdir,
max_episode_len=timestep_limit,
eval_max_episode_len=None,
eval_env=make_env(True),
step_hooks=[
lr_decay_hook,
clip_eps_decay_hook,
],
save_best_so_far_agent=False,
checkpoint_freq=args.eval_interval,
log_interval=args.update_interval)
save_agent_demo(
make_env(True), agent, args.outdir, 10 * args.eval_episode_length
) # originally it was make_env(test=False) which seems strange
# Move result files to correct folder and remove empty folder
move_dir(args.outdir, dst)
os.rmdir(args.outdir)
if args.save_results:
print('Saving result...')
res2.save_data(dst, 10000, 50, N=1)
print('Running evaluate policy...')
ep.eval_policy(a_dir_path=dst)
# else:
# if args.n_experts <= 10:
# print('Running evaluate policy...')
# ep.eval_policy(a_dir_path=dst)
# # print('Running evaluate training...')
# # ets.eval_training(a_dir_path=dst)
# print('Done')
if args.save_report_material:
print('Saving dataframe...')
if args.state_rep == 21:
if args.algo == 'gail':
folder_name = 'gail'
elif args.algo == 'airl':
folder_name = 'airl'
elif args.state_rep == 22:
if args.algo == 'gail':
folder_name = 'gail_dummies'
elif args.algo == 'airl':
folder_name = 'airl_dummies'
elif args.state_rep == 81:
if args.algo == 'gail':
folder_name = 'gail_adams'
elif args.algo == 'airl':
folder_name = 'airl_adams'
elif args.state_rep == 17:
folder_name = 'ail'
elif args.state_rep == 221:
folder_name = 'ail_dummies'
elif args.state_rep == 71:
folder_name = 'ail_adams'
report_material.save_df(dst, folder_name)
if args.save_folder is not None:
print('Saving result to ' + args.save_folder)
os.makedirs(os.path.join(os.getcwd(), args.save_folder), exist_ok=True)
from distutils.dir_util import copy_tree
copy_tree(
os.path.join(os.getcwd(), dst),
os.path.join(os.getcwd(), args.save_folder,
args.outdir.split('/')[-2]))