def run_task(snapshot_config, *_):
"""Defines the main experiment routine.
Args:
snapshot_config (metarl.experiment.SnapshotConfig): Configuration
values for snapshotting.
*_ (object): Hyperparameters (unused).
"""
with LocalTFRunner(snapshot_config=snapshot_config) as runner:
env = TfEnv(env_name='CartPole-v1')
policy = CategoricalLSTMPolicy(name='policy', env_spec=env.spec)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
max_kl_step=0.01,
optimizer=ConjugateGradientOptimizer,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)))
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=4000)
def test_rl2_trpo_pendulum(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
algo = RL2TRPO(
rl2_max_path_length=self.max_path_length,
meta_batch_size=self.meta_batch_size,
task_sampler=self.tasks,
env_spec=self.env_spec,
policy=self.policy,
baseline=self.baseline,
max_path_length=self.max_path_length * self.episode_per_task,
discount=0.99,
max_kl_step=0.01,
optimizer=ConjugateGradientOptimizer,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)))
runner.setup(algo,
self.tasks.sample(self.meta_batch_size),
sampler_cls=LocalSampler,
n_workers=self.meta_batch_size,
worker_class=RL2Worker)
last_avg_ret = runner.train(n_epochs=1,
batch_size=self.episode_per_task *
self.max_path_length *
self.meta_batch_size)
assert last_avg_ret > -40
def trpo_cartpole_recurrent(ctxt, seed, n_epochs, batch_size, plot):
"""Train TRPO with a recurrent policy on CartPole.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
n_epochs (int): Number of epochs for training.
seed (int): Used to seed the random number generator to produce
determinism.
batch_size (int): Batch size used for training.
plot (bool): Whether to plot or not.
"""
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
env = MetaRLEnv(env_name='CartPole-v1')
policy = CategoricalLSTMPolicy(name='policy', env_spec=env.spec)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
max_kl_step=0.01,
optimizer=ConjugateGradientOptimizer,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)))
runner.setup(algo, env)
runner.train(n_epochs=n_epochs, batch_size=batch_size, plot=plot)
def rl2_trpo_halfcheetah(ctxt, seed, max_path_length, meta_batch_size,
n_epochs, episode_per_task):
"""Train TRPO with HalfCheetah environment.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
max_path_length (int): Maximum length of a single rollout.
meta_batch_size (int): Meta batch size.
n_epochs (int): Total number of epochs for training.
episode_per_task (int): Number of training episode per task.
"""
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
tasks = task_sampler.SetTaskSampler(
lambda: RL2Env(env=HalfCheetahVelEnv()))
env_spec = RL2Env(env=HalfCheetahVelEnv()).spec
policy = GaussianGRUPolicy(name='policy',
hidden_dim=64,
env_spec=env_spec,
state_include_action=False)
baseline = LinearFeatureBaseline(env_spec=env_spec)
algo = RL2TRPO(rl2_max_path_length=max_path_length,
meta_batch_size=meta_batch_size,
task_sampler=tasks,
env_spec=env_spec,
policy=policy,
baseline=baseline,
max_path_length=max_path_length * episode_per_task,
discount=0.99,
max_kl_step=0.01,
optimizer=ConjugateGradientOptimizer,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)))
runner.setup(algo,
tasks.sample(meta_batch_size),
sampler_cls=LocalSampler,
n_workers=meta_batch_size,
worker_class=RL2Worker,
worker_args=dict(n_paths_per_trial=episode_per_task))
runner.train(n_epochs=n_epochs,
batch_size=episode_per_task * max_path_length *
meta_batch_size)
def test_trpo_lstm_cartpole(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = MetaRLEnv(normalize(gym.make('CartPole-v1')))
policy = CategoricalLSTMPolicy(name='policy', env_spec=env.spec)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
max_kl_step=0.01,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)))
snapshotter.snapshot_dir = './'
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
assert last_avg_ret > 80
env.close()
def test_gaussian_policies(self, policy_cls):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = MetaRLEnv(normalize(gym.make('Pendulum-v0')))
policy = policy_cls(name='policy', env_spec=env.spec)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(
env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
max_kl_step=0.01,
optimizer=ConjugateGradientOptimizer,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)),
)
runner.setup(algo, env)
runner.train(n_epochs=1, batch_size=4000)
env.close()
def run_metarl(env, envs, tasks, seed, log_dir):
"""Create metarl Tensorflow PPO model and training.
Args:
env (dict): Environment of the task.
seed (int): Random positive integer for the trial.
log_dir (str): Log dir path.
Returns:
str: Path to output csv file
"""
deterministic.set_seed(seed)
snapshot_config = SnapshotConfig(snapshot_dir=log_dir,
snapshot_mode='gap',
snapshot_gap=10)
with LocalTFRunner(snapshot_config) as runner:
policy = GaussianGRUPolicy(
hidden_dims=hyper_parameters['hidden_sizes'],
env_spec=env.spec,
state_include_action=False)
baseline = MetaRLLinearFeatureBaseline(env_spec=env.spec)
inner_algo = RL2TRPO(
env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=hyper_parameters['max_path_length'] * hyper_parameters['rollout_per_task'],
discount=hyper_parameters['discount'],
gae_lambda=hyper_parameters['gae_lambda'],
lr_clip_range=hyper_parameters['lr_clip_range'],
max_kl_step=0.01,
optimizer=ConjugateGradientOptimizer,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5))
)
# Need to pass this if meta_batch_size < num_of_tasks
task_names = list(ML10_ENVS['train'].keys())
algo = RL2(
policy=policy,
inner_algo=inner_algo,
max_path_length=hyper_parameters['max_path_length'],
meta_batch_size=hyper_parameters['meta_batch_size'],
task_sampler=tasks,
task_names=None if hyper_parameters['meta_batch_size'] >= len(task_names) else task_names)
# Set up logger since we are not using run_experiment
tabular_log_file = osp.join(log_dir, 'progress.csv')
text_log_file = osp.join(log_dir, 'debug.log')
dowel_logger.add_output(dowel.TextOutput(text_log_file))
dowel_logger.add_output(dowel.CsvOutput(tabular_log_file))
dowel_logger.add_output(dowel.StdOutput())
dowel_logger.add_output(dowel.TensorBoardOutput(log_dir))
runner.setup(
algo,
envs,
sampler_cls=hyper_parameters['sampler_cls'],
n_workers=hyper_parameters['meta_batch_size'],
worker_class=RL2Worker,
sampler_args=dict(
use_all_workers=hyper_parameters['use_all_workers']),
worker_args=dict(
n_paths_per_trial=hyper_parameters['rollout_per_task']))
# meta evaluator
ML_test_envs = [
TaskIdWrapper(RL2Env(env(*ML10_ARGS['test'][task]['args'],
**ML10_ARGS['test'][task]['kwargs'])), task_id=task_id, task_name=task)
for (task_id, (task, env)) in enumerate(ML10_ENVS['test'].items())
]
test_tasks = task_sampler.EnvPoolSampler(ML_test_envs)
test_tasks.grow_pool(hyper_parameters['n_test_tasks'])
test_task_names = list(ML10_ENVS['test'].keys())
runner.setup_meta_evaluator(test_task_sampler=test_tasks,
n_exploration_traj=hyper_parameters['rollout_per_task'],
n_test_rollouts=hyper_parameters['test_rollout_per_task'],
n_test_tasks=hyper_parameters['n_test_tasks'],
n_workers=hyper_parameters['n_test_tasks'],
test_task_names=None if hyper_parameters['n_test_tasks'] >= len(test_task_names) else test_task_names)
runner.train(n_epochs=hyper_parameters['n_itr'],
batch_size=hyper_parameters['meta_batch_size'] * hyper_parameters['rollout_per_task'] * hyper_parameters['max_path_length'])
dowel_logger.remove_all()
return tabular_log_file