def test_maml_trpo_pendulum():
"""Test PPO with Pendulum environment."""
env = MetaRLEnv(normalize(HalfCheetahDirEnv(), expected_action_scale=10.))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32))
rollouts_per_task = 5
max_path_length = 100
runner = LocalRunner(snapshot_config)
algo = MAMLTRPO(env=env,
policy=policy,
value_function=value_function,
max_path_length=max_path_length,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=5,
batch_size=rollouts_per_task * max_path_length)
assert last_avg_ret > -5
env.close()
def test_ppo_pendulum(self):
"""Test PPO with Pendulum environment."""
deterministic.set_seed(0)
rollouts_per_task = 5
max_path_length = 100
task_sampler = SetTaskSampler(lambda: MetaRLEnv(
normalize(HalfCheetahDirEnv(), expected_action_scale=10.)))
meta_evaluator = MetaEvaluator(test_task_sampler=task_sampler,
max_path_length=max_path_length,
n_test_tasks=1,
n_test_rollouts=10)
runner = LocalRunner(snapshot_config)
algo = MAMLVPG(env=self.env,
policy=self.policy,
value_function=self.value_function,
max_path_length=max_path_length,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1,
meta_evaluator=meta_evaluator)
runner.setup(algo, self.env)
last_avg_ret = runner.train(n_epochs=10,
batch_size=rollouts_per_task *
max_path_length)
assert last_avg_ret > -5
def test_sac_inverted_double_pendulum():
"""Test Sac performance on inverted pendulum."""
# pylint: disable=unexpected-keyword-arg
env = MetaRLEnv(normalize(gym.make('InvertedDoublePendulum-v2')))
deterministic.set_seed(0)
policy = TanhGaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.nn.ReLU,
output_nonlinearity=None,
min_std=np.exp(-20.),
max_std=np.exp(2.),
)
qf1 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=F.relu)
qf2 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=F.relu)
replay_buffer = PathBuffer(capacity_in_transitions=int(1e6), )
runner = LocalRunner(snapshot_config=snapshot_config)
sac = SAC(env_spec=env.spec,
policy=policy,
qf1=qf1,
qf2=qf2,
gradient_steps_per_itr=100,
max_path_length=100,
replay_buffer=replay_buffer,
min_buffer_size=1e3,
target_update_tau=5e-3,
discount=0.99,
buffer_batch_size=64,
reward_scale=1.,
steps_per_epoch=2)
runner.setup(sac, env, sampler_cls=LocalSampler)
if torch.cuda.is_available():
set_gpu_mode(True)
else:
set_gpu_mode(False)
sac.to()
ret = runner.train(n_epochs=12, batch_size=200, plot=False)
# check that automatic entropy tuning is used
assert sac._use_automatic_entropy_tuning
# assert that there was a gradient properly connected to alpha
# this doesn't verify that the path from the temperature objective is
# correct.
assert not torch.allclose(torch.Tensor([1.]), sac._log_alpha.to('cpu'))
# check that policy is learning beyond predecided threshold
assert ret > 85
def test_trpo_pendulum(self):
"""Test TRPO with Pendulum environment."""
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
algo = TRPO(env_spec=self.env.spec,
policy=self.policy,
baseline=self.baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.98)
runner.setup(algo, self.env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 50
def setup_method(self):
"""Setup method which is called before every test."""
self._env = MetaRLEnv(gym.make('InvertedDoublePendulum-v2'))
self._runner = LocalRunner(snapshot_config)
self._policy = GaussianMLPPolicy(env_spec=self._env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
self._params = {
'env_spec': self._env.spec,
'policy': self._policy,
'baseline': LinearFeatureBaseline(env_spec=self._env.spec),
'max_path_length': 100,
'discount': 0.99,
}
def test_ppo_pendulum(self):
"""Test PPO with Pendulum environment."""
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
algo = PPO(env_spec=self.env.spec,
policy=self.policy,
value_function=self.value_function,
max_path_length=100,
discount=0.99,
gae_lambda=0.97,
lr_clip_range=2e-1)
runner.setup(algo, self.env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 0
def test_mtsac_inverted_double_pendulum():
"""Performance regression test of MTSAC on 2 InvDoublePendulum envs."""
env_names = ['InvertedDoublePendulum-v2', 'InvertedDoublePendulum-v2']
task_envs = [MetaRLEnv(env_name=name) for name in env_names]
env = MultiEnvWrapper(task_envs, sample_strategy=round_robin_strategy)
test_envs = MultiEnvWrapper(task_envs,
sample_strategy=round_robin_strategy)
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config=snapshot_config)
policy = TanhGaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.nn.ReLU,
output_nonlinearity=None,
min_std=np.exp(-20.),
max_std=np.exp(2.),
)
qf1 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=F.relu)
qf2 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=F.relu)
replay_buffer = PathBuffer(capacity_in_transitions=int(1e6), )
num_tasks = 2
buffer_batch_size = 128
mtsac = MTSAC(policy=policy,
qf1=qf1,
qf2=qf2,
gradient_steps_per_itr=100,
max_path_length=100,
eval_env=test_envs,
env_spec=env.spec,
num_tasks=num_tasks,
steps_per_epoch=5,
replay_buffer=replay_buffer,
min_buffer_size=1e3,
target_update_tau=5e-3,
discount=0.99,
buffer_batch_size=buffer_batch_size)
runner.setup(mtsac, env, sampler_cls=LocalSampler)
ret = runner.train(n_epochs=8, batch_size=128, plot=False)
assert ret > 130
def test_ddpg_pendulum(self):
"""Test DDPG with Pendulum environment.
This environment has a [-3, 3] action_space bound.
"""
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
env = MetaRLEnv(normalize(gym.make('InvertedPendulum-v2')))
policy = DeterministicMLPPolicy(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=F.relu,
output_nonlinearity=torch.tanh)
exploration_policy = AddOrnsteinUhlenbeckNoise(env.spec,
policy,
sigma=0.2)
qf = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=F.relu)
replay_buffer = PathBuffer(capacity_in_transitions=int(1e6))
algo = DDPG(env_spec=env.spec,
policy=policy,
qf=qf,
replay_buffer=replay_buffer,
steps_per_epoch=20,
n_train_steps=50,
min_buffer_size=int(1e4),
exploration_policy=exploration_policy,
target_update_tau=1e-2,
discount=0.9)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 10
env.close()
def test_setup_no_sampler_cls():
runner = LocalRunner(snapshot_config)
algo = CrashingAlgo()
algo.max_path_length = 100
runner.setup(algo, None)
with pytest.raises(ValueError, match='sampler_cls'):
runner.train(n_epochs=5)
def run_task(snapshot_config, *_):
"""Set up environment and algorithm and run the task.
Args:
snapshot_config (metarl.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
If None, it will create one with default settings.
_ : Unused parameters
"""
env = TfEnv(env_name='InvertedDoublePendulum-v2')
runner = LocalRunner(snapshot_config)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
center_adv=False)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=1024)
def test_maml_trpo_dummy_named_env():
"""Test with dummy environment that has env_name."""
env = MetaRLEnv(
normalize(DummyMultiTaskBoxEnv(), expected_action_scale=10.))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32))
rollouts_per_task = 2
max_path_length = 100
runner = LocalRunner(snapshot_config)
algo = MAMLTRPO(env=env,
policy=policy,
value_function=value_function,
max_path_length=max_path_length,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
runner.setup(algo, env)
runner.train(n_epochs=2, batch_size=rollouts_per_task * max_path_length)
def maml_trpo_metaworld_ml10(ctxt, seed, epochs, rollouts_per_task,
meta_batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
rollouts_per_task (int): Number of rollouts per epoch per task
for training.
meta_batch_size (int): Number of tasks sampled per batch.
"""
set_seed(seed)
env = MetaRLEnv(
normalize(mwb.ML10.get_train_tasks(), expected_action_scale=10.))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(100, 100),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
max_path_length = 100
test_task_names = mwb.ML10.get_test_tasks().all_task_names
test_tasks = [
MetaRLEnv(
normalize(mwb.ML10.from_task(task), expected_action_scale=10.))
for task in test_task_names
]
test_sampler = EnvPoolSampler(test_tasks)
meta_evaluator = MetaEvaluator(test_task_sampler=test_sampler,
max_path_length=max_path_length,
n_test_tasks=len(test_task_names))
runner = LocalRunner(ctxt)
algo = MAMLTRPO(env=env,
policy=policy,
value_function=value_function,
max_path_length=max_path_length,
meta_batch_size=meta_batch_size,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1,
meta_evaluator=meta_evaluator)
runner.setup(algo, env)
runner.train(n_epochs=epochs,
batch_size=rollouts_per_task * max_path_length)
def ppo_pendulum(ctxt=None, seed=1):
"""Train PPO with InvertedDoublePendulum-v2 environment.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
env = MetaRLEnv(env_name='InvertedDoublePendulum-v2')
runner = LocalRunner(ctxt)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = PPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
max_path_length=100,
discount=0.99,
center_adv=False)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=10000)
def test_one_folder(self, meta_train_dir, itrs):
snapshot_config = SnapshotConfig(snapshot_dir=meta_train_dir,
snapshot_mode='all',
snapshot_gap=1)
runner = LocalRunner(snapshot_config=snapshot_config)
meta_sampler = AllSetTaskSampler(self.meta_task_cls)
runner.restore(meta_train_dir)
meta_evaluator = MetaEvaluator(
runner,
test_task_sampler=meta_sampler,
max_path_length=self.max_path_length,
n_test_tasks=meta_sampler.n_tasks,
n_exploration_traj=self.adapt_rollout_per_task,
prefix='')
for itr in itrs:
log_filename = os.path.join(meta_train_dir,
'meta-test-itr_{}.csv'.format(itr))
logger.add_output(CsvOutput(log_filename))
logger.log("Writing into {}".format(log_filename))
runner.restore(meta_train_dir, from_epoch=itr)
meta_evaluator.evaluate(runner._algo, self.test_rollout_per_task)
tabular.record('Iteration', runner._stats.total_epoch)
tabular.record('TotalEnvSteps', runner._stats.total_env_steps)
logger.log(tabular)
logger.dump_output_type(CsvOutput)
logger.remove_output_type(CsvOutput)
def run_metarl(env, seed, log_dir):
'''
Create metarl model and training.
Replace the ddpg with the algorithm you want to run.
:param env: Environment of the task.
:param seed: Random seed for the trial.
:param log_dir: Log dir path.
:return:
'''
deterministic.set_seed(seed)
runner = LocalRunner(snapshot_config)
# Set up params for ddpg
policy = TanhGaussianMLPPolicy2(env_spec=env.spec,
hidden_sizes=params['policy_hidden_sizes'],
hidden_nonlinearity=nn.ReLU,
output_nonlinearity=None)
qf1 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=params['qf_hidden_sizes'],
hidden_nonlinearity=F.relu)
qf2 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=params['qf_hidden_sizes'],
hidden_nonlinearity=F.relu)
replay_buffer = SACReplayBuffer(env_spec=env.spec,
max_size=params['replay_buffer_size'])
sampler_args = {
'agent': policy,
'max_path_length': 1000,
}
sac = SAC(env_spec=env.spec,
policy=policy,
qf1=qf1,
qf2=qf2,
gradient_steps_per_itr=params['gradient_steps_per_itr'],
replay_buffer=replay_buffer,
buffer_batch_size=params['buffer_batch_size'])
# Set up logger since we are not using run_experiment
tabular_log_file = osp.join(log_dir, 'progress.csv')
tensorboard_log_dir = osp.join(log_dir)
dowel_logger.add_output(dowel.StdOutput())
dowel_logger.add_output(dowel.CsvOutput(tabular_log_file))
dowel_logger.add_output(dowel.TensorBoardOutput(tensorboard_log_dir))
runner.setup(algo=sac,
env=env,
sampler_cls=SimpleSampler,
sampler_args=sampler_args)
runner.train(n_epochs=params['n_epochs'],
batch_size=params['gradient_steps_per_itr'])
dowel_logger.remove_all()
return tabular_log_file
def test_setup_no_batch_size():
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
algo = CrashingAlgo()
algo.max_path_length = 100
algo.policy = None
runner.setup(algo, None, sampler_cls=LocalSampler)
with pytest.raises(ValueError, match='batch_size'):
runner.train(n_epochs=5)
def ml1_push_v1_sac(ctxt=None, seed=1):
"""Set up environment and algorithm and run the task."""
runner = LocalRunner(ctxt)
Ml1_reach_envs = get_ML1_envs("push-v1")
Ml1_reach_test_envs = get_ML1_envs_test("push-v1")
env = MTMetaWorldWrapper(Ml1_reach_envs)
policy = TanhGaussianMLPPolicy2(
env_spec=env.spec,
hidden_sizes=[400, 400, 400],
hidden_nonlinearity=nn.ReLU,
output_nonlinearity=None,
min_std=np.exp(-20.),
max_std=np.exp(2.),
)
qf1 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[400, 400, 400],
hidden_nonlinearity=F.relu)
qf2 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[400, 400, 400],
hidden_nonlinearity=F.relu)
replay_buffer = SACReplayBuffer(env_spec=env.spec, max_size=int(1e6))
sampler_args = {'agent': policy, 'max_path_length': 150}
timesteps = 100000000
batch_size = int(150 * env.num_tasks)
num_evaluation_points = 500
epochs = timesteps // batch_size
epoch_cycles = epochs // num_evaluation_points
epochs = epochs // epoch_cycles
sac = MTSAC(env=env,
eval_env_dict=Ml1_reach_test_envs,
env_spec=env.spec,
policy=policy,
qf1=qf1,
qf2=qf2,
gradient_steps_per_itr=250,
epoch_cycles=epoch_cycles,
use_automatic_entropy_tuning=True,
replay_buffer=replay_buffer,
min_buffer_size=7500,
target_update_tau=5e-3,
discount=0.99,
buffer_batch_size=6400)
tu.set_gpu_mode(True)
sac.to('cuda:0')
runner.setup(algo=sac,
env=env,
sampler_cls=SimpleSampler,
sampler_args=sampler_args)
runner.train(n_epochs=epochs, batch_size=batch_size)
def test_ppo_pendulum(self):
"""Test PPO with Pendulum environment."""
deterministic.set_seed(0)
rollouts_per_task = 5
max_path_length = 100
runner = LocalRunner(snapshot_config)
algo = MAMLTRPO(env=self.env,
policy=self.policy,
baseline=self.baseline,
max_path_length=max_path_length,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
runner.setup(algo, self.env)
last_avg_ret = runner.train(n_epochs=5,
batch_size=rollouts_per_task *
max_path_length)
assert last_avg_ret > -5
def test_ddpg_double_pendulum(self):
"""Test DDPG with Pendulum environment."""
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
env = MetaRLEnv(gym.make('InvertedDoublePendulum-v2'))
action_noise = OUStrategy(env.spec, sigma=0.2)
policy = DeterministicMLPPolicy(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=F.relu,
output_nonlinearity=torch.tanh)
qf = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=F.relu)
replay_buffer = SimpleReplayBuffer(env_spec=env.spec,
size_in_transitions=int(1e6),
time_horizon=100)
algo = DDPG(env_spec=env.spec,
policy=policy,
qf=qf,
replay_buffer=replay_buffer,
steps_per_epoch=20,
n_train_steps=50,
min_buffer_size=int(1e4),
exploration_strategy=action_noise,
target_update_tau=1e-2,
discount=0.9)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 45
env.close()
def test_fixed_alpha():
"""Test if using fixed_alpha ensures that alpha is non differentiable."""
env_names = ['InvertedDoublePendulum-v2', 'InvertedDoublePendulum-v2']
task_envs = [MetaRLEnv(env_name=name) for name in env_names]
env = MultiEnvWrapper(task_envs, sample_strategy=round_robin_strategy)
test_envs = MultiEnvWrapper(task_envs,
sample_strategy=round_robin_strategy)
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config=snapshot_config)
policy = TanhGaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.nn.ReLU,
output_nonlinearity=None,
min_std=np.exp(-20.),
max_std=np.exp(2.),
)
qf1 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=F.relu)
qf2 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=F.relu)
replay_buffer = PathBuffer(capacity_in_transitions=int(1e6), )
num_tasks = 2
buffer_batch_size = 128
mtsac = MTSAC(policy=policy,
qf1=qf1,
qf2=qf2,
gradient_steps_per_itr=100,
max_path_length=100,
eval_env=test_envs,
env_spec=env.spec,
num_tasks=num_tasks,
steps_per_epoch=1,
replay_buffer=replay_buffer,
min_buffer_size=1e3,
target_update_tau=5e-3,
discount=0.99,
buffer_batch_size=buffer_batch_size,
fixed_alpha=np.exp(0.5))
if torch.cuda.is_available():
set_gpu_mode(True)
else:
set_gpu_mode(False)
mtsac.to()
assert torch.allclose(torch.Tensor([0.5] * num_tasks),
mtsac._log_alpha.to('cpu'))
runner.setup(mtsac, env, sampler_cls=LocalSampler)
runner.train(n_epochs=1, batch_size=128, plot=False)
assert torch.allclose(torch.Tensor([0.5] * num_tasks),
mtsac._log_alpha.to('cpu'))
assert not mtsac._use_automatic_entropy_tuning
def maml_vpg_half_cheetah_dir(ctxt, seed, epochs, rollouts_per_task,
meta_batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
rollouts_per_task (int): Number of rollouts per epoch per task
for training.
meta_batch_size (int): Number of tasks sampled per batch.
"""
set_seed(seed)
env = MetaRLEnv(normalize(HalfCheetahDirEnv(), expected_action_scale=10.))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
max_path_length = 100
task_sampler = SetTaskSampler(lambda: MetaRLEnv(
normalize(HalfCheetahDirEnv(), expected_action_scale=10.)))
meta_evaluator = MetaEvaluator(test_task_sampler=task_sampler,
max_path_length=max_path_length,
n_test_tasks=1,
n_test_rollouts=10)
runner = LocalRunner(ctxt)
algo = MAMLVPG(env=env,
policy=policy,
value_function=value_function,
max_path_length=max_path_length,
meta_batch_size=meta_batch_size,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1,
meta_evaluator=meta_evaluator)
runner.setup(algo, env)
runner.train(n_epochs=epochs,
batch_size=rollouts_per_task * max_path_length)
def sac_half_cheetah_batch(ctxt=None, seed=1):
"""Set up environment and algorithm and run the task.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
deterministic.set_seed(seed)
runner = LocalRunner(snapshot_config=ctxt)
env = MetaRLEnv(normalize(gym.make('HalfCheetah-v2')))
policy = TanhGaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=[256, 256],
hidden_nonlinearity=nn.ReLU,
output_nonlinearity=None,
min_std=np.exp(-20.),
max_std=np.exp(2.),
)
qf1 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[256, 256],
hidden_nonlinearity=F.relu)
qf2 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[256, 256],
hidden_nonlinearity=F.relu)
replay_buffer = PathBuffer(capacity_in_transitions=int(1e6))
sac = SAC(env_spec=env.spec,
policy=policy,
qf1=qf1,
qf2=qf2,
gradient_steps_per_itr=1000,
max_path_length=500,
replay_buffer=replay_buffer,
min_buffer_size=1e4,
target_update_tau=5e-3,
discount=0.99,
buffer_batch_size=256,
reward_scale=1.,
steps_per_epoch=1)
if torch.cuda.is_available():
set_gpu_mode(True)
else:
set_gpu_mode(False)
sac.to()
runner.setup(algo=sac, env=env, sampler_cls=LocalSampler)
runner.train(n_epochs=1000, batch_size=1000)
def run_metarl(env, seed, log_dir):
"""Create metarl PyTorch MAML model and training.
Args:
env (MetaRLEnv): Environment of the task.
seed (int): Random positive integer for the trial.
log_dir (str): Log dir path.
Returns:
str: Path to output csv file
"""
deterministic.set_seed(seed)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=hyper_parameters['hidden_sizes'],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = MAMLTRPO(env=env,
policy=policy,
baseline=baseline,
max_path_length=hyper_parameters['max_path_length'],
discount=hyper_parameters['discount'],
gae_lambda=hyper_parameters['gae_lambda'],
meta_batch_size=hyper_parameters['meta_batch_size'],
inner_lr=hyper_parameters['inner_lr'],
max_kl_step=hyper_parameters['max_kl'],
num_grad_updates=hyper_parameters['num_grad_update'])
# Set up logger since we are not using run_experiment
tabular_log_file = osp.join(log_dir, 'progress.csv')
dowel_logger.add_output(dowel.StdOutput())
dowel_logger.add_output(dowel.CsvOutput(tabular_log_file))
dowel_logger.add_output(dowel.TensorBoardOutput(log_dir))
snapshot_config = SnapshotConfig(snapshot_dir=log_dir,
snapshot_mode='all',
snapshot_gap=1)
runner = LocalRunner(snapshot_config=snapshot_config)
runner.setup(algo, env, sampler_args=dict(n_envs=5))
runner.train(n_epochs=hyper_parameters['n_epochs'],
batch_size=(hyper_parameters['fast_batch_size'] *
hyper_parameters['max_path_length']))
dowel_logger.remove_all()
return tabular_log_file
def resume_experiment(ctxt, saved_dir):
"""Resume a PyTorch experiment.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
saved_dir (str): Path where snapshots are saved.
"""
runner = LocalRunner(snapshot_config=ctxt)
runner.restore(from_dir=saved_dir)
runner.resume()
def test_setup_no_sampler():
runner = LocalRunner(snapshot_config)
class SupervisedAlgo:
def train(self, runner):
# pylint: disable=undefined-loop-variable
for epoch in runner.step_epochs():
pass
assert epoch == 4
runner.setup(SupervisedAlgo(), None)
runner.train(n_epochs=5)
def ppo_metarl_pytorch(ctxt, env_id, seed):
"""Create metarl PyTorch PPO model and training.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the
snapshotter.
env_id (str): Environment id of the task.
seed (int): Random positive integer for the trial.
"""
deterministic.set_seed(seed)
runner = LocalRunner(ctxt)
env = MetaRLEnv(normalize(gym.make(env_id)))
policy = PyTorch_GMP(env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
policy_optimizer = OptimizerWrapper((torch.optim.Adam, dict(lr=2.5e-4)),
policy,
max_optimization_epochs=10,
minibatch_size=64)
vf_optimizer = OptimizerWrapper((torch.optim.Adam, dict(lr=2.5e-4)),
value_function,
max_optimization_epochs=10,
minibatch_size=64)
algo = PyTorch_PPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
policy_optimizer=policy_optimizer,
vf_optimizer=vf_optimizer,
max_path_length=hyper_parameters['max_path_length'],
discount=0.99,
gae_lambda=0.95,
center_adv=True,
lr_clip_range=0.2)
runner.setup(algo, env)
runner.train(n_epochs=hyper_parameters['n_epochs'],
batch_size=hyper_parameters['batch_size'])
def run_task(snapshot_config, *_):
"""Set up environment and algorithm and run the task."""
runner = LocalRunner(snapshot_config)
env = MetaRLEnv(normalize(gym.make('HalfCheetah-v2')))
policy = TanhGaussianMLPPolicy2(
env_spec=env.spec,
hidden_sizes=[256, 256],
hidden_nonlinearity=nn.ReLU,
output_nonlinearity=None,
min_std=np.exp(-20.),
max_std=np.exp(2.),
)
qf1 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[256, 256],
hidden_nonlinearity=F.relu)
qf2 = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[256, 256],
hidden_nonlinearity=F.relu)
replay_buffer = SimpleReplayBuffer(env_spec=env.spec,
size_in_transitions=int(1e6),
time_horizon=1)
# replay_buffer = SACReplayBuffer(env_spec=env.spec,
# max_size=int(1e6))
sampler_args = {
'agent': policy,
'max_path_length': 1000,
}
sac = SAC(env_spec=env.spec,
policy=policy,
qf1=qf1,
qf2=qf2,
gradient_steps_per_itr=1000,
use_automatic_entropy_tuning=True,
replay_buffer=replay_buffer,
min_buffer_size=1e4,
target_update_tau=5e-3,
discount=0.99,
buffer_batch_size=256,
reward_scale=1.)
runner.setup(algo=sac,
env=env,
sampler_cls=SimpleSampler,
sampler_args=sampler_args)
runner.train(n_epochs=1000, batch_size=1000)
def run_metarl_pytorch(env, seed, log_dir):
"""Create metarl PyTorch PPO model and training.
Args:
env (dict): Environment of the task.
seed (int): Random positive integer for the trial.
log_dir (str): Log dir path.
Returns:
str: Path to output csv file
"""
env = TfEnv(normalize(env))
deterministic.set_seed(seed)
runner = LocalRunner(snapshot_config)
policy = PyTorch_GMP(env.spec,
hidden_sizes=hyper_parameters['hidden_sizes'],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = PyTorch_PPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
optimizer=torch.optim.Adam,
policy_lr=hyper_parameters['learning_rate'],
max_path_length=hyper_parameters['max_path_length'],
discount=hyper_parameters['discount'],
gae_lambda=hyper_parameters['gae_lambda'],
center_adv=hyper_parameters['center_adv'],
lr_clip_range=hyper_parameters['lr_clip_range'])
# Set up logger since we are not using run_experiment
tabular_log_file = osp.join(log_dir, 'progress.csv')
dowel_logger.add_output(dowel.StdOutput())
dowel_logger.add_output(dowel.CsvOutput(tabular_log_file))
dowel_logger.add_output(dowel.TensorBoardOutput(log_dir))
runner.setup(algo, env)
runner.train(n_epochs=hyper_parameters['n_epochs'],
batch_size=hyper_parameters['batch_size'])
dowel_logger.remove_all()
return tabular_log_file
def run_task(snapshot_config, *_):
"""Set up environment and algorithm and run the task.
Args:
snapshot_config (metarl.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
If None, it will create one with default settings.
_ : Unused parameters
"""
runner = LocalRunner(snapshot_config)
env = MetaRLEnv(normalize(gym.make('InvertedDoublePendulum-v2')))
action_noise = OUStrategy(env.spec, sigma=0.2)
policy = DeterministicMLPPolicy(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=F.relu,
output_nonlinearity=torch.tanh)
qf = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=F.relu)
replay_buffer = SimpleReplayBuffer(env_spec=env.spec,
size_in_transitions=int(1e6),
time_horizon=100)
policy_optimizer = (torch.optim.Adagrad, {'lr': 1e-4, 'lr_decay': 0.99})
ddpg = DDPG(env_spec=env.spec,
policy=policy,
qf=qf,
replay_buffer=replay_buffer,
steps_per_epoch=20,
n_train_steps=50,
min_buffer_size=int(1e4),
exploration_strategy=action_noise,
target_update_tau=1e-2,
discount=0.9,
policy_optimizer=policy_optimizer,
qf_optimizer=torch.optim.Adam)
runner.setup(algo=ddpg, env=env)
runner.train(n_epochs=500, batch_size=100)
def mtppo_metaworld_mt10(ctxt, seed, epochs, batch_size, n_worker):
"""Set up environment and algorithm and run the task.
Args:
ctxt (metarl.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
batch_size (int): Number of environment steps in one batch.
n_worker (int): The number of workers the sampler should use.
"""
set_seed(seed)
tasks = mwb.MT10.get_train_tasks().all_task_names
envs = []
for task in tasks:
envs.append(normalize(MetaRLEnv(mwb.MT10.from_task(task))))
env = MultiEnvWrapper(envs,
sample_strategy=round_robin_strategy,
mode='vanilla')
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = PPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
max_path_length=128,
discount=0.99,
gae_lambda=0.95,
center_adv=True,
lr_clip_range=0.2)
runner = LocalRunner(ctxt)
runner.setup(algo, env, n_workers=n_worker)
runner.train(n_epochs=epochs, batch_size=batch_size)
