def test_benchmark_maml(self, _): # pylint: disable=no-self-use
"""Compare benchmarks between metarl and baselines."""
timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')
benchmark_dir = './data/local/benchmarks/maml-ml1-push/%s/' % timestamp
result_json = {}
env_id = 'ML1-Push'
meta_env = TaskIdWrapper2(ML1WithPinnedGoal.get_train_tasks('push-v1'))
seeds = random.sample(range(100), hyper_parameters['n_trials'])
task_dir = osp.join(benchmark_dir, env_id)
plt_file = osp.join(benchmark_dir, '{}_benchmark.png'.format(env_id))
promp_csvs = []
metarl_csvs = []
for trial in range(hyper_parameters['n_trials']):
seed = seeds[trial]
trial_dir = task_dir + '/trial_%d_seed_%d' % (trial + 1, seed)
metarl_dir = trial_dir + '/metarl'
promp_dir = trial_dir + '/promp'
if test_metarl:
# Run metarl algorithm
env = MetaRLEnv(normalize(meta_env, expected_action_scale=10.))
metarl_csv = run_metarl(env, seed, metarl_dir)
metarl_csvs.append(metarl_csv)
env.close()
class TestPPO:
"""Test class for PPO."""
def setup_method(self):
"""Setup method which is called before every test."""
self.env = MetaRLEnv(normalize(gym.make('InvertedDoublePendulum-v2')))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.baseline = LinearFeatureBaseline(env_spec=self.env.spec)
def teardown_method(self):
"""Teardown method which is called after every test."""
self.env.close()
def test_ppo_pendulum(self):
"""Test PPO with Pendulum environment."""
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
algo = PPO(env_spec=self.env.spec,
policy=self.policy,
baseline=self.baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.97,
lr_clip_range=2e-1)
runner.setup(algo, self.env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 0
def test_benchmark_sac(self):
'''
Compare benchmarks between metarl and baselines.
:return:
'''
mujoco1m = benchmarks.get_benchmark('Mujoco1M')
timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')
benchmark_dir = osp.join(os.getcwd(), 'data', 'local', 'benchmarks',
'sac', timestamp)
mujoco_tasks = ['HalfCheetah-v2']
for task in mujoco_tasks:
env = MetaRLEnv(normalize(gym.make(task)))
seeds = [121, 524, 4]
task_dir = osp.join(benchmark_dir, task)
plt_file = osp.join(benchmark_dir, '{}_benchmark.png'.format(task))
relplt_file = osp.join(benchmark_dir,
'{}_benchmark_mean.png'.format(task))
metarl_csvs = []
for trial in range(3):
env.reset()
seed = seeds[trial]
trial_dir = osp.join(
task_dir, 'trial_{}_seed_{}'.format(trial + 1, seed))
metarl_dir = osp.join(trial_dir, 'metarl')
# Run metarl algorithms
metarl_csv = run_metarl(env, seed, metarl_dir)
metarl_csvs.append(metarl_csv)
env.close()
class TestMAML:
"""Test class for MAML."""
def setup_method(self):
"""Setup method which is called before every test."""
self.env = MetaRLEnv(
normalize(HalfCheetahDirEnv(), expected_action_scale=10.))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.baseline = LinearFeatureBaseline(env_spec=self.env.spec)
self.algo = MAMLPPO(env=self.env,
policy=self.policy,
baseline=self.baseline,
max_path_length=100,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
def teardown_method(self):
"""Teardown method which is called after every test."""
self.env.close()
def test_get_exploration_policy(self, set_params, test_params):
"""Test if an independent copy of policy is returned."""
self.policy.apply(partial(set_params, 0.1))
adapt_policy = self.algo.get_exploration_policy()
adapt_policy.apply(partial(set_params, 0.2))
# Old policy should remain untouched
self.policy.apply(partial(test_params, 0.1))
adapt_policy.apply(partial(test_params, 0.2))
def test_adapt_policy(self, set_params, test_params):
"""Test if policy can adapt to samples."""
worker = WorkerFactory(seed=100, max_path_length=100)
sampler = LocalSampler.from_worker_factory(worker, self.policy,
self.env)
self.policy.apply(partial(set_params, 0.1))
adapt_policy = self.algo.get_exploration_policy()
trajs = sampler.obtain_samples(0, 100, adapt_policy)
self.algo.adapt_policy(adapt_policy, trajs)
# Old policy should remain untouched
self.policy.apply(partial(test_params, 0.1))
# Adapted policy should not be identical to old policy
for v1, v2 in zip(adapt_policy.parameters(), self.policy.parameters()):
if v1.data.ne(v2.data).sum() > 0:
break
else:
pytest.fail("Parameters of adapted policy should not be "
"identical to the old policy.")
class TestMAMLPPO:
"""Test class for MAML-PPO."""
def setup_method(self):
"""Setup method which is called before every test."""
self.env = MetaRLEnv(
normalize(HalfCheetahDirEnv(), expected_action_scale=10.))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.baseline = LinearFeatureBaseline(env_spec=self.env.spec)
def teardown_method(self):
"""Teardown method which is called after every test."""
self.env.close()
def test_ppo_pendulum(self):
"""Test PPO with Pendulum environment."""
deterministic.set_seed(0)
rollouts_per_task = 5
max_path_length = 100
runner = LocalRunner(snapshot_config)
algo = MAMLPPO(env=self.env,
policy=self.policy,
baseline=self.baseline,
max_path_length=max_path_length,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
runner.setup(algo, self.env)
last_avg_ret = runner.train(n_epochs=10,
batch_size=rollouts_per_task *
max_path_length)
assert last_avg_ret > -5
def test_ddpg_pendulum(self):
"""Test DDPG with Pendulum environment.
This environment has a [-3, 3] action_space bound.
"""
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
env = MetaRLEnv(normalize(gym.make('InvertedPendulum-v2')))
action_noise = OUStrategy(env.spec, sigma=0.2)
policy = DeterministicMLPPolicy(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=F.relu,
output_nonlinearity=torch.tanh)
qf = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=F.relu)
replay_buffer = SimpleReplayBuffer(env_spec=env.spec,
size_in_transitions=int(1e6),
time_horizon=100)
algo = DDPG(env_spec=env.spec,
policy=policy,
qf=qf,
replay_buffer=replay_buffer,
steps_per_epoch=20,
n_train_steps=50,
min_buffer_size=int(1e4),
exploration_strategy=action_noise,
target_update_tau=1e-2,
discount=0.9)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 10
env.close()
class TestVPG:
"""Test class for VPG."""
@classmethod
def setup_class(cls):
"""Setup method which is called once before all tests in this class."""
deterministic.set_seed(0)
def setup_method(self):
"""Setup method which is called before every test."""
self._env = MetaRLEnv(gym.make('InvertedDoublePendulum-v2'))
self._runner = LocalRunner(snapshot_config)
self._policy = GaussianMLPPolicy(env_spec=self._env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
self._params = {
'env_spec': self._env.spec,
'policy': self._policy,
'baseline': LinearFeatureBaseline(env_spec=self._env.spec),
'max_path_length': 100,
'discount': 0.99,
}
def teardown_method(self):
"""Teardown method which is called after every test."""
self._env.close()
def test_vpg_no_entropy(self):
"""Test VPG with no_entropy."""
self._params['positive_adv'] = True
self._params['use_softplus_entropy'] = True
algo = VPG(**self._params)
self._runner.setup(algo, self._env)
last_avg_ret = self._runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 0
def test_vpg_max(self):
"""Test VPG with maximum entropy."""
self._params['center_adv'] = False
self._params['stop_entropy_gradient'] = True
self._params['entropy_method'] = 'max'
algo = VPG(**self._params)
self._runner.setup(algo, self._env)
last_avg_ret = self._runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 0
def test_vpg_regularized(self):
"""Test VPG with entropy_regularized."""
self._params['entropy_method'] = 'regularized'
algo = VPG(**self._params)
self._runner.setup(algo, self._env)
last_avg_ret = self._runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 0
@pytest.mark.parametrize('algo_param, error, msg', INVALID_ENTROPY_CONFIG)
def test_invalid_entropy_config(self, algo_param, error, msg):
"""Test VPG with invalid entropy config."""
self._params.update(algo_param)
with pytest.raises(error, match=msg):
VPG(**self._params)