def test_get_action_np(self, hidden_sizes):
"""Test get_action function with numpy inputs."""
env_spec = GarageEnv(DummyBoxEnv())
obs_dim = env_spec.observation_space.flat_dim
act_dim = env_spec.action_space.flat_dim
obs = np.ones(obs_dim, dtype=np.float32)
init_std = 2.
policy = GaussianMLPPolicy(env_spec=env_spec,
hidden_sizes=hidden_sizes,
init_std=init_std,
hidden_nonlinearity=None,
std_parameterization='exp',
hidden_w_init=nn.init.ones_,
output_w_init=nn.init.ones_)
dist = policy(torch.from_numpy(obs))[0]
expected_mean = torch.full(
(act_dim, ), obs_dim * (torch.Tensor(hidden_sizes).prod().item()))
expected_variance = init_std**2
action, prob = policy.get_action(obs)
assert np.array_equal(prob['mean'], expected_mean.numpy())
assert dist.variance.equal(torch.full((act_dim, ), expected_variance))
assert action.shape == (act_dim, )
def test_get_action(self, hidden_sizes):
env_spec = TfEnv(DummyBoxEnv())
obs_dim = env_spec.observation_space.flat_dim
act_dim = env_spec.action_space.flat_dim
obs = torch.ones(obs_dim, dtype=torch.float32)
init_std = 2.
policy = GaussianMLPPolicy(env_spec=env_spec,
hidden_sizes=hidden_sizes,
init_std=init_std,
hidden_nonlinearity=None,
std_parameterization='exp',
hidden_w_init=nn.init.ones_,
output_w_init=nn.init.ones_)
dist = policy(obs)
expected_mean = torch.full(
(act_dim, ), obs_dim * (torch.Tensor(hidden_sizes).prod().item()))
expected_variance = init_std**2
action, prob = policy.get_action(obs)
assert prob['mean'].equal(expected_mean)
assert dist.variance.equal(torch.full((act_dim, ), expected_variance))
assert action.shape == (act_dim, )
def setup_method(self):
"""Setup method which is called before every test."""
self.env = normalize(GymEnv(HalfCheetahDirEnv(),
max_episode_length=100),
expected_action_scale=10.)
task_sampler = SetTaskSampler(lambda: normalize(
GymEnv(HalfCheetahDirEnv()), expected_action_scale=10.))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.value_function = GaussianMLPValueFunction(env_spec=self.env.spec,
hidden_sizes=(32, 32))
self.algo = MAMLPPO(env=self.env,
policy=self.policy,
sampler=None,
task_sampler=task_sampler,
value_function=self.value_function,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
def test_get_actions(self, batch_size, hidden_sizes):
"""Test get_actions function."""
env_spec = GymEnv(DummyBoxEnv())
obs_dim = env_spec.observation_space.flat_dim
act_dim = env_spec.action_space.flat_dim
obs = torch.ones([batch_size, obs_dim], dtype=torch.float32)
init_std = 2.
policy = GaussianMLPPolicy(env_spec=env_spec,
hidden_sizes=hidden_sizes,
init_std=init_std,
hidden_nonlinearity=None,
std_parameterization='exp',
hidden_w_init=nn.init.ones_,
output_w_init=nn.init.ones_)
dist = policy(obs)[0]
expected_mean = torch.full([batch_size, act_dim],
obs_dim *
(torch.Tensor(hidden_sizes).prod().item()),
dtype=torch.float)
expected_variance = init_std**2
action, prob = policy.get_actions(obs)
assert np.array_equal(prob['mean'], expected_mean.numpy())
assert dist.variance.equal(
torch.full((batch_size, act_dim),
expected_variance,
dtype=torch.float))
assert action.shape == (batch_size, act_dim)
def test_entropy(self):
"""Test get_entropy method of the policy."""
env_spec = TfEnv(DummyBoxEnv())
init_std = 1.
obs = torch.Tensor([0, 0, 0, 0]).float()
policy = GaussianMLPPolicy(env_spec=env_spec,
hidden_sizes=(1, ),
init_std=init_std,
hidden_nonlinearity=None,
std_parameterization='exp',
hidden_w_init=nn.init.ones_,
output_w_init=nn.init.ones_)
dist = policy(obs)
assert torch.allclose(dist.entropy(), policy.entropy(obs))
def trpo_pendulum(ctxt=None, seed=1):
"""Train TRPO with InvertedDoublePendulum-v2 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
env = TfEnv(env_name='InvertedDoublePendulum-v2')
runner = LocalRunner(ctxt)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
max_path_length=100,
discount=0.99,
center_adv=False)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=1024)
def vpg_pendulum(ctxt=None, seed=1):
"""Train PPO with InvertedDoublePendulum-v2 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by Trainer to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
env = GymEnv('InvertedDoublePendulum-v2')
trainer = Trainer(ctxt)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = VPG(env_spec=env.spec,
policy=policy,
value_function=value_function,
discount=0.99,
center_adv=False)
trainer.setup(algo, env)
trainer.train(n_epochs=100, batch_size=10000)
def main(ctxt=None, seed=0):
env = GymEnv('Pendulum-v0')
print('here env')
experts = load_latest_experts(args.experts_dir, n=5)
print('here experts')
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
irl_model = GAIL(env_spec=env.spec, expert_trajs=experts)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
discount=0.99,
center_adv=False)
trainer = IRLTrainer(ctxt)
trainer.setup(algo,
env,
irl_model,
baseline,
n_itr=args.n_iter,
sampler_cls=MultiprocessingSampler,
zero_environment_reward=True)
trainer.train(n_epochs=1, batch_size=args.batch_size)
def maml_trpo_metaworld_ml1_push(ctxt, seed, epochs, rollouts_per_task,
meta_batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by Trainer to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
rollouts_per_task (int): Number of rollouts per epoch per task
for training.
meta_batch_size (int): Number of tasks sampled per batch.
"""
set_seed(seed)
ml1 = metaworld.ML1('push-v1')
tasks = MetaWorldTaskSampler(ml1, 'train')
env = tasks.sample(1)[0]()
test_sampler = SetTaskSampler(MetaWorldSetTaskEnv,
env=MetaWorldSetTaskEnv(ml1, 'test'))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(100, 100),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
meta_evaluator = MetaEvaluator(test_task_sampler=test_sampler,
n_test_tasks=1,
n_exploration_eps=rollouts_per_task)
sampler = RaySampler(agents=policy,
envs=env,
max_episode_length=env.spec.max_episode_length,
n_workers=meta_batch_size)
trainer = Trainer(ctxt)
algo = MAMLTRPO(env=env,
policy=policy,
sampler=sampler,
task_sampler=tasks,
value_function=value_function,
meta_batch_size=meta_batch_size,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1,
meta_evaluator=meta_evaluator)
trainer.setup(algo, env)
trainer.train(n_epochs=epochs,
batch_size=rollouts_per_task * env.spec.max_episode_length)
def bc_point(ctxt=None, loss='log_prob'):
"""Run Behavioral Cloning on garage.envs.PointEnv.
Args:
ctxt (ExperimentContext): Provided by wrap_experiment.
loss (str): Either 'log_prob' or 'mse'
"""
trainer = Trainer(ctxt)
goal = np.array([1., 1.])
env = PointEnv(goal=goal, max_episode_length=200)
expert = OptimalPolicy(env.spec, goal=goal)
policy = GaussianMLPPolicy(env.spec, [8, 8])
batch_size = 1000
sampler = RaySampler(agents=expert,
envs=env,
max_episode_length=env.spec.max_episode_length)
algo = BC(env.spec,
policy,
batch_size=batch_size,
source=expert,
sampler=sampler,
policy_lr=1e-2,
loss=loss)
trainer.setup(algo, env)
trainer.train(100, batch_size=batch_size)
def test_maml_trpo_dummy_named_env():
"""Test with dummy environment that has env_name."""
env = GarageEnv(
normalize(DummyMultiTaskBoxEnv(), expected_action_scale=10.))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32))
rollouts_per_task = 2
max_episode_length = 100
runner = LocalRunner(snapshot_config)
algo = MAMLTRPO(env=env,
policy=policy,
value_function=value_function,
max_episode_length=max_episode_length,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
runner.setup(algo, env, sampler_cls=LocalSampler)
runner.train(n_epochs=2, batch_size=rollouts_per_task * max_episode_length)
def ppo_setup(env, trainer, args):
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[args.hidden_dim] * args.depth,
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=[args.hidden_dim] *
args.depth,
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = PPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
policy_optimizer=OptimizerWrapper(
(torch.optim.Adam, dict(lr=args.policy_lr)),
policy,
max_optimization_epochs=args.n_minibatches,
minibatch_size=args.minibatch_size),
vf_optimizer=OptimizerWrapper(
(torch.optim.Adam, dict(lr=args.vf_lr)),
value_function,
max_optimization_epochs=args.n_minibatches,
minibatch_size=args.minibatch_size),
**convert_kwargs(args, PPO))
trainer.setup(algo,
env,
sampler_cls=LocalSampler,
worker_class=VecWorker,
worker_args={'n_envs': 8})
return algo
def maml_trpo_metaworld_ml10(ctxt, seed, epochs, rollouts_per_task,
meta_batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
rollouts_per_task (int): Number of rollouts per epoch per task
for training.
meta_batch_size (int): Number of tasks sampled per batch.
"""
set_seed(seed)
env = GarageEnv(
normalize(mwb.ML10.get_train_tasks(), expected_action_scale=10.))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(100, 100),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
max_episode_length = 100
test_task_names = mwb.ML10.get_test_tasks().all_task_names
test_tasks = [
GarageEnv(
normalize(mwb.ML10.from_task(task), expected_action_scale=10.))
for task in test_task_names
]
test_sampler = EnvPoolSampler(test_tasks)
meta_evaluator = MetaEvaluator(test_task_sampler=test_sampler,
max_episode_length=max_episode_length,
n_test_tasks=len(test_task_names))
runner = LocalRunner(ctxt)
algo = MAMLTRPO(env=env,
policy=policy,
value_function=value_function,
max_episode_length=max_episode_length,
meta_batch_size=meta_batch_size,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1,
meta_evaluator=meta_evaluator)
runner.setup(algo, env)
runner.train(n_epochs=epochs,
batch_size=rollouts_per_task * max_episode_length)
def run_task(snapshot_config, *_):
"""Set up environment and algorithm and run the task.
Args:
snapshot_config (garage.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
If None, it will create one with default settings.
_ : Unused parameters
"""
env = TfEnv(env_name='InvertedDoublePendulum-v2')
runner = LocalRunner(snapshot_config)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
max_path_length=100,
discount=0.99,
center_adv=False)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=1024)
def run_task(snapshot_config, *_):
"""Set up environment and algorithm and run the task.
Args:
snapshot_config (garage.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
If None, it will create one with default settings.
_ : Unused parameters
"""
env = TfEnv(env_name='Pusher3DOF-v1')
runner = LocalRunner(snapshot_config)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=49,
discount=0.99,
center_adv=False,
max_kl_step=0.005,
**copyparams)
#runner.setup(algo, env)
#runner.train(n_epochs=100, batch_size=50*250)
runner.restore(
"/home/dell/garage/data/local/pusher/pusher_2020_06_01_23_45_24_0001")
runner.resume(n_epochs=800)
def test_maml_trpo_pendulum():
"""Test PPO with Pendulum environment."""
env = GarageEnv(normalize(HalfCheetahDirEnv(), expected_action_scale=10.))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32))
rollouts_per_task = 5
max_episode_length = 100
runner = LocalRunner(snapshot_config)
algo = MAMLTRPO(env=env,
policy=policy,
value_function=value_function,
max_episode_length=max_episode_length,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
runner.setup(algo, env, sampler_cls=LocalSampler)
last_avg_ret = runner.train(n_epochs=5,
batch_size=rollouts_per_task *
max_episode_length)
assert last_avg_ret > -5
env.close()
def maml_trpo_half_cheetah_dir(ctxt, seed, epochs, episodes_per_task,
meta_batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (ExperimentContext): The experiment configuration used by
:class:`~Trainer` to create the :class:`~Snapshotter`.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
episodes_per_task (int): Number of episodes per epoch per task for
training.
meta_batch_size (int): Number of tasks sampled per batch.
"""
set_seed(seed)
max_episode_length = 100
env = normalize(GymEnv(HalfCheetahDirEnv(),
max_episode_length=max_episode_length),
expected_action_scale=10.)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
task_sampler = SetTaskSampler(
HalfCheetahDirEnv,
wrapper=lambda env, _: normalize(GymEnv(
env, max_episode_length=max_episode_length),
expected_action_scale=10.))
meta_evaluator = MetaEvaluator(test_task_sampler=task_sampler,
n_test_tasks=1,
n_test_episodes=10)
trainer = Trainer(ctxt)
algo = MAMLTRPO(env=env,
policy=policy,
task_sampler=task_sampler,
value_function=value_function,
meta_batch_size=meta_batch_size,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1,
meta_evaluator=meta_evaluator)
trainer.setup(algo, env)
trainer.train(n_epochs=epochs,
batch_size=episodes_per_task * env.spec.max_episode_length)
def maml_trpo_metaworld_ml45(ctxt, seed, epochs, episodes_per_task,
meta_batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (ExperimentContext): The experiment configuration used by
:class:`~Trainer` to create the :class:`~Snapshotter`.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
episodes_per_task (int): Number of episodes per epoch per task
for training.
meta_batch_size (int): Number of tasks sampled per batch.
"""
set_seed(seed)
ml45 = metaworld.ML45()
# pylint: disable=missing-return-doc,missing-return-type-doc
def wrap(env, _):
return normalize(env, expected_action_scale=10.0)
train_task_sampler = MetaWorldTaskSampler(ml45, 'train', wrap)
test_env = wrap(MetaWorldSetTaskEnv(ml45, 'test'), None)
test_task_sampler = SetTaskSampler(MetaWorldSetTaskEnv,
env=test_env,
wrapper=wrap)
env = train_task_sampler.sample(45)[0]()
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(100, 100),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
meta_evaluator = MetaEvaluator(test_task_sampler=test_task_sampler)
trainer = Trainer(ctxt)
algo = MAMLTRPO(env=env,
task_sampler=train_task_sampler,
policy=policy,
value_function=value_function,
meta_batch_size=meta_batch_size,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1,
meta_evaluator=meta_evaluator)
trainer.setup(algo, env, n_workers=meta_batch_size)
trainer.train(n_epochs=epochs,
batch_size=episodes_per_task * env.spec.max_episode_length)
def setup_method(self):
self.env = GarageEnv(normalize(gym.make('InvertedDoublePendulum-v2')))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.baseline = LinearFeatureBaseline(env_spec=self.env.spec)
def test_policy_get_actions(mock_model, input_dim, output_dim, hidden_sizes):
action = torch.randn((output_dim, ))
mock_dist = mock.MagicMock()
mock_dist.rsample.return_value = action
mock_model.return_value = mock_dist
env_spec = mock.MagicMock()
env_spec.observation_space.flat_dim = input_dim
env_spec.action_space.flat_dim = output_dim
policy = GaussianMLPPolicy(env_spec, mock_model)
input = torch.ones(input_dim)
sample = policy.get_actions(input)
assert np.array_equal(sample, action.detach().numpy())
def mtppo_metaworld_mt50(ctxt, seed, epochs, batch_size, n_workers, n_tasks):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by Trainer to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
batch_size (int): Number of environment steps in one batch.
n_workers (int): The number of workers the sampler should use.
n_tasks (int): Number of tasks to use. Should be a multiple of 50.
"""
set_seed(seed)
mt10 = metaworld.MT10()
train_task_sampler = MetaWorldTaskSampler(mt10,
'train',
lambda env, _: normalize(env),
add_env_onehot=True)
assert n_tasks % 50 == 0
assert n_tasks <= 2500
envs = [env_up() for env_up in train_task_sampler.sample(n_tasks)]
env = MultiEnvWrapper(envs,
sample_strategy=round_robin_strategy,
mode='vanilla')
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
sampler = RaySampler(agents=policy,
envs=env,
max_episode_length=env.spec.max_episode_length,
n_workers=n_workers)
algo = PPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
sampler=sampler,
discount=0.99,
gae_lambda=0.95,
center_adv=True,
lr_clip_range=0.2)
trainer = Trainer(ctxt)
trainer.setup(algo, env)
trainer.train(n_epochs=epochs, batch_size=batch_size)
def setup_method(self):
"""Setup method which is called before every test."""
self.env = GarageEnv(normalize(gym.make('InvertedDoublePendulum-v2')))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.value_function = LinearFeatureBaseline(env_spec=self.env.spec)
def test_get_action_dict_space(self):
"""Test if observations from dict obs spaces are properly flattened."""
env = GymEnv(DummyDictEnv(obs_space_type='box', act_space_type='box'))
policy = GaussianMLPPolicy(env_spec=env.spec,
hidden_nonlinearity=None,
hidden_sizes=(1, ),
hidden_w_init=nn.init.ones_,
output_w_init=nn.init.ones_)
obs = env.reset()[0]
action, _ = policy.get_action(obs)
assert env.action_space.shape == action.shape
actions, _ = policy.get_actions(np.array([obs, obs]))
for action in actions:
assert env.action_space.shape == action.shape
actions, _ = policy.get_actions(np.array([obs, obs]))
for action in actions:
assert env.action_space.shape == action.shape
def maml_trpo(ctxt, seed, epochs, rollouts_per_task, meta_batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
rollouts_per_task (int): Number of rollouts per epoch per task
for training.
meta_batch_size (int): Number of tasks sampled per batch.
"""
set_seed(seed)
# @TODO blowing up here...
env = GarageEnv(normalize(HalfCheetahDirEnv(), expected_action_scale=10.))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
max_path_length = 100
task_sampler = SetTaskSampler(lambda: GarageEnv(
normalize(HalfCheetahDirEnv(), expected_action_scale=10.)))
meta_evaluator = MetaEvaluator(test_task_sampler=task_sampler,
max_path_length=max_path_length,
n_test_tasks=1,
n_test_rollouts=10)
runner = LocalRunner(ctxt)
algo = MAMLTRPO(env=env,
policy=policy,
value_function=value_function,
max_path_length=max_path_length,
meta_batch_size=meta_batch_size,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1,
meta_evaluator=meta_evaluator)
runner.setup(algo, env)
runner.train(n_epochs=epochs,
batch_size=rollouts_per_task * max_path_length)
def setup_method(self):
"""Setup method which is called before every test."""
self.env = GarageEnv(
normalize(HalfCheetahDirEnv(), expected_action_scale=10.))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.baseline = LinearFeatureBaseline(env_spec=self.env.spec)
def setup_method(self):
"""Setup method which is called before every test."""
self.env = normalize(
GymEnv('InvertedDoublePendulum-v2', max_episode_length=100))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.value_function = GaussianMLPValueFunction(env_spec=self.env.spec)
def setup_method(self):
"""Setup method which is called before every test."""
self.env = GarageEnv(
normalize(HalfCheetahDirEnv(), expected_action_scale=10.))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.value_function = LinearFeatureBaseline(env_spec=self.env.spec)
self.algo = MAMLPPO(env=self.env,
policy=self.policy,
value_function=self.value_function,
max_path_length=100,
meta_batch_size=5,
discount=0.99,
gae_lambda=1.,
inner_lr=0.1,
num_grad_updates=1)
def setup_method(self):
"""Setup method which is called before every test."""
self.env = normalize(GymEnv(HalfCheetahDirEnv()),
expected_action_scale=10.)
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
self.value_function = GaussianMLPValueFunction(env_spec=self.env.spec,
hidden_sizes=(32, 32))
def test_is_pickleable(self, batch_size, hidden_sizes):
env_spec = TfEnv(DummyBoxEnv())
obs_dim = env_spec.observation_space.flat_dim
obs = torch.ones([batch_size, obs_dim], dtype=torch.float32)
init_std = 2.
policy = GaussianMLPPolicy(env_spec=env_spec,
hidden_sizes=hidden_sizes,
init_std=init_std,
hidden_nonlinearity=None,
std_parameterization='exp',
hidden_w_init=nn.init.ones_,
output_w_init=nn.init.ones_)
output1_action, output1_prob = policy.get_actions(obs)
p = pickle.dumps(policy)
policy_pickled = pickle.loads(p)
output2_action, output2_prob = policy_pickled.get_actions(obs)
assert output1_prob['mean'].equal(output2_prob['mean'])
assert output1_action.shape == output2_action.shape
def tutorial_vpg(ctxt=None):
"""Train VPG with PointEnv environment.
Args:
ctxt (ExperimentContext): The experiment configuration used by
:class:`~LocalRunner` to create the :class:`~Snapshotter`.
"""
set_seed(100)
runner = LocalRunner(ctxt)
env = PointEnv()
policy = GaussianMLPPolicy(env.spec)
algo = SimpleVPG(env.spec, policy)
runner.setup(algo, env)
runner.train(n_epochs=200, batch_size=4000)