def trpo_pendulum(ctxt=None, seed=1):
"""Train TRPO with InvertedDoublePendulum-v2 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
env = TfEnv(env_name='InvertedDoublePendulum-v2')
runner = LocalRunner(ctxt)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
max_path_length=100,
discount=0.99,
center_adv=False)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=1024)
def run_task(snapshot_config, *_):
"""Set up environment and algorithm and run the task.
Args:
snapshot_config (garage.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
If None, it will create one with default settings.
_ : Unused parameters
"""
env = TfEnv(env_name='InvertedDoublePendulum-v2')
runner = LocalRunner(snapshot_config)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
max_path_length=100,
discount=0.99,
center_adv=False)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=1024)
def run_task(snapshot_config, *_):
"""Set up environment and algorithm and run the task.
Args:
snapshot_config (garage.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
If None, it will create one with default settings.
_ : Unused parameters
"""
env = TfEnv(env_name='Pusher3DOF-v1')
runner = LocalRunner(snapshot_config)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=49,
discount=0.99,
center_adv=False,
max_kl_step=0.005,
**copyparams)
#runner.setup(algo, env)
#runner.train(n_epochs=100, batch_size=50*250)
runner.restore(
"/home/dell/garage/data/local/pusher/pusher_2020_06_01_23_45_24_0001")
runner.resume(n_epochs=800)
def load_trpo(env_name="CartPole-v0"):
"""Return an instance of the TRPO algorithm."""
env = GarageEnv(env_name=env_name)
policy = DeterministicMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
vfunc = GaussianMLPValueFunction(env_spec=env.spec)
algo = TRPO(env_spec=env.spec, policy=policy, value_function=vfunc)
return algo
def mttrpo_metaworld_mt50(ctxt, seed, epochs, batch_size, n_workers, n_tasks):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by Trainer to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
batch_size (int): Number of environment steps in one batch.
n_workers (int): The number of workers the sampler should use.
n_tasks (int): Number of tasks to use. Should be a multiple of 50.
"""
set_seed(seed)
mt10 = metaworld.MT10()
train_task_sampler = MetaWorldTaskSampler(mt10,
'train',
lambda env, _: normalize(env),
add_env_onehot=True)
assert n_tasks % 10 == 0
assert n_tasks <= 500
envs = [env_up() for env_up in train_task_sampler.sample(n_tasks)]
env = MultiEnvWrapper(envs,
sample_strategy=round_robin_strategy,
mode='vanilla')
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
sampler = RaySampler(agents=policy,
envs=env,
max_episode_length=env.spec.max_episode_length,
n_workers=n_workers)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
sampler=sampler,
discount=0.99,
gae_lambda=0.95)
trainer = Trainer(ctxt)
trainer.setup(algo, env)
trainer.train(n_epochs=epochs, batch_size=batch_size)
def test_trpo_pendulum(self):
"""Test TRPO with Pendulum environment."""
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
algo = TRPO(env_spec=self.env.spec,
policy=self.policy,
value_function=self.value_function,
discount=0.99,
gae_lambda=0.98)
runner.setup(algo, self.env, sampler_cls=LocalSampler)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 0
def test_trpo_pendulum(self):
"""Test TRPO with Pendulum environment."""
deterministic.set_seed(0)
runner = LocalRunner(snapshot_config)
algo = TRPO(env_spec=self.env.spec,
policy=self.policy,
baseline=self.baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.98)
runner.setup(algo, self.env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 50
def mttrpo_metaworld_mt10(ctxt, seed, epochs, batch_size, n_worker):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
batch_size (int): Number of environment steps in one batch.
n_worker (int): The number of workers the sampler should use.
"""
set_seed(seed)
tasks = mwb.MT10.get_train_tasks().all_task_names
envs = []
for task in tasks:
envs.append(
normalize(GymEnv(mwb.MT10.from_task(task),
max_episode_length=150)))
env = MultiEnvWrapper(envs,
sample_strategy=round_robin_strategy,
mode='vanilla')
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
discount=0.99,
gae_lambda=0.95)
runner = LocalRunner(ctxt)
runner.setup(algo, env, n_workers=n_worker)
runner.train(n_epochs=epochs, batch_size=batch_size)
def test_trpo_pendulum(self):
"""Test TRPO with Pendulum environment."""
deterministic.set_seed(0)
sampler = LocalSampler(
agents=self.policy,
envs=self.env,
max_episode_length=self.env.spec.max_episode_length)
trainer = Trainer(snapshot_config)
algo = TRPO(env_spec=self.env.spec,
policy=self.policy,
value_function=self.value_function,
sampler=sampler,
discount=0.99,
gae_lambda=0.98)
trainer.setup(algo, self.env)
last_avg_ret = trainer.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 0
def mttrpo_metaworld_mt1_push(ctxt, seed, epochs, batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by Trainer to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
batch_size (int): Number of environment steps in one batch.
"""
set_seed(seed)
n_tasks = 50
mt1 = metaworld.MT1('push-v1')
train_task_sampler = MetaWorldTaskSampler(mt1, 'train',
lambda env, _: normalize(env))
envs = [env_up() for env_up in train_task_sampler.sample(n_tasks)]
env = MultiEnvWrapper(envs,
sample_strategy=round_robin_strategy,
mode='vanilla')
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
discount=0.99,
gae_lambda=0.95)
trainer = Trainer(ctxt)
trainer.setup(algo, env)
trainer.train(n_epochs=epochs, batch_size=batch_size)
def trpo_pendulum_ray_sampler(ctxt=None, seed=1):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
# Since this is an example, we are running ray in a reduced state.
# One can comment this line out in order to run ray at full capacity
ray.init(memory=52428800,
object_store_memory=78643200,
ignore_reinit_error=True,
log_to_driver=False,
include_webui=False)
deterministic.set_seed(seed)
env = GarageEnv(env_name='InvertedDoublePendulum-v2')
runner = LocalRunner(ctxt)
policy = GaussianMLPPolicy(env.spec,
hidden_sizes=[32, 32],
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
max_path_length=100,
discount=0.99,
center_adv=False)
runner.setup(algo, env, sampler_cls=RaySampler)
runner.train(n_epochs=100, batch_size=1024)
def mttrpo_metaworld_ml1_push(ctxt, seed, epochs, batch_size):
"""Set up environment and algorithm and run the task.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
epochs (int): Number of training epochs.
batch_size (int): Number of environment steps in one batch.
"""
set_seed(seed)
env = GarageEnv(normalize(mwb.ML1.get_train_tasks('push-v1')))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None,
)
value_function = GaussianMLPValueFunction(env_spec=env.spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=torch.tanh,
output_nonlinearity=None)
algo = TRPO(env_spec=env.spec,
policy=policy,
value_function=value_function,
max_episode_length=128,
discount=0.99,
gae_lambda=0.95)
runner = LocalRunner(ctxt)
runner.setup(algo, env)
runner.train(n_epochs=epochs, batch_size=batch_size)