def test_meta_evaluator_with_tf():
set_seed(100)
tasks = SetTaskSampler(lambda: GarageEnv(PointEnv()))
max_path_length = 200
env = GarageEnv(PointEnv())
n_traj = 3
with tempfile.TemporaryDirectory() as log_dir_name:
ctxt = SnapshotConfig(snapshot_dir=log_dir_name,
snapshot_mode='none',
snapshot_gap=1)
with LocalTFRunner(ctxt) as runner:
meta_eval = MetaEvaluator(test_task_sampler=tasks,
max_path_length=max_path_length,
n_test_tasks=10,
n_exploration_traj=n_traj)
policy = GaussianMLPPolicy(env.spec)
algo = MockAlgo(env, policy, max_path_length, n_traj, meta_eval)
runner.setup(algo, env)
log_file = tempfile.NamedTemporaryFile()
csv_output = CsvOutput(log_file.name)
logger.add_output(csv_output)
meta_eval.evaluate(algo)
algo_pickle = cloudpickle.dumps(algo)
tf.compat.v1.reset_default_graph()
with LocalTFRunner(ctxt) as runner:
algo2 = cloudpickle.loads(algo_pickle)
runner.setup(algo2, env)
runner.train(10, 0)
def test_session(self):
with LocalTFRunner(snapshot_config):
assert tf.compat.v1.get_default_session() is not None, (
'LocalTFRunner() should provide a default tf session.')
sess = tf.compat.v1.Session()
with LocalTFRunner(snapshot_config, sess=sess):
assert tf.compat.v1.get_default_session() is sess, (
'LocalTFRunner(sess) should use sess as default session.')
def trpo_garage_tf(ctxt, env_id, seed):
"""Create garage Tensorflow TROI model and training.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the
snapshotter.
env_id (str): Environment id of the task.
seed (int): Random positive integer for the trial.
"""
deterministic.set_seed(seed)
with LocalTFRunner(ctxt) as runner:
env = normalize(GymEnv(env_id))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=hyper_parameters['hidden_sizes'],
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None,
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
discount=hyper_parameters['discount'],
gae_lambda=hyper_parameters['gae_lambda'],
max_kl_step=hyper_parameters['max_kl'])
runner.setup(algo, env)
runner.train(n_epochs=hyper_parameters['n_epochs'],
batch_size=hyper_parameters['batch_size'])
def multi_env_trpo(ctxt=None, seed=1):
"""Train TRPO on two different PointEnv instances.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(ctxt) as runner:
env1 = normalize(PointEnv(goal=(-1., 0.), max_episode_length=100))
env2 = normalize(PointEnv(goal=(1., 0.), max_episode_length=100))
env = MultiEnvWrapper([env1, env2])
policy = GaussianMLPPolicy(env_spec=env.spec)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
policy_ent_coeff=0.0)
runner.setup(algo, env)
runner.train(n_epochs=40, batch_size=2048, plot=False)
def trpo_cartpole_recurrent(ctxt, seed, n_epochs, batch_size, plot):
"""Train TRPO with a recurrent policy on CartPole.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
n_epochs (int): Number of epochs for training.
seed (int): Used to seed the random number generator to produce
determinism.
batch_size (int): Batch size used for training.
plot (bool): Whether to plot or not.
"""
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
env = GymEnv('CartPole-v1')
policy = CategoricalLSTMPolicy(name='policy', env_spec=env.spec)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_episode_length=100,
discount=0.99,
max_kl_step=0.01,
optimizer=ConjugateGradientOptimizer,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)))
runner.setup(algo, env)
runner.train(n_epochs=n_epochs, batch_size=batch_size, plot=plot)
def test_rl2_ppo_pendulum_wrong_worker(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
with pytest.raises(ValueError):
algo = RL2PPO(rl2_max_episode_length=self.max_episode_length,
meta_batch_size=self.meta_batch_size,
task_sampler=self.tasks,
env_spec=self.env_spec,
policy=self.policy,
baseline=self.baseline,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
optimizer_args=dict(
batch_size=32,
max_episode_length=10,
),
stop_entropy_gradient=True,
entropy_method='max',
policy_ent_coeff=0.02,
center_adv=False,
max_episode_length=self.max_episode_length *
self.episode_per_task)
runner.setup(algo,
self.tasks.sample(self.meta_batch_size),
sampler_cls=LocalSampler,
n_workers=self.meta_batch_size)
runner.train(n_epochs=10,
batch_size=self.episode_per_task *
self.max_episode_length * self.meta_batch_size)
def test_ppo_pendulum_recurrent_continuous_baseline(self):
"""Test PPO with Pendulum environment and recurrent policy."""
with LocalTFRunner(snapshot_config) as runner:
env = normalize(GymEnv('InvertedDoublePendulum-v2'))
policy = GaussianLSTMPolicy(env_spec=env.spec, )
baseline = ContinuousMLPBaseline(
env_spec=env.spec,
hidden_sizes=(32, 32),
)
algo = PPO(
env_spec=env.spec,
policy=policy,
baseline=baseline,
max_episode_length=100,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
optimizer_args=dict(
batch_size=32,
max_episode_length=10,
),
stop_entropy_gradient=True,
entropy_method='max',
policy_ent_coeff=0.02,
center_adv=False,
)
runner.setup(algo, env, sampler_cls=LocalSampler)
last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
assert last_avg_ret > 100
env.close()
def fixture_exp(snapshot_config, sess):
"""Dummy fixture experiment function.
Args:
snapshot_config (garage.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
If None, it will create one with default settings.
sess (tf.Session): An optional TensorFlow session.
A new session will be created immediately if not provided.
Returns:
np.ndarray: Values of the parameters evaluated in
the current session
"""
with LocalTFRunner(snapshot_config=snapshot_config, sess=sess) as runner:
env = GarageEnv(env_name='CartPole-v1')
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(8, 8))
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = VPG(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_episode_length=100,
discount=0.99,
optimizer_args=dict(learning_rate=0.01, ))
runner.setup(algo, env, sampler_cls=LocalSampler)
runner.train(n_epochs=5, batch_size=100)
return policy.get_param_values()
def test_rl2_trpo_pendulum(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
algo = RL2TRPO(
meta_batch_size=self.meta_batch_size,
task_sampler=self.tasks,
env_spec=self.env_spec,
policy=self.policy,
baseline=self.baseline,
episodes_per_trial=self.episode_per_task,
discount=0.99,
max_kl_step=0.01,
optimizer=ConjugateGradientOptimizer,
optimizer_args=dict(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)))
runner.setup(algo,
self.tasks.sample(self.meta_batch_size),
sampler_cls=LocalSampler,
n_workers=self.meta_batch_size,
worker_class=RL2Worker)
last_avg_ret = runner.train(n_epochs=1,
batch_size=self.episode_per_task *
self.max_episode_length *
self.meta_batch_size)
assert last_avg_ret > -40
def test_trpo_cnn_cubecrash(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = normalize(GymEnv('CubeCrash-v0'))
policy = CategoricalCNNPolicy(env_spec=env.spec,
filters=((32, (8, 8)), (64, (4, 4))),
strides=(4, 2),
padding='VALID',
hidden_sizes=(32, 32))
baseline = GaussianCNNBaseline(env_spec=env.spec,
filters=((32, (8, 8)), (64, (4,
4))),
strides=(4, 2),
padding='VALID',
hidden_sizes=(32, 32),
use_trust_region=True)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_episode_length=100,
discount=0.99,
gae_lambda=0.98,
max_kl_step=0.01,
policy_ent_coeff=0.0)
runner.setup(algo, env, sampler_cls=LocalSampler)
last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
assert last_avg_ret > -1.5
env.close()
def trpois_inverted_pendulum(ctxt=None, seed=1):
"""Train TRPO on InvertedPendulum-v2 with importance sampling.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(ctxt) as runner:
env = GarageEnv(normalize(gym.make('InvertedPendulum-v2')))
policy = GaussianMLPPolicy(env_spec=env.spec, hidden_sizes=(32, 32))
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
max_kl_step=0.01)
runner.setup(algo,
env,
sampler_cls=ISSampler,
sampler_args=dict(n_backtrack=1))
runner.train(n_epochs=200, batch_size=4000)
def osimArmResume(ctxt=None,
snapshot_dir='data/local/experiment/osimArm_153',
seed=1):
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
runner.restore(snapshot_dir)
ddpg = runner._algo
env = GarageEnv(Arm2DVecEnv(visualize=True))
env.reset()
policy = ddpg.policy
env.render()
obs = env.step(env.action_space.sample())
steps = 0
n_steps = 100
while True:
if steps == n_steps:
env.close()
break
temp = policy.get_action(obs[0])
obs = env.step(temp[0])
env.render()
steps += 1
def test_rl2_ppo_pendulum_exploration_policy(self):
with LocalTFRunner(snapshot_config, sess=self.sess):
algo = RL2PPO(meta_batch_size=self.meta_batch_size,
task_sampler=self.tasks,
env_spec=self.env_spec,
policy=self.policy,
baseline=self.baseline,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
optimizer_args=dict(
batch_size=32,
max_optimization_epochs=10,
),
stop_entropy_gradient=True,
entropy_method='max',
policy_ent_coeff=0.02,
center_adv=False,
episodes_per_trial=self.episode_per_task)
exploration_policy = algo.get_exploration_policy()
params = exploration_policy.get_param_values()
new_params = np.zeros_like(params)
exploration_policy.set_param_values(new_params)
assert np.array_equal(new_params,
exploration_policy.get_param_values())
def test_ppo_pendulum_flatten_input(self):
"""Test PPO with CartPole to test observation flattening."""
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = GarageEnv(
normalize(ReshapeObservation(gym.make('CartPole-v1'), (2, 2))))
policy = CategoricalMLPPolicy(
env_spec=env.spec,
hidden_nonlinearity=tf.nn.tanh,
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = PPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
policy_ent_coeff=0.0,
optimizer_args=dict(
batch_size=32,
max_epochs=10,
learning_rate=1e-3,
))
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
assert last_avg_ret > 80
def trpo_minigrid(ctxt=None, seed=1):
"""Train TRPO with MiniGrid-FourRooms-v0 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(ctxt) as runner:
env = GarageEnv(env_name='DisabledAntPyBulletEnv-v0')
policy = GaussianMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(128, 64, 32))
# baseline = LinearFeatureBaseline(env_spec=env.spec)
baseline = GaussianMLPBaseline(
env_spec=env.spec
)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
discount=0.99,
max_kl_step=0.001)
runner.setup(algo, env)
runner.train(n_epochs=2000, batch_size=4000)
def trpo_cartpole(ctxt=None, seed=1):
"""Train TRPO with CartPole-v1 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(ctxt) as runner:
env = GarageEnv(env_name='CartPole-v1')
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
max_kl_step=0.01)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=4000)
def vpg_cartpole(ctxt=None, seed=1):
"""Train VPG with CartPole-v1 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
env = GymEnv('CartPole-v1')
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = VPG(env_spec=env.spec,
policy=policy,
baseline=baseline,
discount=0.99,
optimizer_args=dict(learning_rate=0.01, ))
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=10000)
def cem_cartpole(ctxt=None, seed=1):
"""Train CEM with Cartpole-v1 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
env = GarageEnv(env_name='CartPole-v1')
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
baseline = LinearFeatureBaseline(env_spec=env.spec)
n_samples = 20
algo = CEM(env_spec=env.spec,
policy=policy,
baseline=baseline,
best_frac=0.05,
max_path_length=100,
n_samples=n_samples)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=1000)
def test_ddpg_double_pendulum(self):
"""Test DDPG with Pendulum environment."""
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = GarageEnv(gym.make('InvertedDoublePendulum-v2'))
policy = ContinuousMLPPolicy(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.tanh)
exploration_policy = AddOrnsteinUhlenbeckNoise(env.spec,
policy,
sigma=0.2)
qf = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=tf.nn.relu)
replay_buffer = PathBuffer(capacity_in_transitions=int(1e5))
algo = DDPG(
env_spec=env.spec,
policy=policy,
policy_lr=1e-4,
qf_lr=1e-3,
qf=qf,
replay_buffer=replay_buffer,
max_path_length=100,
steps_per_epoch=20,
target_update_tau=1e-2,
n_train_steps=50,
discount=0.9,
min_buffer_size=int(5e3),
exploration_policy=exploration_policy,
)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 60
env.close()
def test_te_ppo(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
algo = TEPPO(env_spec=self.env.spec,
policy=self.policy,
baseline=self.baseline,
inference=self.inference,
max_path_length=self.max_path_length,
discount=0.99,
lr_clip_range=0.2,
policy_ent_coeff=self.policy_ent_coeff,
encoder_ent_coeff=self.encoder_ent_coeff,
inference_ce_coeff=self.inference_ce_coeff,
use_softplus_entropy=True,
optimizer_args=dict(
batch_size=32,
max_epochs=10,
),
inference_optimizer_args=dict(
batch_size=32,
max_epochs=10,
),
center_adv=True,
stop_ce_gradient=True)
runner.setup(algo,
self.env,
sampler_cls=LocalSampler,
sampler_args=None,
worker_class=TaskEmbeddingWorker)
runner.train(n_epochs=1, batch_size=self.batch_size, plot=False)
def test_dm_control_tf_policy(self):
task = ALL_TASKS[0]
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = DMControlEnv.from_suite(*task)
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(32, 32),
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(
env_spec=env.spec,
policy=policy,
baseline=baseline,
max_episode_length=5,
discount=0.99,
max_kl_step=0.01,
)
runner.setup(algo, env, sampler_cls=LocalSampler)
runner.train(n_epochs=1, batch_size=10)
env.close()
def test_rl2_ppo_pendulum_adapted_policy(self):
with LocalTFRunner(snapshot_config, sess=self.sess):
algo = RL2PPO(rl2_max_episode_length=self.max_episode_length,
meta_batch_size=self.meta_batch_size,
task_sampler=self.tasks,
env_spec=self.env_spec,
policy=self.policy,
baseline=self.baseline,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
optimizer_args=dict(
batch_size=32,
max_episode_length=10,
),
stop_entropy_gradient=True,
entropy_method='max',
policy_ent_coeff=0.02,
center_adv=False,
max_episode_length=self.max_episode_length *
self.episode_per_task)
exploration_policy = algo.get_exploration_policy()
adapted_policy = algo.adapt_policy(exploration_policy, [])
(params, hidden) = adapted_policy.get_param_values()
expected_new_params = np.zeros_like(params)
expected_hidden = np.zeros_like(hidden)
adapted_policy.set_param_values(
(expected_new_params, expected_hidden))
(new_params, new_hidden) = adapted_policy.get_param_values()
assert np.array_equal(expected_new_params, new_params)
assert np.array_equal(expected_hidden, new_hidden)
def test_trpo_cnn_cubecrash(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = GarageEnv(normalize(gym.make('CubeCrash-v0')))
policy = CategoricalCNNPolicy(env_spec=env.spec,
filters=((32, (8, 8)), (64, (4, 4))),
strides=(4, 2),
padding='VALID',
hidden_sizes=(32, 32))
baseline = GaussianCNNBaseline(
env_spec=env.spec,
regressor_args=dict(filters=((32, (8, 8)), (64, (4, 4))),
strides=(4, 2),
padding='VALID',
hidden_sizes=(32, 32),
use_trust_region=True))
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.98,
max_kl_step=0.01,
policy_ent_coeff=0.0,
flatten_input=False)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
assert last_avg_ret > -1.5
env.close()
def test_rl2_ppo_pendulum(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
algo = RL2PPO(rl2_max_episode_length=self.max_episode_length,
meta_batch_size=self.meta_batch_size,
task_sampler=self.tasks,
env_spec=self.env_spec,
policy=self.policy,
baseline=self.baseline,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
stop_entropy_gradient=True,
entropy_method='max',
policy_ent_coeff=0.02,
center_adv=False,
max_episode_length=self.max_episode_length *
self.episode_per_task)
runner.setup(
algo,
self.tasks.sample(self.meta_batch_size),
sampler_cls=LocalSampler,
n_workers=self.meta_batch_size,
worker_class=RL2Worker,
worker_args=dict(n_paths_per_trial=self.episode_per_task))
last_avg_ret = runner.train(n_epochs=1,
batch_size=self.episode_per_task *
self.max_episode_length *
self.meta_batch_size)
assert last_avg_ret > -40
def erwr_cartpole(ctxt=None, seed=1):
"""Train with ERWR on CartPole-v1 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
env = GymEnv('CartPole-v1')
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = ERWR(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_episode_length=100,
discount=0.99)
runner.setup(algo=algo, env=env)
runner.train(n_epochs=100, batch_size=10000, plot=False)
def test_ppo_pendulum_gru(self):
"""Test PPO with Pendulum environment and recurrent policy."""
with LocalTFRunner(snapshot_config) as runner:
env = GarageEnv(normalize(gym.make('InvertedDoublePendulum-v2')))
gru_policy = GaussianGRUPolicy(env_spec=env.spec)
baseline = GaussianMLPBaseline(
env_spec=env.spec,
regressor_args=dict(hidden_sizes=(32, 32)),
)
algo = PPO(
env_spec=env.spec,
policy=gru_policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
optimizer_args=dict(
batch_size=32,
max_epochs=10,
),
stop_entropy_gradient=True,
entropy_method='max',
policy_ent_coeff=0.02,
center_adv=False,
)
runner.setup(algo, env, sampler_cls=LocalSampler)
last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
assert last_avg_ret > 80
def her_ddpg_fetchreach(ctxt=None, seed=1):
"""Train DDPG + HER on the goal-conditioned FetchReach env.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
env = GymEnv('FetchReach-v1')
policy = ContinuousMLPPolicy(
env_spec=env.spec,
name='Policy',
hidden_sizes=[256, 256, 256],
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.tanh,
)
exploration_policy = AddOrnsteinUhlenbeckNoise(env.spec,
policy,
sigma=0.2)
qf = ContinuousMLPQFunction(
env_spec=env.spec,
name='QFunction',
hidden_sizes=[256, 256, 256],
hidden_nonlinearity=tf.nn.relu,
)
# pylint: disable=no-member
replay_buffer = HERReplayBuffer(capacity_in_transitions=int(1e6),
replay_k=4,
reward_fn=env.compute_reward,
env_spec=env.spec)
ddpg = DDPG(
env_spec=env.spec,
policy=policy,
policy_lr=1e-3,
qf_lr=1e-3,
qf=qf,
replay_buffer=replay_buffer,
target_update_tau=0.01,
steps_per_epoch=50,
max_episode_length=250,
n_train_steps=40,
discount=0.95,
exploration_policy=exploration_policy,
policy_optimizer=tf.compat.v1.train.AdamOptimizer,
qf_optimizer=tf.compat.v1.train.AdamOptimizer,
buffer_batch_size=256,
)
runner.setup(algo=ddpg, env=env)
runner.train(n_epochs=50, batch_size=256)
def reps_gym_cartpole(ctxt=None, seed=1):
"""Train REPS with CartPole-v0 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
"""
set_seed(seed)
with LocalTFRunner(snapshot_config=ctxt) as runner:
env = GarageEnv(gym.make('CartPole-v0'))
policy = CategoricalMLPPolicy(env_spec=env.spec, hidden_sizes=[32, 32])
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = REPS(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99)
runner.setup(algo, env)
runner.train(n_epochs=100, batch_size=4000, plot=False)
def trpo_swimmer(ctxt=None, seed=1, batch_size=4000):
"""Train TRPO with Swimmer-v2 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
batch_size (int): Number of timesteps to use in each training step.
"""
set_seed(seed)
with LocalTFRunner(ctxt) as runner:
env = GymEnv('Swimmer-v2')
policy = GaussianMLPPolicy(env_spec=env.spec, hidden_sizes=(32, 32))
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_episode_length=500,
discount=0.99,
max_kl_step=0.01)
runner.setup(algo, env)
runner.train(n_epochs=40, batch_size=batch_size)
def her_garage_tf(ctxt, env_id, seed):
"""Create garage TensorFlow HER model and training.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the
snapshotter.
env_id (str): Environment id of the task.
seed (int): Random positive integer for the trial.
"""
deterministic.set_seed(seed)
with LocalTFRunner(ctxt) as runner:
env = GarageEnv(normalize(gym.make(env_id)))
policy = ContinuousMLPPolicy(
env_spec=env.spec,
hidden_sizes=hyper_parameters['policy_hidden_sizes'],
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.tanh,
)
exploration_policy = AddOrnsteinUhlenbeckNoise(
env_spec=env.spec, policy=policy, sigma=hyper_parameters['sigma'])
qf = ContinuousMLPQFunction(
env_spec=env.spec,
hidden_sizes=hyper_parameters['qf_hidden_sizes'],
hidden_nonlinearity=tf.nn.relu,
)
replay_buffer = HERReplayBuffer(
env_spec=env.spec,
capacity_in_transitions=hyper_parameters['replay_buffer_size'],
replay_k=4,
reward_fn=env.compute_reward,
)
algo = DDPG(
env_spec=env.spec,
policy=policy,
qf=qf,
replay_buffer=replay_buffer,
steps_per_epoch=hyper_parameters['steps_per_epoch'],
policy_lr=hyper_parameters['policy_lr'],
qf_lr=hyper_parameters['qf_lr'],
target_update_tau=hyper_parameters['tau'],
n_train_steps=hyper_parameters['n_train_steps'],
discount=hyper_parameters['discount'],
exploration_policy=exploration_policy,
policy_optimizer=tf.compat.v1.train.AdamOptimizer,
qf_optimizer=tf.compat.v1.train.AdamOptimizer,
buffer_batch_size=256,
)
runner.setup(algo, env)
runner.train(n_epochs=hyper_parameters['n_epochs'],
batch_size=hyper_parameters['n_rollout_steps'])