def test_get_qval_sym(self, obs_dim, action_dim):
env = TfEnv(DummyBoxEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.q_functions.'
'continuous_mlp_q_function.MLPMergeModel'),
new=SimpleMLPMergeModel):
qf = ContinuousMLPQFunction(env_spec=env.spec)
env.reset()
obs, _, _, _ = env.step(1)
obs = obs.flatten()
act = np.full(action_dim, 0.5).flatten()
obs_ph, act_ph = qf.inputs
output1 = qf.get_qval([obs], [act])
input_var1 = tf.compat.v1.placeholder(tf.float32,
shape=(None, obs.shape[0]))
input_var2 = tf.compat.v1.placeholder(tf.float32,
shape=(None, act.shape[0]))
q_vals = qf.get_qval_sym(input_var1, input_var2, 'another')
output2 = self.sess.run(q_vals,
feed_dict={
input_var1: [obs],
input_var2: [act]
})
expected_output = np.full((1, ), 0.5)
assert np.array_equal(output1, output2)
assert np.array_equal(output2[0], expected_output)
def test_is_pickleable(self, obs_dim, action_dim):
env = TfEnv(DummyBoxEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.q_functions.'
'continuous_mlp_q_function.MLPMergeModel'),
new=SimpleMLPMergeModel):
qf = ContinuousMLPQFunction(env_spec=env.spec)
env.reset()
obs, _, _, _ = env.step(1)
obs = obs.flatten()
act = np.full(action_dim, 0.5).flatten()
obs_ph, act_ph = qf.inputs
with tf.compat.v1.variable_scope(
'ContinuousMLPQFunction/SimpleMLPMergeModel', reuse=True):
return_var = tf.compat.v1.get_variable('return_var')
# assign it to all one
return_var.load(tf.ones_like(return_var).eval())
output1 = qf.get_qval([obs], [act])
h_data = pickle.dumps(qf)
with tf.compat.v1.Session(graph=tf.Graph()):
qf_pickled = pickle.loads(h_data)
obs_ph_pickled, act_ph_pickled = qf_pickled.inputs
output2 = qf_pickled.get_qval([obs], [act])
assert np.array_equal(output1, output2)
def test_ddpg_double_pendulum(self):
"""Test DDPG with Pendulum environment."""
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = TfEnv(gym.make('InvertedDoublePendulum-v2'))
action_noise = OUStrategy(env.spec, sigma=0.2)
policy = ContinuousMLPPolicy(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.tanh)
qf = ContinuousMLPQFunction(env_spec=env.spec,
hidden_sizes=[64, 64],
hidden_nonlinearity=tf.nn.relu)
replay_buffer = SimpleReplayBuffer(env_spec=env.spec,
size_in_transitions=int(1e5),
time_horizon=100)
algo = DDPG(
env_spec=env.spec,
policy=policy,
policy_lr=1e-4,
qf_lr=1e-3,
qf=qf,
replay_buffer=replay_buffer,
steps_per_epoch=20,
target_update_tau=1e-2,
n_train_steps=50,
discount=0.9,
min_buffer_size=int(5e3),
exploration_strategy=action_noise,
)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=100)
assert last_avg_ret > 60
env.close()
def test_ppo_pendulum_recurrent_continuous_baseline(self):
"""Test PPO with Pendulum environment and recurrent policy."""
with LocalTFRunner(snapshot_config) as runner:
env = TfEnv(normalize(gym.make('InvertedDoublePendulum-v2')))
policy = GaussianLSTMPolicy(env_spec=env.spec, )
baseline = ContinuousMLPBaseline(
env_spec=env.spec,
regressor_args=dict(hidden_sizes=(32, 32)),
)
algo = PPO(
env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
optimizer_args=dict(
batch_size=32,
max_epochs=10,
),
stop_entropy_gradient=True,
entropy_method='max',
policy_ent_coeff=0.02,
center_adv=False,
)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
assert last_avg_ret > 100
env.close()
class TestQfDerivedPolicy(TfGraphTestCase):
def setup_method(self):
super().setup_method()
self.env = TfEnv(DummyDiscreteEnv())
self.qf = SimpleQFunction(self.env.spec)
self.policy = DiscreteQfDerivedPolicy(env_spec=self.env.spec,
qf=self.qf)
self.sess.run(tf.compat.v1.global_variables_initializer())
self.env.reset()
def test_discrete_qf_derived_policy(self):
obs, _, _, _ = self.env.step(1)
action, _ = self.policy.get_action(obs)
assert self.env.action_space.contains(action)
actions, _ = self.policy.get_actions([obs])
for action in actions:
assert self.env.action_space.contains(action)
def test_is_pickleable(self):
with tf.compat.v1.variable_scope('SimpleQFunction/SimpleMLPModel',
reuse=True):
return_var = tf.compat.v1.get_variable('return_var')
# assign it to all one
return_var.load(tf.ones_like(return_var).eval())
obs, _, _, _ = self.env.step(1)
action1, _ = self.policy.get_action(obs)
p = pickle.dumps(self.policy)
with tf.compat.v1.Session(graph=tf.Graph()):
policy_pickled = pickle.loads(p)
action2, _ = policy_pickled.get_action(obs)
assert action1 == action2
def run_task(snapshot_config, *_):
"""Run task.
Args:
snapshot_config (metarl.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
_ (object): Ignored by this function.
"""
with LocalTFRunner(snapshot_config=snapshot_config) as runner:
env1 = TfEnv(normalize(PointEnv(goal=(-1., 0.))))
env2 = TfEnv(normalize(PointEnv(goal=(1., 0.))))
env = MultiEnvWrapper([env1, env2])
policy = GaussianMLPPolicy(env_spec=env.spec)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
policy_ent_coeff=0.0)
runner.setup(algo, env)
runner.train(n_epochs=40, batch_size=2048, plot=False)
def test_is_pickleable(self):
env = TfEnv(DummyDiscreteEnv(obs_dim=(1, ), action_dim=1))
with mock.patch(('metarl.tf.policies.'
'categorical_gru_policy.GRUModel'),
new=SimpleGRUModel):
policy = CategoricalGRUPolicy(env_spec=env.spec,
state_include_action=False)
env.reset()
obs = env.reset()
with tf.compat.v1.variable_scope('CategoricalGRUPolicy/prob_network',
reuse=True):
return_var = tf.compat.v1.get_variable('return_var')
# assign it to all one
return_var.load(tf.ones_like(return_var).eval())
output1 = self.sess.run(
policy.model.outputs[0],
feed_dict={policy.model.input: [[obs.flatten()], [obs.flatten()]]})
p = pickle.dumps(policy)
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
policy_pickled = pickle.loads(p)
# yapf: disable
output2 = sess.run(
policy_pickled.model.outputs[0],
feed_dict={
policy_pickled.model.input: [[obs.flatten()],
[obs.flatten()]]
})
# yapf: enable
assert np.array_equal(output1, output2)
def test_is_pickleable(self):
env = TfEnv(DummyBoxEnv(obs_dim=(1, ), action_dim=(1, )))
with mock.patch(('metarl.tf.policies.'
'gaussian_lstm_policy.GaussianLSTMModel'),
new=SimpleGaussianLSTMModel):
policy = GaussianLSTMPolicy(env_spec=env.spec,
state_include_action=False)
env.reset()
obs = env.reset()
with tf.compat.v1.variable_scope(
'GaussianLSTMPolicy/GaussianLSTMModel', reuse=True):
return_var = tf.compat.v1.get_variable('return_var')
# assign it to all one
return_var.load(tf.ones_like(return_var).eval())
output1 = self.sess.run(
policy.model.networks['default'].mean,
feed_dict={policy.model.input: [[obs.flatten()], [obs.flatten()]]})
p = pickle.dumps(policy)
# yapf: disable
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
policy_pickled = pickle.loads(p)
output2 = sess.run(
policy_pickled.model.networks['default'].mean,
feed_dict={
policy_pickled.model.input: [[obs.flatten()],
[obs.flatten()]]
})
assert np.array_equal(output1, output2)
def test_dm_control_tf_policy(self):
task = ALL_TASKS[0]
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = TfEnv(DmControlEnv.from_suite(*task))
policy = GaussianMLPPolicy(
env_spec=env.spec,
hidden_sizes=(32, 32),
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(
env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=5,
discount=0.99,
max_kl_step=0.01,
)
runner.setup(algo, env)
runner.train(n_epochs=1, batch_size=10)
env.close()
def test_dist_info_sym_include_action(self, obs_dim, action_dim,
hidden_dim):
env = TfEnv(DummyBoxEnv(obs_dim=obs_dim, action_dim=action_dim))
obs_ph = tf.compat.v1.placeholder(
tf.float32, shape=(None, None, env.observation_space.flat_dim))
with mock.patch(('metarl.tf.policies.'
'gaussian_lstm_policy.GaussianLSTMModel'),
new=SimpleGaussianLSTMModel):
policy = GaussianLSTMPolicy(env_spec=env.spec,
state_include_action=True)
policy.reset()
obs = env.reset()
dist_sym = policy.dist_info_sym(
obs_var=obs_ph,
state_info_vars={'prev_action': np.zeros((2, 1) + action_dim)},
name='p2_sym')
dist = self.sess.run(
dist_sym, feed_dict={obs_ph: [[obs.flatten()], [obs.flatten()]]})
assert np.array_equal(dist['mean'], np.full((2, 1) + action_dim, 0.5))
assert np.array_equal(dist['log_std'], np.full((2, 1) + action_dim,
0.5))
def test_dist_info_sym(self, obs_dim, action_dim, filter_dims,
filter_sizes, strides, padding, hidden_sizes):
env = TfEnv(DummyDiscreteEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.policies.'
'categorical_cnn_policy.MLPModel'),
new=SimpleMLPModel):
with mock.patch(('metarl.tf.policies.'
'categorical_cnn_policy.CNNModel'),
new=SimpleCNNModel):
policy = CategoricalCNNPolicy(env_spec=env.spec,
conv_filters=filter_dims,
conv_filter_sizes=filter_sizes,
conv_strides=strides,
conv_pad=padding,
hidden_sizes=hidden_sizes)
env.reset()
obs, _, _, _ = env.step(1)
expected_prob = np.full(action_dim, 0.5)
obs_dim = env.spec.observation_space.shape
state_input = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + obs_dim)
dist1 = policy.dist_info_sym(state_input, name='policy2')
prob = self.sess.run(dist1['prob'], feed_dict={state_input: [obs]})
assert np.array_equal(prob[0], expected_prob)
def test_is_pickleable(self, mock_rand, obs_dim, action_dim):
mock_rand.return_value = 0
env = TfEnv(DummyDiscreteEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.policies.'
'categorical_cnn_policy.MLPModel'),
new=SimpleMLPModel):
with mock.patch(('metarl.tf.policies.'
'categorical_cnn_policy.CNNModel'),
new=SimpleCNNModel):
policy = CategoricalCNNPolicy(env_spec=env.spec,
conv_filters=(32, ),
conv_filter_sizes=(3, ),
conv_strides=(1, ),
conv_pad='SAME',
hidden_sizes=(4, ))
env.reset()
obs, _, _, _ = env.step(1)
with tf.compat.v1.variable_scope(
'CategoricalCNNPolicy/Sequential/MLPModel', reuse=True):
return_var = tf.compat.v1.get_variable('return_var')
# assign it to all one
return_var.load(tf.ones_like(return_var).eval())
output1 = self.sess.run(policy.model.outputs,
feed_dict={policy.model.input: [obs]})
p = pickle.dumps(policy)
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
policy_pickled = pickle.loads(p)
output2 = sess.run(policy_pickled.model.outputs,
feed_dict={policy_pickled.model.input: [obs]})
assert np.array_equal(output1, output2)
def test_vpg_cartpole(self):
"""Test VPG with CartPole-v1 environment."""
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = TfEnv(env_name='CartPole-v1')
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = VPG(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
optimizer_args=dict(
tf_optimizer_args=dict(learning_rate=0.01, )))
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=10000)
assert last_avg_ret > 90
env.close()
def test_cem_cartpole(self):
"""Test CEM with Cartpole-v1 environment."""
with LocalTFRunner(snapshot_config) as runner:
env = TfEnv(env_name='CartPole-v1')
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
baseline = LinearFeatureBaseline(env_spec=env.spec)
n_samples = 10
algo = CEM(env_spec=env.spec,
policy=policy,
baseline=baseline,
best_frac=0.1,
max_path_length=100,
n_samples=n_samples)
runner.setup(algo, env, sampler_cls=OnPolicyVectorizedSampler)
rtn = runner.train(n_epochs=10, batch_size=2048)
assert rtn > 40
env.close()
def test_get_action(self, mock_normal, obs_dim, action_dim):
mock_normal.return_value = 0.5
env = TfEnv(DummyBoxEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.policies.'
'gaussian_mlp_policy.GaussianMLPModel'),
new=SimpleGaussianMLPModel):
policy = GaussianMLPPolicy(env_spec=env.spec)
env.reset()
obs, _, _, _ = env.step(1)
action, prob = policy.get_action(obs)
expected_action = np.full(action_dim, 0.75)
expected_mean = np.full(action_dim, 0.5)
expected_log_std = np.full(action_dim, np.log(0.5))
assert env.action_space.contains(action)
assert np.array_equal(action, expected_action)
assert np.array_equal(prob['mean'], expected_mean)
assert np.array_equal(prob['log_std'], expected_log_std)
actions, probs = policy.get_actions([obs, obs, obs])
for action, mean, log_std in zip(actions, probs['mean'],
probs['log_std']):
assert env.action_space.contains(action)
assert np.array_equal(action, expected_action)
assert np.array_equal(prob['mean'], expected_mean)
assert np.array_equal(prob['log_std'], expected_log_std)
def test_is_pickleable(self, obs_dim, action_dim):
env = TfEnv(DummyBoxEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.policies.'
'gaussian_mlp_policy.GaussianMLPModel'),
new=SimpleGaussianMLPModel):
policy = GaussianMLPPolicy(env_spec=env.spec)
env.reset()
obs, _, _, _ = env.step(1)
obs_dim = env.spec.observation_space.flat_dim
with tf.compat.v1.variable_scope('GaussianMLPPolicy/GaussianMLPModel',
reuse=True):
return_var = tf.compat.v1.get_variable('return_var')
# assign it to all one
return_var.load(tf.ones_like(return_var).eval())
output1 = self.sess.run(
policy.model.outputs[:-1],
feed_dict={policy.model.input: [obs.flatten()]})
p = pickle.dumps(policy)
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
policy_pickled = pickle.loads(p)
output2 = sess.run(
policy_pickled.model.outputs[:-1],
feed_dict={policy_pickled.model.input: [obs.flatten()]})
assert np.array_equal(output1, output2)
def setup_method(self):
self.env = TfEnv(GridWorldEnv(desc='4x4'))
self.policy = ScriptedPolicy(
scripted_actions=[2, 2, 1, 0, 3, 1, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1])
self.algo = Mock(env_spec=self.env.spec,
policy=self.policy,
max_path_length=16)
def test_get_action(self, mock_rand, obs_dim, action_dim, hidden_dim):
mock_rand.return_value = 0
env = TfEnv(DummyDiscreteEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.policies.'
'categorical_gru_policy.GRUModel'),
new=SimpleGRUModel):
policy = CategoricalGRUPolicy(env_spec=env.spec,
state_include_action=False)
policy.reset()
obs = env.reset()
expected_prob = np.full(action_dim, 0.5)
action, agent_info = policy.get_action(obs)
assert env.action_space.contains(action)
assert action == 0
assert np.array_equal(agent_info['prob'], expected_prob)
actions, agent_infos = policy.get_actions([obs])
for action, prob in zip(actions, agent_infos['prob']):
assert env.action_space.contains(action)
assert action == 0
assert np.array_equal(prob, expected_prob)
def test_trpo_cnn_cubecrash(self):
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
env = TfEnv(normalize(gym.make('CubeCrash-v0')))
policy = CategoricalCNNPolicy(env_spec=env.spec,
conv_filters=(32, 64),
conv_filter_sizes=(8, 4),
conv_strides=(4, 2),
conv_pad='VALID',
hidden_sizes=(32, 32))
baseline = GaussianCNNBaseline(env_spec=env.spec,
regressor_args=dict(
num_filters=(32, 64),
filter_dims=(8, 4),
strides=(4, 2),
padding='VALID',
hidden_sizes=(32, 32),
use_trust_region=True))
algo = TRPO(env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.98,
max_kl_step=0.01,
policy_ent_coeff=0.0,
flatten_input=False)
runner.setup(algo, env)
last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
assert last_avg_ret > -0.9
env.close()
def test_get_action(self, mock_normal, obs_dim, action_dim, hidden_dim):
mock_normal.return_value = 0.5
env = TfEnv(DummyBoxEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.policies.'
'gaussian_lstm_policy.GaussianLSTMModel'),
new=SimpleGaussianLSTMModel):
policy = GaussianLSTMPolicy(env_spec=env.spec,
state_include_action=False)
expected_action = np.full(action_dim, 0.5 * np.exp(0.5) + 0.5)
policy.reset()
obs = env.reset()
action, agent_info = policy.get_action(obs)
assert env.action_space.contains(action)
assert np.allclose(action,
np.full(action_dim, expected_action),
atol=1e-6)
expected_mean = np.full(action_dim, 0.5)
assert np.array_equal(agent_info['mean'], expected_mean)
expected_log_std = np.full(action_dim, 0.5)
assert np.array_equal(agent_info['log_std'], expected_log_std)
actions, agent_infos = policy.get_actions([obs])
for action, mean, log_std in zip(actions, agent_infos['mean'],
agent_infos['log_std']):
assert env.action_space.contains(action)
assert np.allclose(action,
np.full(action_dim, expected_action),
atol=1e-6)
assert np.array_equal(mean, expected_mean)
assert np.array_equal(log_std, expected_log_std)
def test_get_action(self, mock_rand, obs_dim, action_dim, filter_dims,
filter_sizes, strides, padding, hidden_sizes):
mock_rand.return_value = 0
env = TfEnv(DummyDiscreteEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.policies.'
'categorical_cnn_policy.MLPModel'),
new=SimpleMLPModel):
with mock.patch(('metarl.tf.policies.'
'categorical_cnn_policy.CNNModel'),
new=SimpleCNNModel):
policy = CategoricalCNNPolicy(env_spec=env.spec,
conv_filters=filter_dims,
conv_filter_sizes=filter_sizes,
conv_strides=strides,
conv_pad=padding,
hidden_sizes=hidden_sizes)
env.reset()
obs, _, _, _ = env.step(1)
action, prob = policy.get_action(obs)
expected_prob = np.full(action_dim, 0.5)
assert env.action_space.contains(action)
assert action == 0
assert np.array_equal(prob['prob'], expected_prob)
actions, probs = policy.get_actions([obs, obs, obs])
for action, prob in zip(actions, probs['prob']):
assert env.action_space.contains(action)
assert action == 0
assert np.array_equal(prob, expected_prob)
def setup_method(self):
super().setup_method()
self.env = TfEnv(DummyDiscreteEnv())
self.qf = SimpleQFunction(self.env.spec)
self.policy = DiscreteQfDerivedPolicy(env_spec=self.env.spec,
qf=self.qf)
self.sess.run(tf.compat.v1.global_variables_initializer())
self.env.reset()
def setup_method(self):
ray.init(local_mode=True, ignore_reinit_error=True)
self.env = TfEnv(GridWorldEnv(desc='4x4'))
self.policy = ScriptedPolicy(
scripted_actions=[2, 2, 1, 0, 3, 1, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1])
self.algo = Mock(env_spec=self.env.spec,
policy=self.policy,
max_path_length=16)
class TestSampler:
"""
Uses mock policy for 4x4 gridworldenv
'4x4': [
'SFFF',
'FHFH',
'FFFH',
'HFFG'
]
0: left
1: down
2: right
3: up
-1: no move
'S' : starting point
'F' or '.': free space
'W' or 'x': wall
'H' or 'o': hole (terminates episode)
'G' : goal
[2,2,1,0,3,1,1,1,2,2,1,1,1,2,2,1]
"""
def setup_method(self):
self.env = TfEnv(GridWorldEnv(desc='4x4'))
self.policy = ScriptedPolicy(
scripted_actions=[2, 2, 1, 0, 3, 1, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1])
self.algo = Mock(env_spec=self.env.spec,
policy=self.policy,
max_path_length=16)
def teardown_method(self):
self.env.close()
def test_ray_batch_sampler(self):
sampler1 = SimpleSampler(self.)
sampler1.start_worker()
sampler2 = OnPolicyVectorizedSampler(self.algo, self.env)
sampler2.start_worker()
trajs1 = sampler1.obtain_samples(0, 16)
trajs2 = sampler2.obtain_samples(0, 1)
assert (trajs1[0]['observations'].shape == np.array(
trajs2[0]['observations']).shape == (6, ))
traj2_action_shape = np.array(trajs2[0]['actions']).shape
assert (trajs1[0]['actions'].shape == traj2_action_shape == (6, ))
assert (sum(trajs1[0]['rewards']) == sum(trajs2[0]['rewards']) == 1)
true_obs = np.array([0, 1, 2, 6, 10, 14])
true_actions = np.array([2, 2, 1, 1, 1, 2])
true_rewards = np.array([0, 0, 0, 0, 0, 1])
for trajectory in trajs1:
assert (np.array_equal(trajectory['observations'], true_obs))
assert (np.array_equal(trajectory['actions'], true_actions))
assert (np.array_equal(trajectory['rewards'], true_rewards))
sampler1.shutdown_worker()
sampler2.shutdown_worker()
def test_baseline(self):
"""Test the baseline initialization."""
box_env = TfEnv(DummyBoxEnv())
deterministic_mlp_baseline = ContinuousMLPBaseline(env_spec=box_env)
gaussian_mlp_baseline = GaussianMLPBaseline(env_spec=box_env)
self.sess.run(tf.compat.v1.global_variables_initializer())
deterministic_mlp_baseline.get_param_values()
gaussian_mlp_baseline.get_param_values()
box_env.close()
def test_output_shape_dueling(self, obs_dim, action_dim):
env = TfEnv(DummyDiscreteEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.q_functions.'
'discrete_mlp_q_function.MLPDuelingModel'),
new=SimpleMLPModel):
qf = DiscreteMLPQFunction(env_spec=env.spec, dueling=True)
env.reset()
obs, _, _, _ = env.step(1)
outputs = self.sess.run(qf.q_vals, feed_dict={qf.input: [obs]})
assert outputs.shape == (1, action_dim)
def test_dqn_cartpole_pickle(self):
"""Test DQN with CartPole environment."""
with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
n_epochs = 10
steps_per_epoch = 10
sampler_batch_size = 500
num_timesteps = n_epochs * steps_per_epoch * sampler_batch_size
env = TfEnv(gym.make('CartPole-v0'))
replay_buffer = SimpleReplayBuffer(env_spec=env.spec,
size_in_transitions=int(1e4),
time_horizon=1)
qf = DiscreteMLPQFunction(env_spec=env.spec, hidden_sizes=(64, 64))
policy = DiscreteQfDerivedPolicy(env_spec=env.spec, qf=qf)
epilson_greedy_strategy = EpsilonGreedyStrategy(
env_spec=env.spec,
total_timesteps=num_timesteps,
max_epsilon=1.0,
min_epsilon=0.02,
decay_ratio=0.1)
algo = DQN(env_spec=env.spec,
policy=policy,
qf=qf,
exploration_strategy=epilson_greedy_strategy,
replay_buffer=replay_buffer,
qf_lr=1e-4,
discount=1.0,
min_buffer_size=int(1e3),
double_q=False,
n_train_steps=500,
grad_norm_clipping=5.0,
steps_per_epoch=steps_per_epoch,
target_network_update_freq=1,
buffer_batch_size=32)
runner.setup(algo, env)
with tf.compat.v1.variable_scope(
'DiscreteMLPQFunction/MLPModel/mlp/hidden_0', reuse=True):
bias = tf.compat.v1.get_variable('bias')
# assign it to all one
old_bias = tf.ones_like(bias).eval()
bias.load(old_bias)
h = pickle.dumps(algo)
with tf.compat.v1.Session(graph=tf.Graph()):
pickle.loads(h)
with tf.compat.v1.variable_scope(
'DiscreteMLPQFunction/MLPModel/mlp/hidden_0',
reuse=True):
new_bias = tf.compat.v1.get_variable('bias')
new_bias = new_bias.eval()
assert np.array_equal(old_bias, new_bias)
env.close()
def setup_method(self):
super().setup_method()
self.env = TfEnv(normalize(gym.make('InvertedDoublePendulum-v2')))
self.policy = GaussianMLPPolicy(
env_spec=self.env.spec,
hidden_sizes=(64, 64),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None,
)
self.baseline = GaussianMLPBaseline(
env_spec=self.env.spec,
regressor_args=dict(hidden_sizes=(32, 32)),
)
def test_dist_info(self, obs_dim, action_dim):
env = TfEnv(DummyDiscreteEnv(obs_dim=obs_dim, action_dim=action_dim))
with mock.patch(('metarl.tf.policies.'
'categorical_mlp_policy.MLPModel'),
new=SimpleMLPModel):
policy = CategoricalMLPPolicy(env_spec=env.spec)
env.reset()
obs, _, _, _ = env.step(1)
expected_prob = np.full(action_dim, 0.5)
policy_probs = policy.dist_info([obs.flatten()])
assert np.array_equal(policy_probs['prob'][0], expected_prob)
def run_task(self, snapshot_config, *_):
config = tf.ConfigProto(device_count={'GPU': 0},
allow_soft_placement=True,
intra_op_parallelism_threads=12,
inter_op_parallelism_threads=12)
sess = tf.Session(config=config)
with LocalTFRunner(snapshot_config=snapshot_config,
sess=sess) as runner:
env = gym.make(self._env)
env = TfEnv(normalize(env))
env.reset()
policy = GaussianGRUPolicy(
env_spec=env.spec,
hidden_dim=32,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None,
)
baseline = GaussianMLPBaseline(
env_spec=env.spec,
regressor_args=dict(
hidden_sizes=(64, 64),
use_trust_region=False,
optimizer=FirstOrderOptimizer,
optimizer_args=dict(
batch_size=32,
max_epochs=10,
tf_optimizer_args=dict(learning_rate=1e-3),
),
),
)
algo = PPO(
env_spec=env.spec,
policy=policy,
baseline=baseline,
max_path_length=100,
discount=0.99,
gae_lambda=0.95,
lr_clip_range=0.2,
policy_ent_coeff=0.0,
optimizer_args=dict(
batch_size=32,
max_epochs=10,
tf_optimizer_args=dict(learning_rate=1e-3),
),
)
runner.setup(algo, env, sampler_args=dict(n_envs=12))
runner.train(n_epochs=5, batch_size=2048)
