def run_train_task(vv):
env = TfEnv(normalize(vv['env'](
log_scale_limit=vv['log_scale_limit'],
target_velocity=vv['target_velocity'],
)))
dynamics_model = MLPDynamicsEnsemble(
name="dyn_model",
env_spec=env.spec,
hidden_sizes=vv['hidden_sizes_model'],
weight_normalization=vv['weight_normalization_model'],
num_models=vv['num_models'],
optimizer=vv['optimizer_model'],
valid_split_ratio=vv['valid_split_ratio'],
rolling_average_persitency=vv['rolling_average_persitency']
)
policy = MAMLGaussianMLPPolicy(
name="policy",
env_spec=env.spec,
hidden_sizes=vv['hidden_sizes_policy'],
hidden_nonlinearity=vv['hidden_nonlinearity_policy'],
grad_step_size=vv['fast_lr'],
trainable_step_size=vv['trainable_step_size'],
bias_transform=vv['bias_transform'],
num_tasks=vv['meta_batch_size']
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = ModelMAMLTRPO(
env=env,
policy=policy,
dynamics_model=dynamics_model,
baseline=baseline,
n_itr=vv['n_itr'],
n_iter=vv['n_itr'],
batch_size_env_samples=vv['batch_size_env_samples'],
batch_size_dynamics_samples=vv['batch_size_dynamics_samples'],
meta_batch_size=vv['meta_batch_size'],
initial_random_samples=vv['initial_random_samples'],
num_maml_steps_per_iter=vv['num_maml_steps_per_iter'],
reset_from_env_traj=vv.get('reset_from_env_traj', False),
max_path_length_env=vv['path_length_env'],
max_path_length_dyn=vv.get('path_length_dyn', None),
dynamic_model_max_epochs=vv.get('dynamic_model_max_epochs', (500, 500)),
discount=vv['discount'],
step_size=vv["meta_step_size"],
num_grad_updates=1,
retrain_model_when_reward_decreases=vv['retrain_model_when_reward_decreases'],
reset_policy_std=vv['reset_policy_std'],
reinit_model_cycle=vv['reinit_model_cycle'],
frac_gpu=vv.get('frac_gpu', 0.85),
log_real_performance=True,
clip_obs=vv.get('clip_obs', True),
entropy_bonus=vv['entropy_bonus'],
tailored_exploration=vv['tailored_exploration']
)
algo.train()
def run_train_task(vv):
import sys
print(vv['exp_prefix'])
sysout_log_path = os.path.join(config.LOG_DIR, 'local', vv['exp_prefix'],
vv['exp_name'], 'stdout.log')
sysout_log_file = open(sysout_log_path, 'w')
sys.stdout = sysout_log_file
env = TfEnv(normalize(vv['env'](log_scale_limit=vv['log_scale_limit'])))
dynamics_model = MLPDynamicsEnsemble(
name="dyn_model",
env_spec=env.spec,
hidden_sizes=vv['hidden_sizes_model'],
weight_normalization=vv['weight_normalization_model'],
num_models=vv['num_models'],
optimizer=vv['optimizer_model'],
valid_split_ratio=vv['valid_split_ratio'],
rolling_average_persitency=vv['rolling_average_persitency'])
policy = MAMLImprovedGaussianMLPPolicy(
name="policy",
env_spec=env.spec,
hidden_sizes=vv['hidden_sizes_policy'],
hidden_nonlinearity=vv['hidden_nonlinearity_policy'],
grad_step_size=vv['fast_lr'],
trainable_step_size=vv['trainable_step_size'],
bias_transform=vv['bias_transform'],
param_noise_std=vv['param_noise_std'])
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = ModelMAMLTRPO(
env=env,
policy=policy,
dynamics_model=dynamics_model,
baseline=baseline,
n_itr=vv['n_itr'],
n_iter=vv['n_itr'],
batch_size_env_samples=vv['batch_size_env_samples'],
batch_size_dynamics_samples=vv['batch_size_dynamics_samples'],
meta_batch_size=vv['meta_batch_size'],
initial_random_samples=vv['initial_random_samples'],
num_maml_steps_per_iter=vv['num_maml_steps_per_iter'],
reset_from_env_traj=vv.get('reset_from_env_traj', False),
max_path_length_env=vv['path_length_env'],
max_path_length_dyn=vv.get('path_length_dyn', None),
discount=vv['discount'],
step_size=vv["meta_step_size"],
num_grad_updates=1,
retrain_model_when_reward_decreases=vv[
'retrain_model_when_reward_decreases'],
reset_policy_std=vv['reset_policy_std'],
reinit_model_cycle=vv['reinit_model_cycle'],
frac_gpu=vv.get('frac_gpu', 0.85),
clip_obs=vv.get('clip_obs', True))
algo.train()
sysout_log_file.close()
def run_train_task(vv):
env = TfEnv(normalize(vv['env'](log_scale_limit=vv['log_scale_limit'])))
dynamics_model = MLPDynamicsEnsemble(
name="dyn_model",
env_spec=env.spec,
hidden_sizes=vv['hidden_sizes_model'],
weight_normalization=vv['weight_normalization_model'],
num_models=vv['num_models'],
optimizer=vv['optimizer_model'],
)
policy = MAMLImprovedGaussianMLPPolicy(
name="policy",
env_spec=env.spec,
hidden_sizes=vv['hidden_sizes_policy'],
hidden_nonlinearity=vv['hidden_nonlinearity_policy'],
grad_step_size=vv['fast_lr'],
trainable_step_size=vv['trainable_step_size'],
bias_transform=vv['bias_transform'],
param_noise_std=vv['param_noise_std'])
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = ModelMAMLTRPO(
env=env,
policy=policy,
dynamics_model=dynamics_model,
baseline=baseline,
n_itr=vv['n_itr'],
batch_size_env_samples=vv['batch_size_env_samples'],
batch_size_dynamics_samples=vv['batch_size_dynamics_samples'],
meta_batch_size=vv['meta_batch_size'],
initial_random_samples=vv['initial_random_samples'],
dynamic_model_epochs=vv['dynamic_model_epochs'],
num_maml_steps_per_iter=vv['num_maml_steps_per_iter'],
reset_from_env_traj=vv['reset_from_env_traj'],
max_path_length_env=vv['path_length_env'],
max_path_length_dyn=vv['path_length_dyn'],
discount=vv['discount'],
step_size=vv["meta_step_size"],
num_grad_updates=1,
retrain_model_when_reward_decreases=vv[
'retrain_model_when_reward_decreases'],
reset_policy_std=vv['reset_policy_std'],
reinit_model_cycle=vv['reinit_model_cycle'],
frac_gpu=vv.get('frac_gpu', 0.85),
)
algo.train()
def test_policy_sampling(self):
# get from data
env = PointEnv()
paths = sample_random_trajectories_point_env(env, num_paths=100, horizon=100)
dynamics_model = MLPDynamicsEnsemble("dyn_model1", env, hidden_sizes=(16,16))
obs = np.concatenate([path['observations'] for path in paths], axis=0)
obs_next = np.concatenate([path['next_observations'] for path in paths], axis=0)
act = np.concatenate([path['actions'] for path in paths], axis=0)
env = TfEnv(normalize(PointEnv()))
policy = GaussianMLPPolicy(
name="policy",
env_spec=env.spec,
hidden_sizes=(16, 16),
hidden_nonlinearity=tf.nn.tanh
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
# fit dynamics model
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
dynamics_model.fit(obs, act, obs_next, epochs=5)
algo = ModelMAMLTRPO(
env=env,
dynamics_model=dynamics_model,
policy=policy,
baseline=baseline,
batch_size=20000,
max_path_length=100,
n_itr=10,
discount=0.99,
step_size=0.01,
)
algo.dynamics_model = dynamics_model
itr = 1
model_sampler = ModelVectorizedSampler(algo)
model_sampler.start_worker()
paths = model_sampler.obtain_samples(itr)
samples_data = model_sampler.process_samples(itr, paths)
print(samples_data.keys())