def test_sprl(ex_dir, env: SimEnv, optimize_mean: bool):
pyrado.set_seed(0)
env = ActNormWrapper(env)
env_sprl_params = [
dict(
name="gravity_const",
target_mean=to.tensor([9.81]),
target_cov_chol_flat=to.tensor([1.0]),
init_mean=to.tensor([9.81]),
init_cov_chol_flat=to.tensor([0.05]),
)
]
radnomizer = DomainRandomizer(
*[SelfPacedDomainParam(**p) for p in env_sprl_params])
env = DomainRandWrapperLive(env, randomizer=radnomizer)
policy = FNNPolicy(env.spec, hidden_sizes=[64, 64], hidden_nonlin=to.tanh)
vfcn_hparam = dict(hidden_sizes=[32, 32], hidden_nonlin=to.relu)
vfcn = FNNPolicy(spec=EnvSpec(env.obs_space, ValueFunctionSpace),
**vfcn_hparam)
critic_hparam = dict(
gamma=0.9844534412010116,
lamda=0.9710614403461155,
num_epoch=10,
batch_size=150,
standardize_adv=False,
lr=0.00016985313083236645,
)
critic = GAE(vfcn, **critic_hparam)
subrtn_hparam = dict(
max_iter=1,
eps_clip=0.12648736789309026,
min_steps=10 * env.max_steps,
num_epoch=3,
batch_size=150,
std_init=0.7573286998997557,
lr=6.999956625305722e-04,
max_grad_norm=1.0,
num_workers=1,
)
algo_hparam = dict(
kl_constraints_ub=8000,
performance_lower_bound=500,
std_lower_bound=0.4,
kl_threshold=200,
max_iter=1,
optimize_mean=optimize_mean,
)
algo = SPRL(env, PPO(ex_dir, env, policy, critic, **subrtn_hparam),
**algo_hparam)
algo.train(snapshot_mode="latest")
assert algo.curr_iter == algo.max_iter
def test_spota_ppo(ex_dir, env: SimEnv, spota_hparam: dict):
pyrado.set_seed(0)
# Environment and domain randomization
randomizer = create_default_randomizer(env)
env = DomainRandWrapperBuffer(env, randomizer)
# Policy and subroutines
policy = FNNPolicy(env.spec, [16, 16], hidden_nonlin=to.tanh)
vfcn = FNN(input_size=env.obs_space.flat_dim,
output_size=1,
hidden_sizes=[16, 16],
hidden_nonlin=to.tanh)
critic_hparam = dict(gamma=0.998,
lamda=0.95,
num_epoch=3,
batch_size=64,
lr=1e-3)
critic_cand = GAE(vfcn, **critic_hparam)
critic_refs = GAE(deepcopy(vfcn), **critic_hparam)
subrtn_hparam_common = dict(
# min_rollouts=0, # will be overwritten by SPOTA
min_steps=0, # will be overwritten by SPOTA
max_iter=2,
num_epoch=3,
eps_clip=0.1,
batch_size=64,
num_workers=1,
std_init=0.5,
lr=1e-2,
)
sr_cand = PPO(ex_dir, env, policy, critic_cand, **subrtn_hparam_common)
sr_refs = PPO(ex_dir, env, deepcopy(policy), critic_refs,
**subrtn_hparam_common)
# Create algorithm and train
algo = SPOTA(ex_dir, env, sr_cand, sr_refs, **spota_hparam)
algo.train()
assert algo.curr_iter == algo.max_iter or algo.stopping_criterion_met()
def test_arpl(ex_dir, env: SimEnv):
pyrado.set_seed(0)
env = ActNormWrapper(env)
env = StateAugmentationWrapper(env, domain_param=None)
policy = FNNPolicy(env.spec, hidden_sizes=[16, 16], hidden_nonlin=to.tanh)
vfcn_hparam = dict(hidden_sizes=[32, 32], hidden_nonlin=to.tanh)
vfcn = FNNPolicy(spec=EnvSpec(env.obs_space, ValueFunctionSpace),
**vfcn_hparam)
critic_hparam = dict(
gamma=0.9844534412010116,
lamda=0.9710614403461155,
num_epoch=10,
batch_size=150,
standardize_adv=False,
lr=0.00016985313083236645,
)
critic = GAE(vfcn, **critic_hparam)
algo_hparam = dict(
max_iter=2,
min_steps=23 * env.max_steps,
min_rollouts=None,
num_epoch=5,
eps_clip=0.085,
batch_size=150,
std_init=0.995,
lr=2e-4,
num_workers=1,
)
arpl_hparam = dict(
max_iter=2,
steps_num=23 * env.max_steps,
halfspan=0.05,
dyn_eps=0.07,
dyn_phi=0.25,
obs_phi=0.1,
obs_eps=0.05,
proc_phi=0.1,
proc_eps=0.03,
torch_observation=True,
)
ppo = PPO(ex_dir, env, policy, critic, **algo_hparam)
algo = ARPL(ex_dir, env, ppo, policy, ppo.expl_strat, **arpl_hparam)
algo.train(snapshot_mode="best")
)
]
env = DomainRandWrapperLive(
env,
randomizer=DomainRandomizer(
*[SelfPacedDomainParam(**p) for p in env_sprl_params]))
sprl_hparam = dict(
kl_constraints_ub=8000,
performance_lower_bound=500,
std_lower_bound=0.4,
kl_threshold=200,
max_iter=args.sprl_iterations,
optimize_mean=not args.cov_only,
)
algo = SPRL(env, PPO(ex_dir, env, policy, critic, **algo_hparam),
**sprl_hparam)
# Save the hyper-parameters
save_dicts_to_yaml(
dict(env=env_hparams, seed=args.seed),
dict(policy=policy_hparam),
dict(critic=critic_hparam, vfcn=vfcn_hparam),
dict(subrtn=algo_hparam, subrtn_name=PPO.name),
dict(algo=sprl_hparam,
algo_name=algo.name,
env_sprl_params=env_sprl_params),
save_dir=ex_dir,
)
# Jeeeha
)
critic = GAE(vfcn, **critic_hparam)
# Subroutine
algo_hparam = dict(
max_iter=200 if policy.name == FNNPolicy.name else 75,
eps_clip=0.12648736789309026,
min_steps=30 * env.max_steps,
num_epoch=7,
batch_size=500,
std_init=0.7573286998997557,
lr=6.999956625305722e-04,
max_grad_norm=1.0,
num_workers=8,
lr_scheduler=lr_scheduler.ExponentialLR,
lr_scheduler_hparam=dict(gamma=0.999),
)
algo = PPO(ex_dir, env, policy, critic, **algo_hparam)
# Save the hyper-parameters
save_dicts_to_yaml(
dict(env=env_hparams, seed=args.seed),
dict(policy=policy_hparam),
dict(critic=critic_hparam, vfcn=vfcn_hparam),
dict(algo=algo_hparam, algo_name=algo.name),
save_dir=ex_dir,
)
# Jeeeha
algo.train(snapshot_mode="latest", seed=args.seed)
)
critic = GAE(vfcn, **critic_hparam)
subrtn_policy_hparam = dict(
max_iter=200,
eps_clip=0.12648736789309026,
min_steps=30 * env_sim.max_steps,
num_epoch=7,
batch_size=500,
std_init=0.7573286998997557,
lr=6.999956625305722e-04,
max_grad_norm=1.0,
num_workers=num_workers,
lr_scheduler=lr_scheduler.ExponentialLR,
lr_scheduler_hparam=dict(gamma=0.999),
)
subrtn_policy = PPO(ex_dir, env_sim, behav_policy, critic,
**subrtn_policy_hparam)
# Subroutine for system identification
prior = DomainRandomizer(
NormalDomainParam(name="mass_rot_pole", mean=0.095, std=0.095 / 10),
NormalDomainParam(name="mass_pend_pole", mean=0.024, std=0.024 / 10),
NormalDomainParam(name="length_rot_pole", mean=0.085, std=0.085 / 10),
NormalDomainParam(name="length_pend_pole", mean=0.129, std=0.129 / 10),
)
ddp_policy_hparam = dict(mapping=dp_map,
trafo_mask=trafo_mask,
scale_params=False)
ddp_policy = DomainDistrParamPolicy(prior=prior, **ddp_policy_hparam)
subsubrtn_distr_hparam = dict(
max_iter=10,
pop_size=None,
# Subroutine
subrtn_hparam = dict(
max_iter=200,
eps_clip=0.12648736789309026,
min_steps=30 * env_sim.max_steps,
num_epoch=7,
batch_size=500,
std_init=0.7573286998997557,
lr=6.999956625305722e-04,
max_grad_norm=1.0,
num_workers=8,
lr_scheduler=lr_scheduler.ExponentialLR,
lr_scheduler_hparam=dict(gamma=0.999),
)
subrtn = PPO(ex_dir, env_sim, policy, critic, **subrtn_hparam)
# Set the boundaries for the GP
dp_nom = QQubeSwingUpSim.get_nominal_domain_param()
ddp_space = BoxSpace(
bound_lo=np.array([
0.8 * dp_nom["mass_pend_pole"],
dp_nom["mass_pend_pole"] / 5000,
0.8 * dp_nom["mass_rot_pole"],
dp_nom["mass_rot_pole"] / 5000,
0.8 * dp_nom["length_pend_pole"],
dp_nom["length_pend_pole"] / 5000,
0.8 * dp_nom["length_rot_pole"],
dp_nom["length_rot_pole"] / 5000,
]),
bound_up=np.array([
standardizer=None,
max_grad_norm=1.0,
lr=5e-4,
)
critic = GAE(vfcn, **critic_hparam)
# Algorithm
algo_hparam = dict(
max_iter=500,
min_steps=20 * env.max_steps,
num_epoch=10,
eps_clip=0.15,
batch_size=512,
max_grad_norm=1.0,
lr=3e-4,
num_workers=12,
)
algo = PPO(ex_dir, env, policy, critic, **algo_hparam)
# Save the hyper-parameters
save_dicts_to_yaml(
dict(env=env_hparams, seed=args.seed),
dict(policy=policy_hparam),
dict(critic=critic_hparam, vfcn=vfcn_hparam),
dict(algo=algo_hparam, algo_name=algo.name),
save_dir=ex_dir,
)
# Jeeeha
algo.train(seed=args.seed, snapshot_mode="best")
def test_basic_meta(ex_dir, policy, env: SimEnv, algo, algo_hparam: dict):
pyrado.set_seed(0)
# Policy and subroutine
env = GaussianObsNoiseWrapper(
env,
noise_std=[
1 / 180 * np.pi,
1 / 180 * np.pi,
0.0025,
0.0025,
2 / 180 * np.pi,
2 / 180 * np.pi,
0.05,
0.05,
],
)
env = ActNormWrapper(env)
env = ActDelayWrapper(env)
randomizer = create_default_randomizer_qbb()
randomizer.add_domain_params(
UniformDomainParam(name="act_delay",
mean=15,
halfspan=15,
clip_lo=0,
roundint=True))
env = DomainRandWrapperLive(env, randomizer)
# Policy
policy_hparam = dict(hidden_sizes=[16, 16], hidden_nonlin=to.tanh) # FNN
policy = FNNPolicy(spec=env.spec, **policy_hparam)
# Critic
vfcn_hparam = dict(hidden_sizes=[16, 16], hidden_nonlin=to.tanh) # FNN
vfcn = FNNPolicy(spec=EnvSpec(env.obs_space, ValueFunctionSpace),
**vfcn_hparam)
critic_hparam = dict(
gamma=0.9995,
lamda=0.98,
num_epoch=2,
batch_size=64,
lr=5e-4,
standardize_adv=False,
)
critic = GAE(vfcn, **critic_hparam)
subrtn_hparam = dict(
max_iter=3,
min_rollouts=5,
num_epoch=2,
eps_clip=0.1,
batch_size=64,
std_init=0.8,
lr=2e-4,
num_workers=1,
)
subrtn = PPO(ex_dir, env, policy, critic, **subrtn_hparam)
algo = algo(env, subrtn, **algo_hparam)
algo.train()
assert algo.curr_iter == algo.max_iter
def test_simopt_cem_ppo(ex_dir, env: SimEnv):
pyrado.set_seed(0)
# Environments
env_real = deepcopy(env)
env_real = ActNormWrapper(env_real)
env_sim = ActNormWrapper(env)
randomizer = DomainRandomizer(
NormalDomainParam(name="mass_rot_pole",
mean=0.0,
std=1e6,
clip_lo=1e-3),
NormalDomainParam(name="mass_pend_pole",
mean=0.0,
std=1e6,
clip_lo=1e-3),
NormalDomainParam(name="length_rot_pole",
mean=0.0,
std=1e6,
clip_lo=1e-3),
NormalDomainParam(name="length_pend_pole",
mean=0.0,
std=1e6,
clip_lo=1e-3),
)
env_sim = DomainRandWrapperLive(env_sim, randomizer)
dp_map = {
0: ("mass_rot_pole", "mean"),
1: ("mass_rot_pole", "std"),
2: ("mass_pend_pole", "mean"),
3: ("mass_pend_pole", "std"),
4: ("length_rot_pole", "mean"),
5: ("length_rot_pole", "std"),
6: ("length_pend_pole", "mean"),
7: ("length_pend_pole", "std"),
}
trafo_mask = [True] * 8
env_sim = MetaDomainRandWrapper(env_sim, dp_map)
# Subroutine for policy improvement
behav_policy_hparam = dict(hidden_sizes=[16, 16], hidden_nonlin=to.tanh)
behav_policy = FNNPolicy(spec=env_sim.spec, **behav_policy_hparam)
vfcn_hparam = dict(hidden_sizes=[16, 16], hidden_nonlin=to.relu)
vfcn = FNNPolicy(spec=EnvSpec(env_sim.obs_space, ValueFunctionSpace),
**vfcn_hparam)
critic_hparam = dict(
gamma=0.99,
lamda=0.98,
num_epoch=2,
batch_size=128,
standardize_adv=True,
lr=8e-4,
max_grad_norm=5.0,
)
critic = GAE(vfcn, **critic_hparam)
subrtn_policy_hparam = dict(
max_iter=2,
eps_clip=0.13,
min_steps=4 * env_sim.max_steps,
num_epoch=3,
batch_size=128,
std_init=0.75,
lr=3e-04,
max_grad_norm=1.0,
num_workers=1,
)
subrtn_policy = PPO(ex_dir, env_sim, behav_policy, critic,
**subrtn_policy_hparam)
prior = DomainRandomizer(
NormalDomainParam(name="mass_rot_pole", mean=0.095, std=0.095 / 10),
NormalDomainParam(name="mass_pend_pole", mean=0.024, std=0.024 / 10),
NormalDomainParam(name="length_rot_pole", mean=0.085, std=0.085 / 10),
NormalDomainParam(name="length_pend_pole", mean=0.129, std=0.129 / 10),
)
ddp_policy_hparam = dict(mapping=dp_map,
trafo_mask=trafo_mask,
scale_params=True)
ddp_policy = DomainDistrParamPolicy(prior=prior, **ddp_policy_hparam)
subsubrtn_distr_hparam = dict(
max_iter=2,
pop_size=10,
num_init_states_per_domain=1,
num_is_samples=8,
expl_std_init=1e-2,
expl_std_min=1e-5,
extra_expl_std_init=1e-2,
extra_expl_decay_iter=5,
num_workers=1,
)
subsubrtn_distr = CEM(ex_dir, env_sim, ddp_policy,
**subsubrtn_distr_hparam)
subrtn_distr_hparam = dict(
metric=None,
obs_dim_weight=[1, 1, 1, 1, 10, 10],
num_rollouts_per_distr=3,
num_workers=1,
)
subrtn_distr = SysIdViaEpisodicRL(subsubrtn_distr,
behavior_policy=behav_policy,
**subrtn_distr_hparam)
# Algorithm
algo_hparam = dict(
max_iter=1,
num_eval_rollouts=5,
warmstart=True,
)
algo = SimOpt(ex_dir, env_sim, env_real, subrtn_policy, subrtn_distr,
**algo_hparam)
algo.train()
assert algo.curr_iter == algo.max_iter
def train_and_eval(trial: optuna.Trial, study_dir: str, seed: int):
"""
Objective function for the Optuna `Study` to maximize.
.. note::
Optuna expects only the `trial` argument, thus we use `functools.partial` to sneak in custom arguments.
:param trial: Optuna Trial object for hyper-parameter optimization
:param study_dir: the parent directory for all trials in this study
:param seed: seed value for the random number generators, pass `None` for no seeding
:return: objective function value
"""
# Synchronize seeds between Optuna trials
pyrado.set_seed(seed)
# Environment
env = QBallBalancerSim(dt=1 / 250.0, max_steps=1500)
env = ActNormWrapper(env)
# Learning rate scheduler
lrs_gamma = trial.suggest_categorical("exp_lr_scheduler_gamma",
[None, 0.99, 0.995, 0.999])
if lrs_gamma is not None:
lr_sched = lr_scheduler.ExponentialLR
lr_sched_hparam = dict(gamma=lrs_gamma)
else:
lr_sched, lr_sched_hparam = None, dict()
# Policy
policy = FNNPolicy(
spec=env.spec,
hidden_sizes=trial.suggest_categorical("hidden_sizes_policy",
[(16, 16), (32, 32), (64, 64)]),
hidden_nonlin=fcn_from_str(
trial.suggest_categorical("hidden_nonlin_policy",
["to_tanh", "to_relu"])),
)
# Critic
vfcn = FNN(
input_size=env.obs_space.flat_dim,
output_size=1,
hidden_sizes=trial.suggest_categorical("hidden_sizes_critic",
[(16, 16), (32, 32), (64, 64)]),
hidden_nonlin=fcn_from_str(
trial.suggest_categorical("hidden_nonlin_critic",
["to_tanh", "to_relu"])),
)
critic_hparam = dict(
batch_size=250,
gamma=trial.suggest_uniform("gamma_critic", 0.99, 1.0),
lamda=trial.suggest_uniform("lamda_critic", 0.95, 1.0),
num_epoch=trial.suggest_int("num_epoch_critic", 1, 10),
lr=trial.suggest_loguniform("lr_critic", 1e-5, 1e-3),
standardize_adv=trial.suggest_categorical("standardize_adv_critic",
[True, False]),
max_grad_norm=trial.suggest_categorical("max_grad_norm_critic",
[None, 1.0, 5.0]),
lr_scheduler=lr_sched,
lr_scheduler_hparam=lr_sched_hparam,
)
critic = GAE(vfcn, **critic_hparam)
# Algorithm
algo_hparam = dict(
num_workers=1, # parallelize via optuna n_jobs
max_iter=300,
batch_size=250,
min_steps=trial.suggest_int("num_rollouts_algo", 10, 30) *
env.max_steps,
num_epoch=trial.suggest_int("num_epoch_algo", 1, 10),
eps_clip=trial.suggest_uniform("eps_clip_algo", 0.05, 0.2),
std_init=trial.suggest_uniform("std_init_algo", 0.5, 1.0),
lr=trial.suggest_loguniform("lr_algo", 1e-5, 1e-3),
max_grad_norm=trial.suggest_categorical("max_grad_norm_algo",
[None, 1.0, 5.0]),
lr_scheduler=lr_sched,
lr_scheduler_hparam=lr_sched_hparam,
)
algo = PPO(osp.join(study_dir, f"trial_{trial.number}"), env, policy,
critic, **algo_hparam)
# Train without saving the results
algo.train(snapshot_mode="latest", seed=seed)
# Evaluate
min_rollouts = 1000
sampler = ParallelRolloutSampler(env,
policy,
num_workers=1,
min_rollouts=min_rollouts)
ros = sampler.sample()
mean_ret = sum([r.undiscounted_return() for r in ros]) / min_rollouts
return mean_ret
critic_refs = GAE(deepcopy(vfcn), **critic_hparam)
subrtn_hparam_cand = dict(
max_iter=400,
# min_rollouts=0, # will be overwritten by SPOTA
min_steps=0, # will be overwritten by SPOTA
num_epoch=1,
eps_clip=0.1,
batch_size=100,
std_init=0.8,
max_grad_norm=1.0,
lr=1e-4,
)
subrtn_hparam_refs = deepcopy(subrtn_hparam_cand)
sr_cand = PPO(ex_dir, env, policy, critic_cand, **subrtn_hparam_cand)
sr_refs = PPO(ex_dir, env, deepcopy(policy), critic_refs, **subrtn_hparam_refs)
# Meta-Algorithm
algo_hparam = dict(
max_iter=10,
alpha=0.05,
beta=0.1,
nG=20,
nJ=180,
ntau=5,
nc_init=10,
nr_init=1,
sequence_cand=sequence_add_init,
sequence_refs=sequence_const,
warmstart_cand=True,
def train_and_eval(trial: optuna.Trial, study_dir: str, seed: int):
"""
Objective function for the Optuna `Study` to maximize.
.. note::
Optuna expects only the `trial` argument, thus we use `functools.partial` to sneak in custom arguments.
:param trial: Optuna Trial object for hyper-parameter optimization
:param study_dir: the parent directory for all trials in this study
:param seed: seed value for the random number generators, pass `None` for no seeding
:return: objective function value
"""
# Synchronize seeds between Optuna trials
pyrado.set_seed(seed)
# Environments
env_hparams = dict(dt=1 / 100., max_steps=600)
env_real = QQubeSwingUpSim(**env_hparams)
env_real.domain_param = dict(
Mr=0.095 * 0.9, # 0.095*0.9 = 0.0855
Mp=0.024 * 1.1, # 0.024*1.1 = 0.0264
Lr=0.085 * 0.9, # 0.085*0.9 = 0.0765
Lp=0.129 * 1.1, # 0.129*1.1 = 0.1419
)
env_sim = QQubeSwingUpSim(**env_hparams)
randomizer = DomainRandomizer(
NormalDomainParam(name='Mr', mean=0., std=1e6, clip_lo=1e-3),
NormalDomainParam(name='Mp', mean=0., std=1e6, clip_lo=1e-3),
NormalDomainParam(name='Lr', mean=0., std=1e6, clip_lo=1e-3),
NormalDomainParam(name='Lp', mean=0., std=1e6, clip_lo=1e-3),
)
env_sim = DomainRandWrapperLive(env_sim, randomizer)
dp_map = {
0: ('Mr', 'mean'),
1: ('Mr', 'std'),
2: ('Mp', 'mean'),
3: ('Mp', 'std'),
4: ('Lr', 'mean'),
5: ('Lr', 'std'),
6: ('Lp', 'mean'),
7: ('Lp', 'std')
}
trafo_mask = [True] * 8
env_sim = MetaDomainRandWrapper(env_sim, dp_map)
# Subroutine for policy improvement
behav_policy_hparam = dict(hidden_sizes=[64, 64], hidden_nonlin=to.tanh)
behav_policy = FNNPolicy(spec=env_sim.spec, **behav_policy_hparam)
vfcn_hparam = dict(hidden_sizes=[64, 64], hidden_nonlin=to.tanh)
vfcn = FNNPolicy(spec=EnvSpec(env_sim.obs_space, ValueFunctionSpace),
**vfcn_hparam)
critic_hparam = dict(
gamma=0.9885,
lamda=0.9648,
num_epoch=2,
batch_size=500,
standardize_adv=False,
lr=5.792e-4,
max_grad_norm=1.,
)
critic = GAE(vfcn, **critic_hparam)
subrtn_policy_hparam = dict(
max_iter=200,
min_steps=3 * 23 * env_sim.max_steps,
num_epoch=7,
eps_clip=0.0744,
batch_size=500,
std_init=0.9074,
lr=3.446e-04,
max_grad_norm=1.,
num_workers=1,
)
subrtn_policy = PPO(study_dir, env_sim, behav_policy, critic,
**subrtn_policy_hparam)
# Subroutine for system identification
prior_std_denom = trial.suggest_uniform('prior_std_denom', 5, 20)
prior = DomainRandomizer(
NormalDomainParam(name='Mr', mean=0.095, std=0.095 / prior_std_denom),
NormalDomainParam(name='Mp', mean=0.024, std=0.024 / prior_std_denom),
NormalDomainParam(name='Lr', mean=0.085, std=0.085 / prior_std_denom),
NormalDomainParam(name='Lp', mean=0.129, std=0.129 / prior_std_denom),
)
ddp_policy = DomainDistrParamPolicy(
mapping=dp_map,
trafo_mask=trafo_mask,
prior=prior,
scale_params=trial.suggest_categorical('ddp_policy_scale_params',
[True, False]),
)
subsubrtn_distr_hparam = dict(
max_iter=trial.suggest_categorical('subsubrtn_distr_max_iter', [20]),
pop_size=trial.suggest_int('pop_size', 50, 500),
num_rollouts=1,
num_is_samples=trial.suggest_int('num_is_samples', 5, 20),
expl_std_init=trial.suggest_loguniform('expl_std_init', 1e-3, 1e-1),
expl_std_min=trial.suggest_categorical('expl_std_min', [1e-4]),
extra_expl_std_init=trial.suggest_loguniform('expl_std_init', 1e-3,
1e-1),
extra_expl_decay_iter=trial.suggest_int('extra_expl_decay_iter', 0,
10),
num_workers=1,
)
csv_logger = create_csv_step_logger(
osp.join(study_dir, f'trial_{trial.number}'))
subsubrtn_distr = CEM(study_dir,
env_sim,
ddp_policy,
**subsubrtn_distr_hparam,
logger=csv_logger)
obs_vel_weight = trial.suggest_loguniform('obs_vel_weight', 1, 100)
subrtn_distr_hparam = dict(
metric=None,
obs_dim_weight=[1, 1, 1, 1, obs_vel_weight, obs_vel_weight],
num_rollouts_per_distr=trial.suggest_int('num_rollouts_per_distr', 20,
100),
num_workers=1,
)
subrtn_distr = SysIdViaEpisodicRL(subsubrtn_distr, behav_policy,
**subrtn_distr_hparam)
# Algorithm
algo_hparam = dict(
max_iter=trial.suggest_categorical('algo_max_iter', [10]),
num_eval_rollouts=trial.suggest_categorical('algo_num_eval_rollouts',
[5]),
warmstart=trial.suggest_categorical('algo_warmstart', [True]),
thold_succ_subrtn=trial.suggest_categorical('algo_thold_succ_subrtn',
[50]),
subrtn_snapshot_mode='latest',
)
algo = SimOpt(study_dir,
env_sim,
env_real,
subrtn_policy,
subrtn_distr,
**algo_hparam,
logger=csv_logger)
# Jeeeha
algo.train(seed=args.seed)
# Evaluate
min_rollouts = 1000
sampler = ParallelRolloutSampler(
env_real, algo.policy, num_workers=1,
min_rollouts=min_rollouts) # parallelize via optuna n_jobs
ros = sampler.sample()
mean_ret = sum([r.undiscounted_return() for r in ros]) / min_rollouts
return mean_ret
def train_and_eval(trial: optuna.Trial, study_dir: str, seed: int):
"""
Objective function for the Optuna `Study` to maximize.
.. note::
Optuna expects only the `trial` argument, thus we use `functools.partial` to sneak in custom arguments.
:param trial: Optuna Trial object for hyper-parameter optimization
:param study_dir: the parent directory for all trials in this study
:param seed: seed value for the random number generators, pass `None` for no seeding
:return: objective function value
"""
# Synchronize seeds between Optuna trials
pyrado.set_seed(seed)
# Environment
env_hparams = dict(dt=1/100., max_steps=600)
env = QQubeSwingUpSim(**env_hparams)
env = ActNormWrapper(env)
# Learning rate scheduler
lrs_gamma = trial.suggest_categorical('exp_lr_scheduler_gamma', [None, 0.995, 0.999])
if lrs_gamma is not None:
lr_sched = lr_scheduler.ExponentialLR
lr_sched_hparam = dict(gamma=lrs_gamma)
else:
lr_sched, lr_sched_hparam = None, dict()
# Policy
policy_hparam = dict(
hidden_sizes=trial.suggest_categorical('hidden_sizes_policy', [(16, 16), (32, 32), (64, 64)]),
hidden_nonlin=fcn_from_str(trial.suggest_categorical('hidden_nonlin_policy', ['to_tanh', 'to_relu'])),
) # FNN
# policy_hparam = dict(
# hidden_size=trial.suggest_categorical('hidden_size_policy', [16, 32, 64]),
# num_recurrent_layers=trial.suggest_categorical('num_recurrent_layers_policy', [1, 2]),
# ) # LSTM & GRU
policy = FNNPolicy(spec=env.spec, **policy_hparam)
# policy = GRUPolicy(spec=env.spec, **policy_hparam)
# Critic
vfcn_hparam = dict(
hidden_sizes=trial.suggest_categorical('hidden_sizes_critic', [(16, 16), (32, 32), (64, 64)]),
hidden_nonlin=fcn_from_str(trial.suggest_categorical('hidden_nonlin_critic', ['to_tanh', 'to_relu'])),
)
# vfcn_hparam = dict(
# hidden_size=trial.suggest_categorical('hidden_size_critic', [16, 32, 64]),
# num_recurrent_layers=trial.suggest_categorical('num_recurrent_layers_critic', [1, 2]),
# ) # LSTM & GRU
vfcn = FNNPolicy(spec=EnvSpec(env.obs_space, ValueFunctionSpace), **vfcn_hparam)
# vfcn = GRUPolicy(spec=EnvSpec(env.obs_space, ValueFunctionSpace), **vfcn_hparam)
critic_hparam = dict(
batch_size=500,
gamma=trial.suggest_uniform('gamma_critic', 0.98, 1.),
lamda=trial.suggest_uniform('lamda_critic', 0.95, 1.),
num_epoch=trial.suggest_int('num_epoch_critic', 1, 10),
lr=trial.suggest_loguniform('lr_critic', 1e-5, 1e-3),
standardize_adv=trial.suggest_categorical('standardize_adv_critic', [False]),
max_grad_norm=trial.suggest_categorical('max_grad_norm_critic', [None, 1., 5.]),
lr_scheduler=lr_sched,
lr_scheduler_hparam=lr_sched_hparam
)
critic = GAE(vfcn, **critic_hparam)
# Algorithm
algo_hparam = dict(
num_workers=1, # parallelize via optuna n_jobs
max_iter=250,
batch_size=500,
min_steps=trial.suggest_int('num_rollouts_algo', 10, 30)*env.max_steps,
num_epoch=trial.suggest_int('num_epoch_algo', 1, 10),
eps_clip=trial.suggest_uniform('eps_clip_algo', 0.05, 0.2),
std_init=trial.suggest_uniform('std_init_algo', 0.5, 1.0),
lr=trial.suggest_loguniform('lr_algo', 1e-5, 1e-3),
max_grad_norm=trial.suggest_categorical('max_grad_norm_algo', [None, 1., 5.]),
lr_scheduler=lr_sched,
lr_scheduler_hparam=lr_sched_hparam
)
csv_logger = create_csv_step_logger(osp.join(study_dir, f'trial_{trial.number}'))
algo = PPO(osp.join(study_dir, f'trial_{trial.number}'), env, policy, critic, **algo_hparam, logger=csv_logger)
# Train without saving the results
algo.train(snapshot_mode='latest', seed=seed)
# Evaluate
min_rollouts = 1000
sampler = ParallelRolloutSampler(env, policy, num_workers=1,
min_rollouts=min_rollouts) # parallelize via optuna n_jobs
ros = sampler.sample()
mean_ret = sum([r.undiscounted_return() for r in ros])/min_rollouts
return mean_ret
