def test_training_parameter_exploring(ex_dir, env: SimEnv, algo, algo_hparam):
# Environment and policy
env = ActNormWrapper(env)
policy_hparam = dict(feats=FeatureStack([const_feat, identity_feat]))
policy = LinearPolicy(spec=env.spec, **policy_hparam)
# Get initial return for comparison
rets_before = np.zeros(5)
for i in range(rets_before.size):
rets_before[i] = rollout(env, policy, eval=True,
seed=i).undiscounted_return()
# Create the algorithm and train
algo_hparam['num_workers'] = 1
algo = algo(ex_dir, env, policy, **algo_hparam)
algo.train()
policy.param_values = algo.best_policy_param # mimic saving and loading
# Compare returns before and after training for max_iter iteration
rets_after = np.zeros_like(rets_before)
for i in range(rets_before.size):
rets_after[i] = rollout(env, policy, eval=True,
seed=i).undiscounted_return()
assert all(rets_after > rets_before)
def create_bob_setup():
# Environments
env_hparams = dict(dt=1 / 100., max_steps=500)
env_real = BallOnBeamSim(**env_hparams)
env_real.domain_param = dict(
# l_beam=1.95,
# ang_offset=-0.03,
g=10.81)
env_sim = BallOnBeamSim(**env_hparams)
randomizer = DomainRandomizer(
# NormalDomainParam(name='l_beam', mean=0, std=1e-12, clip_lo=1.5, clip_up=3.5),
# UniformDomainParam(name='ang_offset', mean=0, halfspan=1e-12),
NormalDomainParam(name='g', mean=0, std=1e-12), )
env_sim = DomainRandWrapperLive(env_sim, randomizer)
dp_map = {
# 0: ('l_beam', 'mean'), 1: ('l_beam', 'std'),
# 2: ('ang_offset', 'mean'), 3: ('ang_offset', 'halfspan')
0: ('g', 'mean'),
1: ('g', 'std')
}
env_sim = MetaDomainRandWrapper(env_sim, dp_map)
# Policies (the behavioral policy needs to be deterministic)
behavior_policy = LinearPolicy(env_sim.spec,
feats=FeatureStack(
[identity_feat, sin_feat]))
behavior_policy.param_values = to.tensor(
[3.8090, -3.8036, -1.0786, -2.4510, -0.9875, -1.3252, 3.1503, 1.4443])
prior = DomainRandomizer(
# NormalDomainParam(name='l_beam', mean=2.05, std=2.05/10),
# UniformDomainParam(name='ang_offset', mean=0.03, halfspan=0.03/10),
NormalDomainParam(name='g', mean=8.81, std=8.81 / 10), )
# trafo_mask = [False, True, False, True]
trafo_mask = [True, True]
ddp_policy = DomainDistrParamPolicy(mapping=dp_map,
trafo_mask=trafo_mask,
prior=prior,
scale_params=True)
return env_sim, env_real, env_hparams, dp_map, behavior_policy, ddp_policy