def test_training_parameter_exploring(ex_dir, env: SimEnv, algo, algo_hparam): # Environment and policy env = ActNormWrapper(env) policy_hparam = dict(feats=FeatureStack([const_feat, identity_feat])) policy = LinearPolicy(spec=env.spec, **policy_hparam) # Get initial return for comparison rets_before = np.zeros(5) for i in range(rets_before.size): rets_before[i] = rollout(env, policy, eval=True, seed=i).undiscounted_return() # Create the algorithm and train algo_hparam['num_workers'] = 1 algo = algo(ex_dir, env, policy, **algo_hparam) algo.train() policy.param_values = algo.best_policy_param # mimic saving and loading # Compare returns before and after training for max_iter iteration rets_after = np.zeros_like(rets_before) for i in range(rets_before.size): rets_after[i] = rollout(env, policy, eval=True, seed=i).undiscounted_return() assert all(rets_after > rets_before)
def create_bob_setup(): # Environments env_hparams = dict(dt=1 / 100., max_steps=500) env_real = BallOnBeamSim(**env_hparams) env_real.domain_param = dict( # l_beam=1.95, # ang_offset=-0.03, g=10.81) env_sim = BallOnBeamSim(**env_hparams) randomizer = DomainRandomizer( # NormalDomainParam(name='l_beam', mean=0, std=1e-12, clip_lo=1.5, clip_up=3.5), # UniformDomainParam(name='ang_offset', mean=0, halfspan=1e-12), NormalDomainParam(name='g', mean=0, std=1e-12), ) env_sim = DomainRandWrapperLive(env_sim, randomizer) dp_map = { # 0: ('l_beam', 'mean'), 1: ('l_beam', 'std'), # 2: ('ang_offset', 'mean'), 3: ('ang_offset', 'halfspan') 0: ('g', 'mean'), 1: ('g', 'std') } env_sim = MetaDomainRandWrapper(env_sim, dp_map) # Policies (the behavioral policy needs to be deterministic) behavior_policy = LinearPolicy(env_sim.spec, feats=FeatureStack( [identity_feat, sin_feat])) behavior_policy.param_values = to.tensor( [3.8090, -3.8036, -1.0786, -2.4510, -0.9875, -1.3252, 3.1503, 1.4443]) prior = DomainRandomizer( # NormalDomainParam(name='l_beam', mean=2.05, std=2.05/10), # UniformDomainParam(name='ang_offset', mean=0.03, halfspan=0.03/10), NormalDomainParam(name='g', mean=8.81, std=8.81 / 10), ) # trafo_mask = [False, True, False, True] trafo_mask = [True, True] ddp_policy = DomainDistrParamPolicy(mapping=dp_map, trafo_mask=trafo_mask, prior=prior, scale_params=True) return env_sim, env_real, env_hparams, dp_map, behavior_policy, ddp_policy