def single_wrappers(single_venv, scheduler, our_idx, normalize, rew_shape,
rew_shape_params, victim_index, victim_path, victim_type,
debug, env_name, load_policy, lookback_params,
transparent_params, log_callbacks, save_callbacks):
if rew_shape:
rew_shape_venv = apply_reward_wrapper(single_env=single_venv,
scheduler=scheduler,
shaping_params=rew_shape_params,
agent_idx=our_idx)
log_callbacks.append(lambda logger, locals, globals: rew_shape_venv.
log_callback(logger))
single_venv = rew_shape_venv
for anneal_type in ['noise', 'rew_shape']:
if scheduler.is_conditional(anneal_type):
scheduler.set_annealer_get_logs(anneal_type,
rew_shape_venv.get_logs)
if lookback_params['lb_num'] > 0:
lookback_venv = LookbackRewardVecWrapper(single_venv, env_name, debug,
victim_index, victim_path,
victim_type,
transparent_params,
**lookback_params)
single_venv = lookback_venv
if normalize:
normalized_venv = VecNormalize(single_venv)
if load_policy['path'] is not None:
if load_policy['type'] == 'zoo':
raise ValueError(
"Trying to normalize twice. Bansal et al's Zoo agents normalize "
"implicitly. Please set normalize=False to disable VecNormalize."
)
normalized_venv.load_running_average(load_policy['path'])
save_callbacks.append(
lambda root_dir: normalized_venv.save_running_average(root_dir))
single_venv = normalized_venv
return single_venv
def single_wrappers(single_venv, scheduler, our_idx, normalize, load_policy,
rew_shape, rew_shape_params, log_callbacks, save_callbacks):
if rew_shape:
rew_shape_venv = apply_reward_wrapper(single_env=single_venv, scheduler=scheduler,
shaping_params=rew_shape_params, agent_idx=our_idx)
log_callbacks.append(lambda logger, locals, globals: rew_shape_venv.log_callback(logger))
single_venv = rew_shape_venv
for anneal_type in ['noise', 'rew_shape']:
if scheduler.is_conditional(anneal_type):
scheduler.set_annealer_get_logs(anneal_type, rew_shape_venv.get_logs)
if normalize:
if load_policy['type'] == 'zoo':
raise ValueError("Trying to normalize twice. Bansal et al's Zoo agents normalize "
"implicitly. Please set normalize=False to disable VecNormalize.")
normalized_venv = VecNormalize(single_venv)
save_callbacks.append(lambda root_dir: normalized_venv.save_running_average(root_dir))
single_venv = normalized_venv
return single_venv
def single_wrappers(
single_venv,
scheduler,
our_idx,
normalize,
normalize_observations,
rew_shape,
rew_shape_params,
embed_index,
embed_paths,
embed_types,
debug,
env_name,
load_policy,
lookback_params,
transparent_params,
log_callbacks,
save_callbacks,
):
if rew_shape:
rew_shape_venv = apply_reward_wrapper(
single_env=single_venv,
scheduler=scheduler,
shaping_params=rew_shape_params,
agent_idx=our_idx,
)
log_callbacks.append(lambda logger, locals, globals: rew_shape_venv.log_callback(logger))
single_venv = rew_shape_venv
for anneal_type in ["noise", "rew_shape"]:
if scheduler.is_conditional(anneal_type):
scheduler.set_annealer_get_logs(anneal_type, rew_shape_venv.get_logs)
if lookback_params["lb_num"] > 0:
if len(embed_types) > 1:
raise ValueError("Lookback is not supported with multiple embedded agents")
embed_path = embed_paths[0]
embed_type = embed_types[0]
lookback_venv = LookbackRewardVecWrapper(
single_venv,
env_name,
debug,
embed_index,
embed_path,
embed_type,
transparent_params,
**lookback_params,
)
single_venv = lookback_venv
if normalize:
if normalize_observations:
if load_policy["path"] is not None:
if load_policy["type"] == "zoo":
raise ValueError(
"Trying to normalize twice. Bansal et al's Zoo agents normalize "
"implicitly. Please set normalize=False to disable VecNormalize."
)
normalized_venv = VecNormalize(single_venv)
else:
normalized_venv = VecNormalize(single_venv, norm_obs=False)
if load_policy["path"] is not None and load_policy["type"] != "zoo":
normalized_venv.load_running_average(load_policy["path"])
save_callbacks.append(lambda root_dir: normalized_venv.save_running_average(root_dir))
single_venv = normalized_venv
return single_venv
env=vec_norm_env,
learning_rate=lambda f: 1e-4 * f,
ent_coef=1e-2,
n_steps=256,
nminibatches=8,
noptepochs=8,
cliprange=0.2,
cliprange_vf=0.2,
tensorboard_log='logs',
verbose=1)
model.learn(total_timesteps=int(1e7),
seed=0,
log_interval=100,
tb_log_name=log_str,
callback=stop_by_keyboard)
signal.signal(signal.SIGINT, original_sigint_handler)
model.save("logs/{}_1/trained_model".format(log_str))
vec_norm_env.save_running_average("logs/{}_1".format(log_str))
# render trained agent
env = VecNormalize(DummyVecEnv([lambda: gym.make(env_id)]),
norm_reward=True)
env.load_running_average("logs/{}_1".format(log_str))
obs = env.reset()
while True:
action, _states = model.predict(obs, deterministic=True)
obs, rewards, dones, info = env.step(action)
env.render()