def run_test(
env: str,
model: ModelManager__Union,
replay_memory_size: int,
train_every_ts: int,
train_after_ts: int,
num_episodes: int,
max_steps: Optional[int],
last_score_bar: float,
):
env = EnvFactory.make(env)
env.seed(SEED)
env.action_space.seed(SEED)
normalization = build_normalizer(env)
logger.info(f"Normalization is {normalization}")
manager = model.value
trainer = manager.initialize_trainer(
use_gpu=False,
reward_options=RewardOptions(),
normalization_data_map=normalization,
)
policy = manager.create_policy()
replay_buffer = ReplayBuffer.create_from_env(
env=env,
replay_memory_size=replay_memory_size,
batch_size=trainer.minibatch_size,
)
post_step = train_with_replay_buffer_post_step(
replay_buffer=replay_buffer,
trainer=trainer,
training_freq=train_every_ts,
batch_size=trainer.minibatch_size,
replay_burnin=train_after_ts,
)
agent = Agent.create_for_env(env,
policy=policy,
post_transition_callback=post_step)
reward_history = []
for i in range(num_episodes):
logger.info(f"running episode {i}")
ep_reward = run_episode(env=env, agent=agent, max_steps=max_steps)
reward_history.append(ep_reward)
assert reward_history[-1] >= last_score_bar, (
f"reward after {len(reward_history)} episodes is {reward_history[-1]},"
f"less than < {last_score_bar}...\n"
f"Full reward history: {reward_history}")
return reward_history
def run_test(
env_name: str,
model: ModelManager__Union,
replay_memory_size: int,
train_every_ts: int,
train_after_ts: int,
num_train_episodes: int,
max_steps: Optional[int],
passing_score_bar: float,
num_eval_episodes: int,
use_gpu: bool,
):
env = EnvFactory.make(env_name)
env.seed(SEED)
env.action_space.seed(SEED)
normalization = build_normalizer(env)
logger.info(f"Normalization is: \n{pprint.pformat(normalization)}")
manager = model.value
try:
# pyre-fixme[16]: `Env` has no attribute `state_feature_config_provider`.
manager.state_feature_config_provider = env.state_feature_config_provider
logger.info(
f"Using environment's state_feature_config_provider.\n"
f"{manager.state_feature_config_provider}"
)
except AttributeError:
logger.info("state_feature_config_provider override not applicable")
trainer = manager.initialize_trainer(
use_gpu=use_gpu,
reward_options=RewardOptions(),
normalization_data_map=normalization,
)
training_policy = manager.create_policy(serving=False)
replay_buffer = ReplayBuffer.create_from_env(
env=env,
replay_memory_size=replay_memory_size,
batch_size=trainer.minibatch_size,
)
device = torch.device("cuda") if use_gpu else torch.device("cpu")
# first fill the replay buffer to burn_in
train_after_ts = max(train_after_ts, trainer.minibatch_size)
fill_replay_buffer(
env=env, replay_buffer=replay_buffer, desired_size=train_after_ts
)
post_step = train_with_replay_buffer_post_step(
replay_buffer=replay_buffer,
env=env,
trainer=trainer,
training_freq=train_every_ts,
batch_size=trainer.minibatch_size,
device=device,
)
agent = Agent.create_for_env(
env, policy=training_policy, post_transition_callback=post_step, device=device
)
writer = SummaryWriter()
with summary_writer_context(writer):
train_rewards = []
for i in range(num_train_episodes):
trajectory = run_episode(
env=env, agent=agent, mdp_id=i, max_steps=max_steps
)
ep_reward = trajectory.calculate_cumulative_reward()
train_rewards.append(ep_reward)
logger.info(
f"Finished training episode {i} (len {len(trajectory)})"
f" with reward {ep_reward}."
)
logger.info("============Train rewards=============")
logger.info(train_rewards)
logger.info(f"average: {np.mean(train_rewards)};\tmax: {np.max(train_rewards)}")
# Check whether the max score passed the score bar; we explore during training
# the return could be bad (leading to flakiness in C51 and QRDQN).
assert np.max(train_rewards) >= passing_score_bar, (
f"max reward ({np.max(train_rewards)})after training for "
f"{len(train_rewards)} episodes is less than < {passing_score_bar}.\n"
)
serving_policy = manager.create_policy(serving=True)
agent = Agent.create_for_env_with_serving_policy(env, serving_policy)
eval_rewards = evaluate_for_n_episodes(
n=num_eval_episodes, env=env, agent=agent, max_steps=max_steps
).squeeze(1)
logger.info("============Eval rewards==============")
logger.info(eval_rewards)
logger.info(f"average: {np.mean(eval_rewards)};\tmax: {np.max(eval_rewards)}")
assert np.mean(eval_rewards) >= passing_score_bar, (
f"Predictor reward is {np.mean(eval_rewards)},"
f"less than < {passing_score_bar}.\n"
)
def run_test(
env_name: str,
model: ModelManager__Union,
replay_memory_size: int,
train_every_ts: int,
train_after_ts: int,
num_train_episodes: int,
max_steps: Optional[int],
passing_score_bar: float,
num_eval_episodes: int,
use_gpu: bool,
):
env = EnvFactory.make(env_name)
env.seed(SEED)
env.action_space.seed(SEED)
normalization = build_normalizer(env)
logger.info(f"Normalization is: \n{pprint.pformat(normalization)}")
manager = model.value
trainer = manager.initialize_trainer(
use_gpu=use_gpu,
reward_options=RewardOptions(),
normalization_data_map=normalization,
)
training_policy = manager.create_policy(serving=False)
replay_buffer = ReplayBuffer.create_from_env(
env=env,
replay_memory_size=replay_memory_size,
batch_size=trainer.minibatch_size,
)
device = torch.device("cuda") if use_gpu else None
# first fill the replay buffer to burn_in
train_after_ts = max(train_after_ts, trainer.minibatch_size)
fill_replay_buffer(env=env,
replay_buffer=replay_buffer,
desired_size=train_after_ts)
post_step = train_with_replay_buffer_post_step(
replay_buffer=replay_buffer,
env=env,
trainer=trainer,
training_freq=train_every_ts,
batch_size=trainer.minibatch_size,
device=device,
)
agent = Agent.create_for_env(
env,
policy=training_policy,
post_transition_callback=post_step,
# pyre-fixme[6]: Expected `Union[str, torch.device]` for 4th param but got
# `Optional[torch.device]`.
device=device,
)
writer = SummaryWriter()
with summary_writer_context(writer):
train_rewards = []
for i in range(num_train_episodes):
trajectory = run_episode(env=env,
agent=agent,
mdp_id=i,
max_steps=max_steps)
ep_reward = trajectory.calculate_cumulative_reward()
train_rewards.append(ep_reward)
logger.info(
f"Finished training episode {i} with reward {ep_reward}.")
assert train_rewards[-1] >= passing_score_bar, (
f"reward after {len(train_rewards)} episodes is {train_rewards[-1]},"
f"less than < {passing_score_bar}...\n"
f"Full reward history: {train_rewards}")
logger.info("============Train rewards=============")
logger.info(train_rewards)
serving_policy = manager.create_policy(serving=True)
agent = Agent.create_for_env_with_serving_policy(env, serving_policy)
eval_rewards = evaluate_for_n_episodes(n=num_eval_episodes,
env=env,
agent=agent,
max_steps=max_steps).squeeze(1)
assert np.mean(eval_rewards) >= passing_score_bar, (
f"Predictor reward is {np.mean(eval_rewards)},"
f"less than < {passing_score_bar}...\n"
f"Full eval rewards: {eval_rewards}.")
logger.info("============Eval rewards==============")
logger.info(eval_rewards)
def run_test(
env: Env__Union,
model: ModelManager__Union,
replay_memory_size: int,
train_every_ts: int,
train_after_ts: int,
num_train_episodes: int,
passing_score_bar: float,
num_eval_episodes: int,
use_gpu: bool,
minibatch_size: Optional[int] = None,
):
env = env.value
normalization = build_normalizer(env)
logger.info(f"Normalization is: \n{pprint.pformat(normalization)}")
manager = model.value
trainer = manager.initialize_trainer(
use_gpu=use_gpu,
reward_options=RewardOptions(),
normalization_data_map=normalization,
)
training_policy = manager.create_policy(serving=False)
# pyre-fixme[16]: Module `pl` has no attribute `LightningModule`.
if not isinstance(trainer, pl.LightningModule):
if minibatch_size is None:
minibatch_size = trainer.minibatch_size
assert minibatch_size == trainer.minibatch_size
assert minibatch_size is not None
replay_buffer = ReplayBuffer(replay_capacity=replay_memory_size,
batch_size=minibatch_size)
device = torch.device("cuda") if use_gpu else torch.device("cpu")
# first fill the replay buffer to burn_in
train_after_ts = max(train_after_ts, minibatch_size)
fill_replay_buffer(env=env,
replay_buffer=replay_buffer,
desired_size=train_after_ts)
# pyre-fixme[16]: Module `pl` has no attribute `LightningModule`.
if isinstance(trainer, pl.LightningModule):
agent = Agent.create_for_env(env, policy=training_policy)
# TODO: Simplify this setup by creating LightningDataModule
dataset = ReplayBufferDataset.create_for_trainer(
trainer,
env,
agent,
replay_buffer,
batch_size=minibatch_size,
training_frequency=train_every_ts,
num_episodes=num_train_episodes,
max_steps=200,
)
data_loader = torch.utils.data.DataLoader(dataset,
collate_fn=identity_collate)
# pyre-fixme[16]: Module `pl` has no attribute `Trainer`.
pl_trainer = pl.Trainer(max_epochs=1, gpus=int(use_gpu))
pl_trainer.fit(trainer, data_loader)
# TODO: Also check train_reward
else:
post_step = train_with_replay_buffer_post_step(
replay_buffer=replay_buffer,
env=env,
trainer=trainer,
training_freq=train_every_ts,
batch_size=trainer.minibatch_size,
device=device,
)
env.seed(SEED)
env.action_space.seed(SEED)
train_rewards = train_policy(
env,
training_policy,
num_train_episodes,
post_step=post_step,
post_episode=None,
use_gpu=use_gpu,
)
# Check whether the max score passed the score bar; we explore during training
# the return could be bad (leading to flakiness in C51 and QRDQN).
assert np.max(train_rewards) >= passing_score_bar, (
f"max reward ({np.max(train_rewards)}) after training for "
f"{len(train_rewards)} episodes is less than < {passing_score_bar}.\n"
)
serving_policy = manager.create_policy(serving=True)
eval_rewards = eval_policy(env,
serving_policy,
num_eval_episodes,
serving=True)
assert (
eval_rewards.mean() >= passing_score_bar
), f"Eval reward is {eval_rewards.mean()}, less than < {passing_score_bar}.\n"
def run_test(
env: Env__Union,
model: ModelManager__Union,
replay_memory_size: int,
train_every_ts: int,
train_after_ts: int,
num_train_episodes: int,
passing_score_bar: float,
num_eval_episodes: int,
use_gpu: bool,
):
env = env.value
normalization = build_normalizer(env)
logger.info(f"Normalization is: \n{pprint.pformat(normalization)}")
manager = model.value
trainer = manager.initialize_trainer(
use_gpu=use_gpu,
reward_options=RewardOptions(),
normalization_data_map=normalization,
)
training_policy = manager.create_policy(serving=False)
replay_buffer = ReplayBuffer(replay_capacity=replay_memory_size,
batch_size=trainer.minibatch_size)
device = torch.device("cuda") if use_gpu else torch.device("cpu")
# first fill the replay buffer to burn_in
train_after_ts = max(train_after_ts, trainer.minibatch_size)
fill_replay_buffer(env=env,
replay_buffer=replay_buffer,
desired_size=train_after_ts)
post_step = train_with_replay_buffer_post_step(
replay_buffer=replay_buffer,
env=env,
trainer=trainer,
training_freq=train_every_ts,
batch_size=trainer.minibatch_size,
device=device,
)
env.seed(SEED)
env.action_space.seed(SEED)
train_rewards = train_policy(
env,
training_policy,
num_train_episodes,
post_step=post_step,
post_episode=None,
use_gpu=use_gpu,
)
# Check whether the max score passed the score bar; we explore during training
# the return could be bad (leading to flakiness in C51 and QRDQN).
assert np.max(train_rewards) >= passing_score_bar, (
f"max reward ({np.max(train_rewards)}) after training for "
f"{len(train_rewards)} episodes is less than < {passing_score_bar}.\n")
serving_policy = manager.create_policy(serving=True)
eval_rewards = eval_policy(env,
serving_policy,
num_eval_episodes,
serving=True)
assert (
eval_rewards.mean() >= passing_score_bar
), f"Eval reward is {eval_rewards.mean()}, less than < {passing_score_bar}.\n"
def run_test(
env: str,
model: ModelManager__Union,
replay_memory_size: int,
train_every_ts: int,
train_after_ts: int,
num_train_episodes: int,
max_steps: Optional[int],
passing_score_bar: float,
num_eval_episodes: int,
use_gpu: bool,
):
env = EnvFactory.make(env)
env.seed(SEED)
env.action_space.seed(SEED)
normalization = build_normalizer(env)
logger.info(f"Normalization is {normalization}")
manager = model.value
trainer = manager.initialize_trainer(
use_gpu=use_gpu,
reward_options=RewardOptions(),
normalization_data_map=normalization,
)
replay_buffer = ReplayBuffer.create_from_env(
env=env,
replay_memory_size=replay_memory_size,
batch_size=trainer.minibatch_size,
)
device = torch.device("cuda") if use_gpu else None
post_step = train_with_replay_buffer_post_step(
replay_buffer=replay_buffer,
trainer=trainer,
training_freq=train_every_ts,
batch_size=trainer.minibatch_size,
replay_burnin=train_after_ts,
device=device,
)
training_policy = manager.create_policy(serving=False)
agent = Agent.create_for_env(env,
policy=training_policy,
post_transition_callback=post_step,
device=device)
train_rewards = []
for i in range(num_train_episodes):
ep_reward = run_episode(env=env, agent=agent, max_steps=max_steps)
train_rewards.append(ep_reward)
logger.info(f"Finished training episode {i} with reward {ep_reward}.")
assert train_rewards[-1] >= passing_score_bar, (
f"reward after {len(train_rewards)} episodes is {train_rewards[-1]},"
f"less than < {passing_score_bar}...\n"
f"Full reward history: {train_rewards}")
logger.info("============Train rewards=============")
logger.info(train_rewards)
def gym_to_reagent_serving(
obs: np.array) -> Tuple[torch.Tensor, torch.Tensor]:
obs_tensor = torch.tensor(obs).float().unsqueeze(0)
presence_tensor = torch.ones_like(obs_tensor)
return (obs_tensor, presence_tensor)
serving_policy = manager.create_policy(serving=True)
agent = Agent.create_for_env(env,
policy=serving_policy,
obs_preprocessor=gym_to_reagent_serving)
eval_rewards = []
for i in range(num_eval_episodes):
ep_reward = run_episode(env=env, agent=agent, max_steps=max_steps)
eval_rewards.append(ep_reward)
logger.info(f"Finished eval episode {i} with reward {ep_reward}.")
assert np.mean(eval_rewards) >= passing_score_bar, (
f"Predictor reward is {np.mean(eval_rewards)},"
f"less than < {passing_score_bar}...\n"
f"Full eval rewards: {eval_rewards}.")
logger.info("============Eval rewards==============")
logger.info(eval_rewards)
