def test_predictor_export(self):
"""Verify that q-values before model export equal q-values after
model export. Meant to catch issues with export logic."""
environment = Gridworld()
trainer = trainer = self.get_sarsa_trainer(environment, False)
samples = Samples(
mdp_ids=["0"],
sequence_numbers=[0],
states=[{0: 1.0, 1: 1.0, 2: 1.0, 3: 1.0, 4: 1.0, 5: 1.0, 15: 1.0, 24: 1.0}],
actions=["D"],
action_probabilities=[0.5],
rewards=[0],
possible_actions=[["R", "D"]],
next_states=[{5: 1.0}],
next_actions=["U"],
terminals=[False],
possible_next_actions=[["R", "U", "D"]],
)
tdps = environment.preprocess_samples(samples, 1)
pre_export_q_values = trainer.q_network(tdps[0].states).detach().numpy()
predictor = trainer.predictor()
with tempfile.TemporaryDirectory() as tmpdirname:
tmp_path = os.path.join(tmpdirname, "model")
predictor.save(tmp_path, "minidb")
new_predictor = DQNPredictor.load(tmp_path, "minidb", False)
post_export_q_values = new_predictor.predict([samples.states[0]])
for i, action in enumerate(environment.ACTIONS):
self.assertAlmostEquals(
pre_export_q_values[0][i], post_export_q_values[0][action], places=4
)
def test_knn_dqn_trainer(self):
environment = Gridworld()
samples = environment.generate_samples(200000, 1.0)
evaluator = GridworldEvaluator(environment, False, DISCOUNT, False, samples)
parameters = self.get_parameters(environment)
trainer = KNNDQNTrainer(parameters, environment.normalization)
tdps = environment.preprocess_samples(
samples, self.minibatch_size, one_hot_action=False
)
predictor = trainer.predictor(environment.ACTIONS)
evaluator.evaluate(predictor)
print(
"Pre-Training eval: ",
evaluator.mc_loss[-1],
evaluator.value_doubly_robust[-1],
)
pre_train_loss = evaluator.mc_loss[-1]
for tdp in tdps:
tdp.rewards = tdp.rewards.flatten()
tdp.not_terminals = tdp.not_terminals.flatten()
trainer.train(tdp)
predictor = trainer.predictor(environment.ACTIONS)
evaluator.evaluate(predictor)
print(
"Post-Training eval: ",
evaluator.mc_loss[-1],
evaluator.value_doubly_robust[-1],
)
self.assertLess(evaluator.mc_loss[-1], pre_train_loss)
def test_evaluator_ground_truth(self):
environment = Gridworld()
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, _ = environment.generate_samples(100000, 1.0)
true_values = environment.true_values_for_sample(
states, actions, False)
# Hijack the reward timeline to insert the ground truth
reward_timelines = []
for tv in true_values:
reward_timelines.append({0: tv})
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(trainer, DISCOUNT)
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
for tdp in tdps:
trainer.stream_tdp(tdp, evaluator)
self.assertLess(evaluator.td_loss[-1], 0.05)
self.assertLess(evaluator.mc_loss[-1], 0.05)
def test_trainer_sarsa(self):
environment = Gridworld()
samples = environment.generate_samples(100000, 1.0)
evaluator = GridworldEvaluator(environment, False)
trainer = self.get_sarsa_trainer(environment)
predictor = trainer.predictor()
tdps = environment.preprocess_samples(samples, self.minibatch_size)
evaluator.evaluate(predictor)
print(
"Pre-Training eval: ",
evaluator.mc_loss[-1],
evaluator.reward_doubly_robust[-1],
)
self.assertGreater(evaluator.mc_loss[-1], 0.15)
for tdp in tdps:
trainer.train_numpy(tdp, None)
evaluator.evaluate(predictor)
print(
"Post-Training eval: ",
evaluator.mc_loss[-1],
evaluator.reward_doubly_robust[-1],
)
self.assertLess(evaluator.mc_loss[-1], 0.05)
self.assertGreater(
evaluator.reward_doubly_robust[-1], evaluator.reward_doubly_robust[-2]
)
def test_reward_boost(self):
environment = Gridworld()
reward_boost = {"L": 100, "R": 200, "U": 300, "D": 400}
trainer = self.get_sarsa_trainer_reward_boost(environment,
reward_boost)
predictor = trainer.predictor()
samples = environment.generate_samples(150000, 1.0)
rewards_update = []
for action, reward in zip(samples.actions, samples.rewards):
rewards_update.append(reward - reward_boost[action])
samples.rewards = rewards_update
evaluator = GridworldEvaluator(environment, False, DISCOUNT, False,
samples)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
evaluator.evaluate(predictor)
print(
"Pre-Training eval: ",
evaluator.mc_loss[-1],
evaluator.value_doubly_robust[-1],
)
for _ in range(2):
for tdp in tdps:
trainer.train_numpy(tdp, None)
evaluator.evaluate(predictor)
print(
"Post-Training eval: ",
evaluator.mc_loss[-1],
evaluator.value_doubly_robust[-1],
)
self.assertLess(evaluator.mc_loss[-1], 0.1)
def test_trainer_sarsa(self):
environment = Gridworld()
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
evaluator = GridworldEvaluator(environment, False)
trainer = self.get_sarsa_trainer(environment)
predictor = trainer.predictor()
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
self.assertGreater(evaluator.evaluate(predictor), 0.15)
for tdp in tdps:
trainer.stream_tdp(tdp, None)
evaluator.evaluate(predictor)
self.assertLess(evaluator.evaluate(predictor), 0.05)
def test_reward_boost(self):
environment = Gridworld()
reward_boost = {'L': 100, 'R': 200, 'U': 300, 'D': 400}
trainer = self.get_sarsa_trainer_reward_boost(environment,
reward_boost)
predictor = trainer.predictor()
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
rewards_update = []
for action, reward in zip(actions, rewards):
rewards_update.append(reward - reward_boost[action])
evaluator = GridworldEvaluator(environment, False)
tdps = environment.preprocess_samples(
states,
actions,
rewards_update,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
self.assertGreater(evaluator.evaluate(predictor), 0.15)
for tdp in tdps:
trainer.train_numpy(tdp, None)
self.assertLess(evaluator.evaluate(predictor), 0.05)
def test_evaluator_timeline(self):
environment = Gridworld()
samples = environment.generate_samples(100000, 1.0)
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(1)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
trainer.train_numpy(tdp, evaluator)
self.assertLess(evaluator.td_loss[-1], 0.2)
self.assertLess(evaluator.mc_loss[-1], 0.2)
def test_predictor_torch_export(self):
"""Verify that q-values before model export equal q-values after
model export. Meant to catch issues with export logic."""
environment = Gridworld()
samples = Samples(
mdp_ids=["0"],
sequence_numbers=[0],
sequence_number_ordinals=[1],
states=[{0: 1.0, 1: 1.0, 2: 1.0, 3: 1.0, 4: 1.0, 5: 1.0, 15: 1.0, 24: 1.0}],
actions=["D"],
action_probabilities=[0.5],
rewards=[0],
possible_actions=[["R", "D"]],
next_states=[{5: 1.0}],
next_actions=["U"],
terminals=[False],
possible_next_actions=[["R", "U", "D"]],
)
tdps = environment.preprocess_samples(samples, 1)
assert len(tdps) == 1, "Invalid number of data pages"
trainer, exporter = self.get_modular_sarsa_trainer_exporter(
environment, {}, False
)
input = rlt.PreprocessedState.from_tensor(tdps[0].states)
pre_export_q_values = trainer.q_network(input).q_values.detach().numpy()
preprocessor = Preprocessor(environment.normalization, False)
cpu_q_network = trainer.q_network.cpu_model()
cpu_q_network.eval()
dqn_with_preprocessor = DiscreteDqnWithPreprocessor(cpu_q_network, preprocessor)
serving_module = DiscreteDqnPredictorWrapper(
dqn_with_preprocessor, action_names=environment.ACTIONS
)
with tempfile.TemporaryDirectory() as tmpdirname:
buf = export_module_to_buffer(serving_module)
tmp_path = os.path.join(tmpdirname, "model")
with open(tmp_path, "wb") as f:
f.write(buf.getvalue())
f.close()
predictor = DiscreteDqnTorchPredictor(torch.jit.load(tmp_path))
post_export_q_values = predictor.predict([samples.states[0]])
for i, action in enumerate(environment.ACTIONS):
self.assertAlmostEqual(
float(pre_export_q_values[0][i]),
float(post_export_q_values[0][action]),
places=4,
)
def test_trainer_maxq(self):
environment = Gridworld()
maxq_sarsa_parameters = DiscreteActionModelParameters(
actions=environment.ACTIONS,
rl=RLParameters(
gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True,
),
training=TrainingParameters(
layers=[-1, 1],
activations=["linear"],
minibatch_size=self.minibatch_size,
learning_rate=0.01,
optimizer="ADAM",
),
)
# construct the new trainer that using maxq
maxq_trainer = DiscreteActionTrainer(
maxq_sarsa_parameters, environment.normalization
)
samples = environment.generate_samples(100000, 1.0)
predictor = maxq_trainer.predictor()
tdps = environment.preprocess_samples(samples, self.minibatch_size)
evaluator = GridworldEvaluator(environment, True)
evaluator.evaluate(predictor)
print(
"Pre-Training eval: ",
evaluator.mc_loss[-1],
evaluator.reward_doubly_robust[-1],
)
self.assertGreater(evaluator.mc_loss[-1], 0.3)
for _ in range(5):
for tdp in tdps:
maxq_trainer.train_numpy(tdp, None)
evaluator.evaluate(predictor)
print(
"Post-Training eval: ",
evaluator.mc_loss[-1],
evaluator.reward_doubly_robust[-1],
)
self.assertLess(evaluator.mc_loss[-1], 0.1)
self.assertGreater(
evaluator.reward_doubly_robust[-1], evaluator.reward_doubly_robust[-2]
)
def _test_reward_boost(self, use_gpu=False, use_all_avail_gpus=False):
environment = Gridworld()
reward_boost = {"L": 100, "R": 200, "U": 300, "D": 400}
trainer = self.get_sarsa_trainer_reward_boost(
environment,
reward_boost,
False,
use_gpu=use_gpu,
use_all_avail_gpus=use_all_avail_gpus,
)
predictor = trainer.predictor()
samples = environment.generate_samples(100000, 1.0, DISCOUNT)
rewards_update = []
for action, reward in zip(samples.actions, samples.rewards):
rewards_update.append(reward - reward_boost[action])
samples.rewards = rewards_update
evaluator = GridworldEvaluator(environment, False, DISCOUNT, False,
samples)
tdps = environment.preprocess_samples(samples,
self.minibatch_size,
use_gpu=use_gpu)
with tempfile.TemporaryDirectory() as tmpdirname:
tmp_path = os.path.join(tmpdirname, "model")
predictor.save(tmp_path, "minidb")
new_predictor = DQNPredictor.load(tmp_path, "minidb", False)
evaluator.evaluate(new_predictor)
print(
"Pre-Training eval: ",
evaluator.mc_loss[-1],
evaluator.value_doubly_robust[-1],
)
for tdp in tdps:
trainer.train(tdp, None)
predictor = trainer.predictor()
with tempfile.TemporaryDirectory() as tmpdirname:
tmp_path = os.path.join(tmpdirname, "model")
predictor.save(tmp_path, "minidb")
new_predictor = DQNPredictor.load(tmp_path, "minidb", False)
evaluator.evaluate(new_predictor)
print(
"Post-Training eval: ",
evaluator.mc_loss[-1],
evaluator.value_doubly_robust[-1],
)
self.assertLess(evaluator.mc_loss[-1], 0.1)
def test_evaluator_ground_truth_no_dueling(self):
environment = Gridworld()
samples = environment.generate_samples(500000, 1.0, DISCOUNT)
true_values = environment.true_values_for_sample(
samples.states, samples.actions, False)
# Hijack the reward timeline to insert the ground truth
samples.episode_values = true_values
trainer = self.get_sarsa_trainer(environment, False)
evaluator = Evaluator(environment.ACTIONS, 10, DISCOUNT, None, None)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
trainer.train(tdp, evaluator)
self.assertLess(evaluator.mc_loss[-1], 0.1)
def test_trainer_maxq(self):
environment = Gridworld()
maxq_sarsa_parameters = DiscreteActionModelParameters(
actions=environment.ACTIONS,
rl=RLParameters(gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True),
training=TrainingParameters(
layers=[-1, 1],
activations=['linear'],
minibatch_size=self.minibatch_size,
learning_rate=0.01,
optimizer='ADAM',
))
# construct the new trainer that using maxq
maxq_trainer = DiscreteActionTrainer(
maxq_sarsa_parameters,
environment.normalization,
)
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
predictor = maxq_trainer.predictor()
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
evaluator = GridworldEvaluator(environment, True)
print("Pre-Training eval", evaluator.evaluate(predictor))
self.assertGreater(evaluator.evaluate(predictor), 0.3)
for _ in range(2):
for tdp in tdps:
maxq_trainer.stream_tdp(tdp, None)
evaluator.evaluate(predictor)
print("Post-Training eval", evaluator.evaluate(predictor))
self.assertLess(evaluator.evaluate(predictor), 0.1)
def test_evaluator_ground_truth(self):
environment = Gridworld()
samples = environment.generate_samples(200000, 1.0)
true_values = environment.true_values_for_sample(
samples.states, samples.actions, False)
# Hijack the reward timeline to insert the ground truth
samples.reward_timelines = []
for tv in true_values:
samples.reward_timelines.append({0: tv})
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(environment.ACTIONS, 10, DISCOUNT, None, None)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for _ in range(2):
for tdp in tdps:
trainer.train_numpy(tdp, evaluator)
self.assertLess(evaluator.mc_loss[-1], 0.1)
def test_evaluator_timeline(self):
environment = Gridworld()
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(trainer, DISCOUNT)
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
for tdp in tdps:
trainer.stream_tdp(tdp, evaluator)
self.assertLess(evaluator.td_loss[-1], 0.2)
self.assertLess(evaluator.mc_loss[-1], 0.2)