def test_ddpg_trainer(self):
environment = GridworldContinuous()
samples = environment.generate_samples(200000, 1.0)
epochs = 3
trainer = DDPGTrainer(
self.get_ddpg_parameters(),
environment.normalization,
environment.normalization_action,
)
evaluator = GridworldDDPGEvaluator(environment, True)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for epoch in range(epochs):
print("On epoch {} of {}".format(epoch + 1, epochs))
critic_predictor = trainer.predictor()
evaluator.evaluate_critic(critic_predictor)
for tdp in tdps:
training_samples = [
tdp.states,
tdp.actions,
tdp.rewards.flatten(),
tdp.next_states,
None,
1 - tdp.not_terminals.flatten(), # done
None,
None,
[1 for i in range(len(tdp.states))], # time diff
]
trainer.train(training_samples)
critic_predictor = trainer.predictor()
error = evaluator.evaluate_critic(critic_predictor)
print("gridworld MAE: {0:.3f}".format(error))
def generate_samples(self, num_transitions, epsilon,
discount_factor) -> Samples:
samples = GridworldContinuous.generate_samples(self, num_transitions,
epsilon,
discount_factor)
enum_states = []
for state in samples.states:
enum_states.append({0: float(list(state.keys())[0])})
enum_next_states = []
for state in samples.next_states:
enum_next_states.append({0: float(list(state.keys())[0])})
return Samples(
mdp_ids=samples.mdp_ids,
sequence_numbers=samples.sequence_numbers,
states=enum_states,
actions=samples.actions,
action_probabilities=samples.action_probabilities,
rewards=samples.rewards,
possible_actions=samples.possible_actions,
next_states=enum_next_states,
next_actions=samples.next_actions,
terminals=samples.terminals,
possible_next_actions=samples.possible_next_actions,
episode_values=samples.episode_values,
)
def generate_samples(self,
num_transitions,
epsilon,
with_possible=True) -> Samples:
samples = GridworldContinuous.generate_samples(self, num_transitions,
epsilon, with_possible)
enum_states = []
for state in samples.states:
enum_states.append({0: float(list(state.keys())[0])})
enum_next_states = []
for state in samples.next_states:
enum_next_states.append({0: float(list(state.keys())[0])})
return Samples(
mdp_ids=samples.mdp_ids,
sequence_numbers=samples.sequence_numbers,
states=enum_states,
actions=samples.actions,
propensities=samples.propensities,
rewards=samples.rewards,
next_states=enum_next_states,
next_actions=samples.next_actions,
terminals=samples.terminals,
possible_next_actions=samples.possible_next_actions,
reward_timelines=samples.reward_timelines,
)
def test_evaluator_ground_truth(self):
environment = GridworldContinuous()
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, _ = environment.generate_samples(100000, 1.0)
true_values = environment.true_values_for_sample(
states, actions, False)
# Hijack the reward timeline to insert the ground truth
reward_timelines = []
for tv in true_values:
reward_timelines.append({0: tv})
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(trainer, DISCOUNT)
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
for tdp in tdps:
trainer.train_numpy(tdp, evaluator)
self.assertLess(evaluator.td_loss[-1], 0.05)
self.assertLess(evaluator.mc_loss[-1], 0.12)
def test_ddpg_trainer(self):
environment = GridworldContinuous()
samples = environment.generate_samples(500000, 0.25)
trainer = DDPGTrainer(
self.get_ddpg_parameters(),
environment.normalization,
environment.normalization_action,
environment.min_action_range,
environment.max_action_range,
)
evaluator = GridworldDDPGEvaluator(environment, True, DISCOUNT, False,
samples)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
critic_predictor = trainer.predictor(actor=False)
evaluator.evaluate_critic(critic_predictor)
for tdp in tdps:
tdp.rewards = tdp.rewards.flatten()
tdp.not_terminals = tdp.not_terminals.flatten()
trainer.train(tdp)
# Make sure actor predictor works
actor = trainer.predictor(actor=True)
evaluator.evaluate_actor(actor)
# Evaluate critic predicor for correctness
critic_predictor = trainer.predictor(actor=False)
error = evaluator.evaluate_critic(critic_predictor)
print("gridworld MAE: {0:.3f}".format(error))
def test_trainer_maxq(self):
environment = GridworldContinuous()
rl_parameters = self.get_sarsa_parameters()
new_rl_parameters = ContinuousActionModelParameters(
rl=RLParameters(
gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True,
),
training=rl_parameters.training,
knn=rl_parameters.knn,
)
maxq_trainer = ContinuousActionDQNTrainer(
new_rl_parameters,
environment.normalization,
environment.normalization_action,
)
samples = environment.generate_samples(100000, 1.0)
predictor = maxq_trainer.predictor()
tdps = environment.preprocess_samples(samples, self.minibatch_size)
evaluator = GridworldContinuousEvaluator(environment, True)
self.assertGreater(evaluator.evaluate(predictor), 0.2)
for _ in range(2):
for tdp in tdps:
maxq_trainer.train_numpy(tdp, None)
evaluator.evaluate(predictor)
self.assertLess(evaluator.evaluate(predictor), 0.15)
def test_trainer_sarsa(self):
environment = GridworldContinuous()
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
trainer = self.get_sarsa_trainer(environment)
predictor = trainer.predictor()
evaluator = GridworldContinuousEvaluator(environment, False)
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
self.assertGreater(evaluator.evaluate(predictor), 0.15)
for tdp in tdps:
trainer.train_numpy(tdp, None)
evaluator.evaluate(predictor)
self.assertLess(evaluator.evaluate(predictor), 0.05)
def test_gridworld_continuous_generate_samples(self):
env = GridworldContinuous()
num_samples = 1000
num_steps = 5
samples = env.generate_samples(
num_samples, epsilon=1.0, discount_factor=0.9, multi_steps=num_steps
)
self._check_samples(samples, num_samples, num_steps, True)
def test_evaluator_timeline(self):
environment = GridworldContinuous()
samples = environment.generate_samples(100000, 1.0)
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(1)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
trainer.train_numpy(tdp, evaluator)
self.assertLess(evaluator.td_loss[-1], 0.2)
self.assertLess(evaluator.mc_loss[-1], 0.2)
def test_gridworld_continuous_generate_samples(self):
env = GridworldContinuous()
num_samples = 1000
num_steps = 5
samples = env.generate_samples(
num_samples,
epsilon=1.0,
discount_factor=0.9,
multi_steps=num_steps,
include_shorter_samples_at_start=True,
include_shorter_samples_at_end=True,
)
self._check_samples(samples, num_samples, num_steps, True)
def test_evaluator_ground_truth(self):
environment = GridworldContinuous()
samples = environment.generate_samples(500000, 1.0, DISCOUNT)
# Hijack the reward timeline to insert the ground truth
samples.episode_values = environment.true_values_for_sample(
samples.states, samples.actions, False)
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(None, 10, DISCOUNT, None, None)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
trainer.train(tdp, evaluator)
self.assertLess(evaluator.mc_loss[-1], 0.15)
def test_trainer_sarsa(self):
environment = GridworldContinuous()
samples = environment.generate_samples(100000, 1.0)
trainer = self.get_sarsa_trainer(environment)
predictor = trainer.predictor()
evaluator = GridworldContinuousEvaluator(environment, False, DISCOUNT,
False, samples)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
trainer.train_numpy(tdp, None)
evaluator.evaluate(predictor)
self.assertLess(evaluator.evaluate(predictor), 0.15)
def test_trainer_sarsa_factorized(self):
environment = GridworldContinuous()
samples = environment.generate_samples(500000, 1.0, DISCOUNT)
trainer = self.get_sarsa_trainer(
environment, self.get_sarsa_parameters_factorized())
predictor = trainer.predictor()
evaluator = GridworldContinuousEvaluator(environment, False, DISCOUNT,
False, samples)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
trainer.train(tdp)
predictor = trainer.predictor()
evaluator.evaluate(predictor)
self.assertLess(evaluator.evaluate(predictor), 0.15)
def test_evaluator_ground_truth(self):
environment = GridworldContinuous()
samples = environment.generate_samples(100000, 1.0)
true_values = environment.true_values_for_sample(
samples.states, samples.actions, False)
# Hijack the reward timeline to insert the ground truth
samples.reward_timelines = []
for tv in true_values:
samples.reward_timelines.append({0: tv})
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(1)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
trainer.train_numpy(tdp, evaluator)
self.assertLess(evaluator.td_loss[-1], 0.05)
self.assertLess(evaluator.mc_loss[-1], 0.12)
def test_trainer_sarsa(self):
environment = GridworldContinuous()
samples = environment.generate_samples(150000, 1.0)
trainer = self.get_sarsa_trainer(environment)
predictor = trainer.predictor()
evaluator = GridworldContinuousEvaluator(environment, False, DISCOUNT,
False, samples)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
tdp.rewards = tdp.rewards.flatten()
tdp.not_terminals = tdp.not_terminals.flatten()
trainer.train(tdp)
predictor = trainer.predictor()
evaluator.evaluate(predictor)
self.assertLess(evaluator.evaluate(predictor), 0.15)
def test_evaluator_ground_truth(self):
environment = GridworldContinuous()
samples = environment.generate_samples(200000, 1.0)
true_values = environment.true_values_for_sample(
samples.states, samples.actions, False)
# Hijack the reward timeline to insert the ground truth
samples.reward_timelines = []
for tv in true_values:
samples.reward_timelines.append({0: tv})
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(None, 10, DISCOUNT)
tdps = environment.preprocess_samples(samples, self.minibatch_size)
for tdp in tdps:
tdp.rewards = tdp.rewards.flatten()
tdp.not_terminals = tdp.not_terminals.flatten()
trainer.train(tdp, evaluator)
self.assertLess(evaluator.mc_loss[-1], 0.15)
def generate_samples(
self, num_transitions, epsilon, with_possible=True
) -> Tuple[List[Dict[str, float]], List[Dict[str, float]], List[float],
List[Dict[str, float]], List[Dict[str, float]], List[bool],
List[List[Dict[str, float]]], List[Dict[int, float]]]:
states, actions, rewards, next_states, next_actions, is_terminals, \
possible_next_actions, reward_timelines = \
GridworldContinuous.generate_samples(
self, num_transitions, epsilon, with_possible)
enum_states = []
for state in states:
enum_states.append({'0': float(list(state.keys())[0])})
enum_next_states = []
for state in next_states:
enum_next_states.append({'0': float(list(state.keys())[0])})
return (
enum_states, actions, rewards, enum_next_states, next_actions,
is_terminals, possible_next_actions, reward_timelines
)
def test_trainer_maxq(self):
environment = GridworldContinuous()
rl_parameters = self.get_sarsa_parameters()
new_rl_parameters = ContinuousActionModelParameters(
rl=RLParameters(
gamma=DISCOUNT,
target_update_rate=0.5,
reward_burnin=10,
maxq_learning=True,
),
training=rl_parameters.training,
knn=rl_parameters.knn)
maxq_trainer = ContinuousActionDQNTrainer(
new_rl_parameters,
environment.normalization,
environment.normalization_action,
)
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
predictor = maxq_trainer.predictor()
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
evaluator = GridworldContinuousEvaluator(environment, True)
self.assertGreater(evaluator.evaluate(predictor), 0.4)
for _ in range(2):
for tdp in tdps:
maxq_trainer.stream_tdp(tdp)
evaluator.evaluate(predictor)
self.assertLess(evaluator.evaluate(predictor), 0.1)
def test_evaluator_timeline(self):
environment = GridworldContinuous()
states, actions, rewards, next_states, next_actions, is_terminal,\
possible_next_actions, reward_timelines = \
environment.generate_samples(100000, 1.0)
trainer = self.get_sarsa_trainer(environment)
evaluator = Evaluator(trainer, DISCOUNT)
tdps = environment.preprocess_samples(
states,
actions,
rewards,
next_states,
next_actions,
is_terminal,
possible_next_actions,
reward_timelines,
self.minibatch_size,
)
for tdp in tdps:
trainer.train_numpy(tdp, evaluator)
self.assertLess(evaluator.td_loss[-1], 0.2)
self.assertLess(evaluator.mc_loss[-1], 0.2)
def test_gridworld_continuous_generate_samples(self):
env = GridworldContinuous()
num_samples = 1000
num_steps = 5
samples = env.generate_samples(num_samples,
epsilon=1.0,
discount_factor=0.9,
multi_steps=num_steps)
for i in range(num_samples):
if samples.terminals[i][0]:
break
if i < num_samples - 1:
self.assertEqual(samples.mdp_ids[i], samples.mdp_ids[i + 1])
self.assertEqual(samples.sequence_numbers[i] + 1,
samples.sequence_numbers[i + 1])
for j in range(len(samples.terminals[i])):
self.assertEqual(samples.rewards[i][j],
samples.rewards[i + j][0])
self.assertDictEqual(samples.next_states[i][j],
samples.next_states[i + j][0])
self.assertDictEqual(samples.next_actions[i][j],
samples.next_actions[i + j][0])
self.assertEqual(samples.terminals[i][j],
samples.terminals[i + j][0])
self.assertListEqual(
samples.possible_next_actions[i][j],
samples.possible_next_actions[i + j][0],
)
if samples.terminals[i][j]:
continue
self.assertDictEqual(samples.next_states[i][j],
samples.states[i + j + 1])
self.assertDictEqual(samples.next_actions[i][j],
samples.actions[i + j + 1])
self.assertListEqual(
samples.possible_next_actions[i][j],
samples.possible_actions[i + j + 1],
)
single_step_samples = samples.to_single_step()
for i in range(num_samples):
if single_step_samples.terminals[i] is True:
break
self.assertEqual(single_step_samples.mdp_ids[i],
samples.mdp_ids[i])
self.assertEqual(single_step_samples.sequence_numbers[i],
samples.sequence_numbers[i])
self.assertDictEqual(single_step_samples.states[i],
samples.states[i])
self.assertDictEqual(single_step_samples.actions[i],
samples.actions[i])
self.assertEqual(
single_step_samples.action_probabilities[i],
samples.action_probabilities[i],
)
self.assertEqual(single_step_samples.rewards[i],
samples.rewards[i][0])
self.assertListEqual(single_step_samples.possible_actions[i],
samples.possible_actions[i])
self.assertDictEqual(single_step_samples.next_states[i],
samples.next_states[i][0])
self.assertDictEqual(single_step_samples.next_actions[i],
samples.next_actions[i][0])
self.assertEqual(single_step_samples.terminals[i],
samples.terminals[i][0])
self.assertListEqual(
single_step_samples.possible_next_actions[i],
samples.possible_next_actions[i][0],
)