def test_ddpg_trainer(self): environment = GridworldContinuous() samples = environment.generate_samples(200000, 1.0) epochs = 3 trainer = DDPGTrainer( self.get_ddpg_parameters(), environment.normalization, environment.normalization_action, ) evaluator = GridworldDDPGEvaluator(environment, True) tdps = environment.preprocess_samples(samples, self.minibatch_size) for epoch in range(epochs): print("On epoch {} of {}".format(epoch + 1, epochs)) critic_predictor = trainer.predictor() evaluator.evaluate_critic(critic_predictor) for tdp in tdps: training_samples = [ tdp.states, tdp.actions, tdp.rewards.flatten(), tdp.next_states, None, 1 - tdp.not_terminals.flatten(), # done None, None, [1 for i in range(len(tdp.states))], # time diff ] trainer.train(training_samples) critic_predictor = trainer.predictor() error = evaluator.evaluate_critic(critic_predictor) print("gridworld MAE: {0:.3f}".format(error))
def generate_samples(self, num_transitions, epsilon, discount_factor) -> Samples: samples = GridworldContinuous.generate_samples(self, num_transitions, epsilon, discount_factor) enum_states = [] for state in samples.states: enum_states.append({0: float(list(state.keys())[0])}) enum_next_states = [] for state in samples.next_states: enum_next_states.append({0: float(list(state.keys())[0])}) return Samples( mdp_ids=samples.mdp_ids, sequence_numbers=samples.sequence_numbers, states=enum_states, actions=samples.actions, action_probabilities=samples.action_probabilities, rewards=samples.rewards, possible_actions=samples.possible_actions, next_states=enum_next_states, next_actions=samples.next_actions, terminals=samples.terminals, possible_next_actions=samples.possible_next_actions, episode_values=samples.episode_values, )
def generate_samples(self, num_transitions, epsilon, with_possible=True) -> Samples: samples = GridworldContinuous.generate_samples(self, num_transitions, epsilon, with_possible) enum_states = [] for state in samples.states: enum_states.append({0: float(list(state.keys())[0])}) enum_next_states = [] for state in samples.next_states: enum_next_states.append({0: float(list(state.keys())[0])}) return Samples( mdp_ids=samples.mdp_ids, sequence_numbers=samples.sequence_numbers, states=enum_states, actions=samples.actions, propensities=samples.propensities, rewards=samples.rewards, next_states=enum_next_states, next_actions=samples.next_actions, terminals=samples.terminals, possible_next_actions=samples.possible_next_actions, reward_timelines=samples.reward_timelines, )
def test_evaluator_ground_truth(self): environment = GridworldContinuous() states, actions, rewards, next_states, next_actions, is_terminal,\ possible_next_actions, _ = environment.generate_samples(100000, 1.0) true_values = environment.true_values_for_sample( states, actions, False) # Hijack the reward timeline to insert the ground truth reward_timelines = [] for tv in true_values: reward_timelines.append({0: tv}) trainer = self.get_sarsa_trainer(environment) evaluator = Evaluator(trainer, DISCOUNT) tdps = environment.preprocess_samples( states, actions, rewards, next_states, next_actions, is_terminal, possible_next_actions, reward_timelines, self.minibatch_size, ) for tdp in tdps: trainer.train_numpy(tdp, evaluator) self.assertLess(evaluator.td_loss[-1], 0.05) self.assertLess(evaluator.mc_loss[-1], 0.12)
def test_ddpg_trainer(self): environment = GridworldContinuous() samples = environment.generate_samples(500000, 0.25) trainer = DDPGTrainer( self.get_ddpg_parameters(), environment.normalization, environment.normalization_action, environment.min_action_range, environment.max_action_range, ) evaluator = GridworldDDPGEvaluator(environment, True, DISCOUNT, False, samples) tdps = environment.preprocess_samples(samples, self.minibatch_size) critic_predictor = trainer.predictor(actor=False) evaluator.evaluate_critic(critic_predictor) for tdp in tdps: tdp.rewards = tdp.rewards.flatten() tdp.not_terminals = tdp.not_terminals.flatten() trainer.train(tdp) # Make sure actor predictor works actor = trainer.predictor(actor=True) evaluator.evaluate_actor(actor) # Evaluate critic predicor for correctness critic_predictor = trainer.predictor(actor=False) error = evaluator.evaluate_critic(critic_predictor) print("gridworld MAE: {0:.3f}".format(error))
def test_trainer_maxq(self): environment = GridworldContinuous() rl_parameters = self.get_sarsa_parameters() new_rl_parameters = ContinuousActionModelParameters( rl=RLParameters( gamma=DISCOUNT, target_update_rate=0.5, reward_burnin=10, maxq_learning=True, ), training=rl_parameters.training, knn=rl_parameters.knn, ) maxq_trainer = ContinuousActionDQNTrainer( new_rl_parameters, environment.normalization, environment.normalization_action, ) samples = environment.generate_samples(100000, 1.0) predictor = maxq_trainer.predictor() tdps = environment.preprocess_samples(samples, self.minibatch_size) evaluator = GridworldContinuousEvaluator(environment, True) self.assertGreater(evaluator.evaluate(predictor), 0.2) for _ in range(2): for tdp in tdps: maxq_trainer.train_numpy(tdp, None) evaluator.evaluate(predictor) self.assertLess(evaluator.evaluate(predictor), 0.15)
def test_trainer_sarsa(self): environment = GridworldContinuous() states, actions, rewards, next_states, next_actions, is_terminal,\ possible_next_actions, reward_timelines = \ environment.generate_samples(100000, 1.0) trainer = self.get_sarsa_trainer(environment) predictor = trainer.predictor() evaluator = GridworldContinuousEvaluator(environment, False) tdps = environment.preprocess_samples( states, actions, rewards, next_states, next_actions, is_terminal, possible_next_actions, reward_timelines, self.minibatch_size, ) self.assertGreater(evaluator.evaluate(predictor), 0.15) for tdp in tdps: trainer.train_numpy(tdp, None) evaluator.evaluate(predictor) self.assertLess(evaluator.evaluate(predictor), 0.05)
def test_gridworld_continuous_generate_samples(self): env = GridworldContinuous() num_samples = 1000 num_steps = 5 samples = env.generate_samples( num_samples, epsilon=1.0, discount_factor=0.9, multi_steps=num_steps ) self._check_samples(samples, num_samples, num_steps, True)
def test_evaluator_timeline(self): environment = GridworldContinuous() samples = environment.generate_samples(100000, 1.0) trainer = self.get_sarsa_trainer(environment) evaluator = Evaluator(1) tdps = environment.preprocess_samples(samples, self.minibatch_size) for tdp in tdps: trainer.train_numpy(tdp, evaluator) self.assertLess(evaluator.td_loss[-1], 0.2) self.assertLess(evaluator.mc_loss[-1], 0.2)
def test_gridworld_continuous_generate_samples(self): env = GridworldContinuous() num_samples = 1000 num_steps = 5 samples = env.generate_samples( num_samples, epsilon=1.0, discount_factor=0.9, multi_steps=num_steps, include_shorter_samples_at_start=True, include_shorter_samples_at_end=True, ) self._check_samples(samples, num_samples, num_steps, True)
def test_evaluator_ground_truth(self): environment = GridworldContinuous() samples = environment.generate_samples(500000, 1.0, DISCOUNT) # Hijack the reward timeline to insert the ground truth samples.episode_values = environment.true_values_for_sample( samples.states, samples.actions, False) trainer = self.get_sarsa_trainer(environment) evaluator = Evaluator(None, 10, DISCOUNT, None, None) tdps = environment.preprocess_samples(samples, self.minibatch_size) for tdp in tdps: trainer.train(tdp, evaluator) self.assertLess(evaluator.mc_loss[-1], 0.15)
def test_trainer_sarsa(self): environment = GridworldContinuous() samples = environment.generate_samples(100000, 1.0) trainer = self.get_sarsa_trainer(environment) predictor = trainer.predictor() evaluator = GridworldContinuousEvaluator(environment, False, DISCOUNT, False, samples) tdps = environment.preprocess_samples(samples, self.minibatch_size) for tdp in tdps: trainer.train_numpy(tdp, None) evaluator.evaluate(predictor) self.assertLess(evaluator.evaluate(predictor), 0.15)
def test_trainer_sarsa_factorized(self): environment = GridworldContinuous() samples = environment.generate_samples(500000, 1.0, DISCOUNT) trainer = self.get_sarsa_trainer( environment, self.get_sarsa_parameters_factorized()) predictor = trainer.predictor() evaluator = GridworldContinuousEvaluator(environment, False, DISCOUNT, False, samples) tdps = environment.preprocess_samples(samples, self.minibatch_size) for tdp in tdps: trainer.train(tdp) predictor = trainer.predictor() evaluator.evaluate(predictor) self.assertLess(evaluator.evaluate(predictor), 0.15)
def test_evaluator_ground_truth(self): environment = GridworldContinuous() samples = environment.generate_samples(100000, 1.0) true_values = environment.true_values_for_sample( samples.states, samples.actions, False) # Hijack the reward timeline to insert the ground truth samples.reward_timelines = [] for tv in true_values: samples.reward_timelines.append({0: tv}) trainer = self.get_sarsa_trainer(environment) evaluator = Evaluator(1) tdps = environment.preprocess_samples(samples, self.minibatch_size) for tdp in tdps: trainer.train_numpy(tdp, evaluator) self.assertLess(evaluator.td_loss[-1], 0.05) self.assertLess(evaluator.mc_loss[-1], 0.12)
def test_trainer_sarsa(self): environment = GridworldContinuous() samples = environment.generate_samples(150000, 1.0) trainer = self.get_sarsa_trainer(environment) predictor = trainer.predictor() evaluator = GridworldContinuousEvaluator(environment, False, DISCOUNT, False, samples) tdps = environment.preprocess_samples(samples, self.minibatch_size) for tdp in tdps: tdp.rewards = tdp.rewards.flatten() tdp.not_terminals = tdp.not_terminals.flatten() trainer.train(tdp) predictor = trainer.predictor() evaluator.evaluate(predictor) self.assertLess(evaluator.evaluate(predictor), 0.15)
def test_evaluator_ground_truth(self): environment = GridworldContinuous() samples = environment.generate_samples(200000, 1.0) true_values = environment.true_values_for_sample( samples.states, samples.actions, False) # Hijack the reward timeline to insert the ground truth samples.reward_timelines = [] for tv in true_values: samples.reward_timelines.append({0: tv}) trainer = self.get_sarsa_trainer(environment) evaluator = Evaluator(None, 10, DISCOUNT) tdps = environment.preprocess_samples(samples, self.minibatch_size) for tdp in tdps: tdp.rewards = tdp.rewards.flatten() tdp.not_terminals = tdp.not_terminals.flatten() trainer.train(tdp, evaluator) self.assertLess(evaluator.mc_loss[-1], 0.15)
def generate_samples( self, num_transitions, epsilon, with_possible=True ) -> Tuple[List[Dict[str, float]], List[Dict[str, float]], List[float], List[Dict[str, float]], List[Dict[str, float]], List[bool], List[List[Dict[str, float]]], List[Dict[int, float]]]: states, actions, rewards, next_states, next_actions, is_terminals, \ possible_next_actions, reward_timelines = \ GridworldContinuous.generate_samples( self, num_transitions, epsilon, with_possible) enum_states = [] for state in states: enum_states.append({'0': float(list(state.keys())[0])}) enum_next_states = [] for state in next_states: enum_next_states.append({'0': float(list(state.keys())[0])}) return ( enum_states, actions, rewards, enum_next_states, next_actions, is_terminals, possible_next_actions, reward_timelines )
def test_trainer_maxq(self): environment = GridworldContinuous() rl_parameters = self.get_sarsa_parameters() new_rl_parameters = ContinuousActionModelParameters( rl=RLParameters( gamma=DISCOUNT, target_update_rate=0.5, reward_burnin=10, maxq_learning=True, ), training=rl_parameters.training, knn=rl_parameters.knn) maxq_trainer = ContinuousActionDQNTrainer( new_rl_parameters, environment.normalization, environment.normalization_action, ) states, actions, rewards, next_states, next_actions, is_terminal,\ possible_next_actions, reward_timelines = \ environment.generate_samples(100000, 1.0) predictor = maxq_trainer.predictor() tdps = environment.preprocess_samples( states, actions, rewards, next_states, next_actions, is_terminal, possible_next_actions, reward_timelines, self.minibatch_size, ) evaluator = GridworldContinuousEvaluator(environment, True) self.assertGreater(evaluator.evaluate(predictor), 0.4) for _ in range(2): for tdp in tdps: maxq_trainer.stream_tdp(tdp) evaluator.evaluate(predictor) self.assertLess(evaluator.evaluate(predictor), 0.1)
def test_evaluator_timeline(self): environment = GridworldContinuous() states, actions, rewards, next_states, next_actions, is_terminal,\ possible_next_actions, reward_timelines = \ environment.generate_samples(100000, 1.0) trainer = self.get_sarsa_trainer(environment) evaluator = Evaluator(trainer, DISCOUNT) tdps = environment.preprocess_samples( states, actions, rewards, next_states, next_actions, is_terminal, possible_next_actions, reward_timelines, self.minibatch_size, ) for tdp in tdps: trainer.train_numpy(tdp, evaluator) self.assertLess(evaluator.td_loss[-1], 0.2) self.assertLess(evaluator.mc_loss[-1], 0.2)
def test_gridworld_continuous_generate_samples(self): env = GridworldContinuous() num_samples = 1000 num_steps = 5 samples = env.generate_samples(num_samples, epsilon=1.0, discount_factor=0.9, multi_steps=num_steps) for i in range(num_samples): if samples.terminals[i][0]: break if i < num_samples - 1: self.assertEqual(samples.mdp_ids[i], samples.mdp_ids[i + 1]) self.assertEqual(samples.sequence_numbers[i] + 1, samples.sequence_numbers[i + 1]) for j in range(len(samples.terminals[i])): self.assertEqual(samples.rewards[i][j], samples.rewards[i + j][0]) self.assertDictEqual(samples.next_states[i][j], samples.next_states[i + j][0]) self.assertDictEqual(samples.next_actions[i][j], samples.next_actions[i + j][0]) self.assertEqual(samples.terminals[i][j], samples.terminals[i + j][0]) self.assertListEqual( samples.possible_next_actions[i][j], samples.possible_next_actions[i + j][0], ) if samples.terminals[i][j]: continue self.assertDictEqual(samples.next_states[i][j], samples.states[i + j + 1]) self.assertDictEqual(samples.next_actions[i][j], samples.actions[i + j + 1]) self.assertListEqual( samples.possible_next_actions[i][j], samples.possible_actions[i + j + 1], ) single_step_samples = samples.to_single_step() for i in range(num_samples): if single_step_samples.terminals[i] is True: break self.assertEqual(single_step_samples.mdp_ids[i], samples.mdp_ids[i]) self.assertEqual(single_step_samples.sequence_numbers[i], samples.sequence_numbers[i]) self.assertDictEqual(single_step_samples.states[i], samples.states[i]) self.assertDictEqual(single_step_samples.actions[i], samples.actions[i]) self.assertEqual( single_step_samples.action_probabilities[i], samples.action_probabilities[i], ) self.assertEqual(single_step_samples.rewards[i], samples.rewards[i][0]) self.assertListEqual(single_step_samples.possible_actions[i], samples.possible_actions[i]) self.assertDictEqual(single_step_samples.next_states[i], samples.next_states[i][0]) self.assertDictEqual(single_step_samples.next_actions[i], samples.next_actions[i][0]) self.assertEqual(single_step_samples.terminals[i], samples.terminals[i][0]) self.assertListEqual( single_step_samples.possible_next_actions[i], samples.possible_next_actions[i][0], )