def true_values_for_sample(self, enum_states, actions, assume_optimal_policy: bool): states = [] for state in enum_states: states.append({int(list(state.values())[0]): 1}) return Gridworld.true_values_for_sample(self, states, actions, assume_optimal_policy)
def test_evaluator_ground_truth(self): environment = Gridworld() states, actions, rewards, next_states, next_actions, is_terminal,\ possible_next_actions, _ = environment.generate_samples(100000, 1.0) true_values = environment.true_values_for_sample( states, actions, False) # Hijack the reward timeline to insert the ground truth reward_timelines = [] for tv in true_values: reward_timelines.append({0: tv}) trainer = self.get_sarsa_trainer(environment) evaluator = Evaluator(trainer, DISCOUNT) tdps = environment.preprocess_samples( states, actions, rewards, next_states, next_actions, is_terminal, possible_next_actions, reward_timelines, self.minibatch_size, ) for tdp in tdps: trainer.stream_tdp(tdp, evaluator) self.assertLess(evaluator.td_loss[-1], 0.05) self.assertLess(evaluator.mc_loss[-1], 0.05)
def test_evaluator_ground_truth_no_dueling(self): environment = Gridworld() samples = environment.generate_samples(500000, 1.0, DISCOUNT) true_values = environment.true_values_for_sample( samples.states, samples.actions, False) # Hijack the reward timeline to insert the ground truth samples.episode_values = true_values trainer = self.get_sarsa_trainer(environment, False) evaluator = Evaluator(environment.ACTIONS, 10, DISCOUNT, None, None) tdps = environment.preprocess_samples(samples, self.minibatch_size) for tdp in tdps: trainer.train(tdp, evaluator) self.assertLess(evaluator.mc_loss[-1], 0.1)
def test_evaluator_ground_truth(self): environment = Gridworld() samples = environment.generate_samples(200000, 1.0) true_values = environment.true_values_for_sample( samples.states, samples.actions, False) # Hijack the reward timeline to insert the ground truth samples.reward_timelines = [] for tv in true_values: samples.reward_timelines.append({0: tv}) trainer = self.get_sarsa_trainer(environment) evaluator = Evaluator(environment.ACTIONS, 10, DISCOUNT, None, None) tdps = environment.preprocess_samples(samples, self.minibatch_size) for _ in range(2): for tdp in tdps: trainer.train_numpy(tdp, evaluator) self.assertLess(evaluator.mc_loss[-1], 0.1)