def test_q_learning_iterate_value_q_pi_tabular_policy_ne(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, 20) epsilon = 0.05 q_S_A_1 = TabularStateActionValueEstimator(mdp_environment, epsilon, None) mdp_agent_1 = StochasticMdpAgent('test', random_state, q_S_A_1.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent_1, environment=mdp_environment, num_improvements=5, num_episodes_per_improvement=10, num_updates_per_improvement=None, alpha=None, mode=Mode.Q_LEARNING, n_steps=None, planning_environment=None, make_final_policy_greedy=True, q_S_A=q_S_A_1) q_S_A_2 = TabularStateActionValueEstimator(mdp_environment, epsilon, None) mdp_agent_2 = StochasticMdpAgent('test', random_state, q_S_A_2.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent_2, environment=mdp_environment, num_improvements=5, num_episodes_per_improvement=5, num_updates_per_improvement=None, alpha=None, mode=Mode.Q_LEARNING, n_steps=None, planning_environment=None, make_final_policy_greedy=True, q_S_A=q_S_A_2) test_state = mdp_environment.SS[5] test_action = test_state.AA[0] assert q_S_A_1 != q_S_A_2 assert q_S_A_1[test_state] != q_S_A_2[test_state] assert q_S_A_1[test_state][test_action] != q_S_A_2[test_state][test_action]
def test_iterate_value_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=3000, num_episodes_per_improvement=1, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_iteration_of_value_q_pi.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open( f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_iteration_of_value_q_pi.pickle', 'rb') as file: pi_fixture, q_S_A_fixture = pickle.load(file) assert tabular_pi_legacy_eq(mdp_agent.pi, pi_fixture) and tabular_estimator_legacy_eq( q_S_A, q_S_A_fixture)
def test_iterate_value_q_pi_with_pdf(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi( agent=mdp_agent, environment=mdp_environment, num_improvements=3000, num_episodes_per_improvement=1, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, num_improvements_per_plot=1500, pdf_save_path=tempfile.NamedTemporaryFile(delete=False).name) with pytest.raises(ValueError, match='Epsilon must be >= 0'): q_S_A.epsilon = -1.0 q_S_A.improve_policy(mdp_agent, states=None, event=PolicyImprovementEvent.MAKING_POLICY_GREEDY) q_S_A.epsilon = 0.0 assert q_S_A.improve_policy( mdp_agent, None, PolicyImprovementEvent.MAKING_POLICY_GREEDY) == 14
def test_learn(): random_state = RandomState(12345) gym = Gym(random_state=random_state, T=None, gym_id='CartPole-v1') q_S_A = TabularStateActionValueEstimator(gym, 0.05, 0.001) mdp_agent = StochasticMdpAgent('agent', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent, environment=gym, num_improvements=10, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=0.1, mode=Mode.SARSA, n_steps=1, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_gym.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open(f'{os.path.dirname(__file__)}/fixtures/test_gym.pickle', 'rb') as file: fixture_pi, fixture_q_S_A = pickle.load(file) assert tabular_pi_legacy_eq(mdp_agent.pi, fixture_pi) and tabular_estimator_legacy_eq( q_S_A, fixture_q_S_A)
def test_evaluate_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, None, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) evaluated_states, _ = evaluate_q_pi(agent=mdp_agent, environment=mdp_environment, num_episodes=1000, exploring_starts=True, update_upon_every_visit=False, q_S_A=q_S_A) assert len( q_S_A) == len(evaluated_states) + 2 # terminal states aren't evaluated assert all(s in q_S_A for s in evaluated_states) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_action_value.pickle', 'wb') as file: # pickle.dump(q_S_A, file) with open( f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_action_value.pickle', 'rb') as file: fixture = pickle.load(file) assert tabular_estimator_legacy_eq(q_S_A, fixture)
def test_iterate_value_q_pi_with_pdf(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi( agent=mdp_agent, environment=mdp_environment, num_improvements=10, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=0.1, mode=Mode.Q_LEARNING, n_steps=1, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, num_improvements_per_plot=5, pdf_save_path=tempfile.NamedTemporaryFile(delete=False).name)
def test_n_step_q_learning_iterate_value_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=10, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=0.1, mode=Mode.Q_LEARNING, n_steps=3, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_td_n_step_q_learning_iteration_of_value_q_pi.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open( f'{os.path.dirname(__file__)}/fixtures/test_td_n_step_q_learning_iteration_of_value_q_pi.pickle', 'rb') as file: fixture_pi, fixture_q_S_A = pickle.load(file) assert tabular_pi_legacy_eq(mdp_agent.pi, fixture_pi) and tabular_estimator_legacy_eq( q_S_A, fixture_q_S_A)
def test_human_player(): random_state = RandomState(12345) human = Human() def mock_input(prompt: str) -> str: s = human.most_recent_state selected_a = sample_list_item(s.AA, probs=None, random_state=random_state) return selected_a.name human.get_input = mock_input mancala: Mancala = Mancala(random_state=random_state, T=None, initial_count=4, player_2=human) epsilon = 0.05 q_S_A = TabularStateActionValueEstimator(mancala, epsilon, None) p1 = StochasticMdpAgent('player 1', random_state, q_S_A.get_initial_policy(), 1) state = mancala.reset_for_new_run(p1) p1.reset_for_new_run(state) a = p1.act(0) state, reward = mancala.advance(state, 0, a, p1) assert mancala.board[7].count == 0 and state.i == 1 and reward.i == 2
def test_invalid_iterate_value_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.0, None) # target agent mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) # episode generation (behavior) policy off_policy_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) with pytest.raises( ValueError, match= 'Planning environments are not currently supported for Monte Carlo iteration.' ): iterate_value_q_pi( agent=mdp_agent, environment=mdp_environment, num_improvements=100, num_episodes_per_improvement=1, update_upon_every_visit=True, planning_environment=TrajectorySamplingMdpPlanningEnvironment( 'foo', random_state, StochasticEnvironmentModel(), 100, None), make_final_policy_greedy=False, q_S_A=q_S_A, off_policy_agent=off_policy_agent) # test warning...no off-policy agent with epsilon=0.0 q_S_A.epsilon = 0.0 iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=100, num_episodes_per_improvement=1, update_upon_every_visit=True, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, off_policy_agent=None)
def main(): random = RandomState(12345) gridworld = Gridworld.example_4_1(random, None) # the bottom-right corner (3,3) is a goal state. get the states surrounding this goal. these will become the sticky # states. sticky_states = [ gridworld.grid[2, 2], gridworld.grid[2, 3], gridworld.grid[3, 2] ] # amplify all negative rewards in the sticky states by a factor of 10, keeping the probabilities the same. for sticky_state in sticky_states: for a in gridworld.p_S_prime_R_given_S_A[sticky_state]: for s_prime in gridworld.p_S_prime_R_given_S_A[sticky_state][a]: gridworld.p_S_prime_R_given_S_A[sticky_state][a][s_prime] = { Reward(r.i, (r.r * 10.0 if r.r < 0.0 else r.r)): gridworld.p_S_prime_R_given_S_A[sticky_state][a][s_prime] [r] for r in gridworld.p_S_prime_R_given_S_A[sticky_state][a] [s_prime] } epsilon = 0.1 q_S_A = TabularStateActionValueEstimator( environment=gridworld, epsilon=epsilon, continuous_state_discretization_resolution=None) pi = q_S_A.get_initial_policy() mdp_agent = StochasticMdpAgent(name='agent', random_state=random, pi=pi, gamma=1.0) # iterate the agents policy using q-learning temporal differencing iterate_value_q_pi(agent=mdp_agent, environment=gridworld, num_improvements=20, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=None, mode=Mode.Q_LEARNING, n_steps=None, planning_environment=None, make_final_policy_greedy=True, q_S_A=q_S_A, num_improvements_per_plot=20) for s in pi: print(f'State {s.i}:') for a in pi[s]: if pi[s][a] > 0.0: print(f'\tPr({a.name}): {pi[s][a]}')
def test_invalid_improve_policy_with_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) epsilon = 0.0 q_S_A = TabularStateActionValueEstimator(mdp_environment, epsilon, None) # target agent mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) with pytest.raises(ValueError, match='Epsilon must be >= 0'): improve_policy_with_q_pi(mdp_agent, {}, -1)
def train_thread_target(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=1000000, num_episodes_per_improvement=10, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, thread_manager=thread_manager, num_improvements_per_plot=10)
def test_sarsa_iterate_value_q_pi_with_trajectory_planning(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) planning_environment = TrajectorySamplingMdpPlanningEnvironment( 'test planning', random_state, StochasticEnvironmentModel(), 10, None) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=100, num_episodes_per_improvement=1, num_updates_per_improvement=None, alpha=0.1, mode=Mode.SARSA, n_steps=1, planning_environment=planning_environment, make_final_policy_greedy=True, q_S_A=q_S_A) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_td_iteration_of_value_q_pi_planning.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open( f'{os.path.dirname(__file__)}/fixtures/test_td_iteration_of_value_q_pi_planning.pickle', 'rb') as file: pi_fixture, q_S_A_fixture = pickle.load(file) assert tabular_pi_legacy_eq(mdp_agent.pi, pi_fixture) and tabular_estimator_legacy_eq( q_S_A, q_S_A_fixture)
def test_evaluate_q_pi_invalid_n_steps(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) epsilon = 0.05 q_S_A = TabularStateActionValueEstimator(mdp_environment, epsilon, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) with pytest.raises(ValueError): evaluate_q_pi(agent=mdp_agent, environment=mdp_environment, num_episodes=5, num_updates_per_improvement=None, alpha=0.1, mode=Mode.Q_LEARNING, n_steps=-1, planning_environment=None, q_S_A=q_S_A)
def test_invalid_epsilon_iterate_value_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.0, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) with pytest.raises(ValueError, match='epsilon must be strictly > 0 for TD-learning'): iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=10, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=0.1, mode=Mode.Q_LEARNING, n_steps=3, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A)
def test_learn(): random_state = RandomState(12345) mancala: Mancala = Mancala(random_state=random_state, T=None, initial_count=4, player_2=StochasticMdpAgent( 'player 2', random_state, TabularPolicy(None, None), 1)) q_S_A = TabularStateActionValueEstimator(mancala, 0.05, None) p1 = StochasticMdpAgent('player 1', random_state, q_S_A.get_initial_policy(), 1) checkpoint_path = iterate_value_q_pi( agent=p1, environment=mancala, num_improvements=3, num_episodes_per_improvement=100, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, num_improvements_per_checkpoint=3, checkpoint_path=tempfile.NamedTemporaryFile(delete=False).name) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_mancala.pickle', 'wb') as file: # pickle.dump(p1.pi, file) with open(f'{os.path.dirname(__file__)}/fixtures/test_mancala.pickle', 'rb') as file: fixture = pickle.load(file) assert tabular_pi_legacy_eq(p1.pi, fixture) resumed_p1 = resume_from_checkpoint(checkpoint_path=checkpoint_path, resume_function=iterate_value_q_pi, num_improvements=2) # run same number of improvements without checkpoint...result should be the same. random_state = RandomState(12345) mancala: Mancala = Mancala(random_state=random_state, T=None, initial_count=4, player_2=StochasticMdpAgent( 'player 2', random_state, TabularPolicy(None, None), 1)) q_S_A = TabularStateActionValueEstimator(mancala, 0.05, None) no_checkpoint_p1 = StochasticMdpAgent('player 1', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=no_checkpoint_p1, environment=mancala, num_improvements=5, num_episodes_per_improvement=100, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A) assert no_checkpoint_p1.pi == resumed_p1.pi