def test_evaluate_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, None, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) evaluated_states, _ = evaluate_q_pi(agent=mdp_agent, environment=mdp_environment, num_episodes=1000, exploring_starts=True, update_upon_every_visit=False, q_S_A=q_S_A) assert len( q_S_A) == len(evaluated_states) + 2 # terminal states aren't evaluated assert all(s in q_S_A for s in evaluated_states) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_action_value.pickle', 'wb') as file: # pickle.dump(q_S_A, file) with open( f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_action_value.pickle', 'rb') as file: fixture = pickle.load(file) assert tabular_estimator_legacy_eq(q_S_A, fixture)
def test_iterate_value_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=3000, num_episodes_per_improvement=1, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_iteration_of_value_q_pi.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open( f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_iteration_of_value_q_pi.pickle', 'rb') as file: pi_fixture, q_S_A_fixture = pickle.load(file) assert tabular_pi_legacy_eq(mdp_agent.pi, pi_fixture) and tabular_estimator_legacy_eq( q_S_A, q_S_A_fixture)
def test_value_iteration(): # run policy iteration on v_pi random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent_v_pi_policy_iteration = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) iterate_policy_v_pi(mdp_agent_v_pi_policy_iteration, mdp_environment, 0.001, True) # run value iteration on v_pi random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent_v_pi_value_iteration = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) iterate_value_v_pi(mdp_agent_v_pi_value_iteration, mdp_environment, 0.001, 1, True) assert mdp_agent_v_pi_policy_iteration.pi == mdp_agent_v_pi_value_iteration.pi # run value iteration on q_pi random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent_q_pi_value_iteration = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) iterate_value_q_pi(mdp_agent_q_pi_value_iteration, mdp_environment, 0.001, 1, True) assert mdp_agent_q_pi_value_iteration.pi == mdp_agent_v_pi_policy_iteration.pi
def test_iterate_value_q_pi_with_pdf(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi( agent=mdp_agent, environment=mdp_environment, num_improvements=10, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=0.1, mode=Mode.Q_LEARNING, n_steps=1, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, num_improvements_per_plot=5, pdf_save_path=tempfile.NamedTemporaryFile(delete=False).name)
def test_learn(): random_state = RandomState(12345) gym = Gym(random_state=random_state, T=None, gym_id='CartPole-v1') q_S_A = TabularStateActionValueEstimator(gym, 0.05, 0.001) mdp_agent = StochasticMdpAgent('agent', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent, environment=gym, num_improvements=10, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=0.1, mode=Mode.SARSA, n_steps=1, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_gym.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open(f'{os.path.dirname(__file__)}/fixtures/test_gym.pickle', 'rb') as file: fixture_pi, fixture_q_S_A = pickle.load(file) assert tabular_pi_legacy_eq(mdp_agent.pi, fixture_pi) and tabular_estimator_legacy_eq( q_S_A, fixture_q_S_A)
def test_n_step_q_learning_iterate_value_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=10, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=0.1, mode=Mode.Q_LEARNING, n_steps=3, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_td_n_step_q_learning_iteration_of_value_q_pi.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open( f'{os.path.dirname(__file__)}/fixtures/test_td_n_step_q_learning_iteration_of_value_q_pi.pickle', 'rb') as file: fixture_pi, fixture_q_S_A = pickle.load(file) assert tabular_pi_legacy_eq(mdp_agent.pi, fixture_pi) and tabular_estimator_legacy_eq( q_S_A, fixture_q_S_A)
def test_human_player(): random_state = RandomState(12345) human = Human() def mock_input(prompt: str) -> str: s = human.most_recent_state selected_a = sample_list_item(s.AA, probs=None, random_state=random_state) return selected_a.name human.get_input = mock_input mancala: Mancala = Mancala(random_state=random_state, T=None, initial_count=4, player_2=human) epsilon = 0.05 q_S_A = TabularStateActionValueEstimator(mancala, epsilon, None) p1 = StochasticMdpAgent('player 1', random_state, q_S_A.get_initial_policy(), 1) state = mancala.reset_for_new_run(p1) p1.reset_for_new_run(state) a = p1.act(0) state, reward = mancala.advance(state, 0, a, p1) assert mancala.board[7].count == 0 and state.i == 1 and reward.i == 2
def test_learn(): random_state = RandomState(12345) mancala: Mancala = Mancala(random_state=random_state, T=None, initial_count=4, player_2=None) p1 = ActionValueMdpAgent( 'player 1', random_state, 1, TabularStateActionValueEstimator(mancala, 0.05, None)) checkpoint_path = iterate_value_q_pi( agent=p1, environment=mancala, num_improvements=3, num_episodes_per_improvement=100, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, num_improvements_per_checkpoint=3, checkpoint_path=tempfile.NamedTemporaryFile(delete=False).name) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_mancala.pickle', 'wb') as file: # pickle.dump(p1.pi, file) with open(f'{os.path.dirname(__file__)}/fixtures/test_mancala.pickle', 'rb') as file: fixture = pickle.load(file) assert tabular_pi_legacy_eq(p1.pi, fixture) resumed_p1 = resume_from_checkpoint(checkpoint_path=checkpoint_path, resume_function=iterate_value_q_pi, num_improvements=2) # run same number of improvements without checkpoint...result should be the same. random_state = RandomState(12345) mancala: Mancala = Mancala(random_state=random_state, T=None, initial_count=4, player_2=None) no_checkpoint_p1 = ActionValueMdpAgent( 'player 1', random_state, 1, TabularStateActionValueEstimator(mancala, 0.05, None)) iterate_value_q_pi(agent=no_checkpoint_p1, environment=mancala, num_improvements=5, num_episodes_per_improvement=100, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False) assert no_checkpoint_p1.pi == resumed_p1.pi
def main(): random = RandomState(12345) gridworld = Gridworld.example_4_1(random, None) # the bottom-right corner (3,3) is a goal state. get the states surrounding this goal. these will become the sticky # states. sticky_states = [ gridworld.grid[2, 2], gridworld.grid[2, 3], gridworld.grid[3, 2] ] # amplify all negative rewards in the sticky states by a factor of 10, keeping the probabilities the same. for sticky_state in sticky_states: for a in gridworld.p_S_prime_R_given_S_A[sticky_state]: for s_prime in gridworld.p_S_prime_R_given_S_A[sticky_state][a]: gridworld.p_S_prime_R_given_S_A[sticky_state][a][s_prime] = { Reward(r.i, (r.r * 10.0 if r.r < 0.0 else r.r)): gridworld.p_S_prime_R_given_S_A[sticky_state][a][s_prime] [r] for r in gridworld.p_S_prime_R_given_S_A[sticky_state][a] [s_prime] } epsilon = 0.1 q_S_A = TabularStateActionValueEstimator( environment=gridworld, epsilon=epsilon, continuous_state_discretization_resolution=None) pi = q_S_A.get_initial_policy() mdp_agent = StochasticMdpAgent(name='agent', random_state=random, pi=pi, gamma=1.0) # iterate the agents policy using q-learning temporal differencing iterate_value_q_pi(agent=mdp_agent, environment=gridworld, num_improvements=20, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=None, mode=Mode.Q_LEARNING, n_steps=None, planning_environment=None, make_final_policy_greedy=True, q_S_A=q_S_A, num_improvements_per_plot=20) for s in pi: print(f'State {s.i}:') for a in pi[s]: if pi[s][a] > 0.0: print(f'\tPr({a.name}): {pi[s][a]}')
def test_iterate_value_q_pi_with_pdf(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi( agent=mdp_agent, environment=mdp_environment, num_improvements=3000, num_episodes_per_improvement=1, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, num_improvements_per_plot=1500, pdf_save_path=tempfile.NamedTemporaryFile(delete=False).name) with pytest.raises(ValueError, match='Epsilon must be >= 0'): q_S_A.epsilon = -1.0 q_S_A.improve_policy(mdp_agent, states=None, event=PolicyImprovementEvent.MAKING_POLICY_GREEDY) q_S_A.epsilon = 0.0 assert q_S_A.improve_policy( mdp_agent, None, PolicyImprovementEvent.MAKING_POLICY_GREEDY) == 14
def test_run(): random_state = RandomState(12345) mdp_environment: GamblersProblem = GamblersProblem( 'gamblers problem', random_state=random_state, T=None, p_h=0.4) agent = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) monitor = Monitor() state = mdp_environment.reset_for_new_run(agent) agent.reset_for_new_run(state) mdp_environment.run(agent, monitor) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_run.pickle', 'wb') as file: # pickle.dump(monitor, file) with open(f'{os.path.dirname(__file__)}/fixtures/test_run.pickle', 'rb') as file: fixture = pickle.load(file) assert monitor.t_average_reward == fixture.t_average_reward
def test_sarsa_iterate_value_q_pi_with_trajectory_planning(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None) mdp_agent = ActionValueMdpAgent('test', random_state, 1, q_S_A) planning_environment = TrajectorySamplingMdpPlanningEnvironment( 'test planning', random_state, StochasticEnvironmentModel(), 10, None) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=100, num_episodes_per_improvement=1, num_updates_per_improvement=None, alpha=0.1, mode=Mode.SARSA, n_steps=1, planning_environment=planning_environment, make_final_policy_greedy=True) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_td_iteration_of_value_q_pi_planning.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open( f'{os.path.dirname(__file__)}/fixtures/test_td_iteration_of_value_q_pi_planning.pickle', 'rb') as file: pi_fixture, q_S_A_fixture = pickle.load(file) assert tabular_pi_legacy_eq(mdp_agent.pi, pi_fixture) and tabular_estimator_legacy_eq( q_S_A, q_S_A_fixture)
def test_invalid_improve_policy_with_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) epsilon = 0.0 q_S_A = TabularStateActionValueEstimator(mdp_environment, epsilon, None) # target agent mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) with pytest.raises(ValueError, match='Epsilon must be >= 0'): improve_policy_with_q_pi(mdp_agent, {}, -1)
def test_invalid_epsilon(): with pytest.raises(ValueError, match='epsilon must be >= 0'): TabularStateActionValueEstimator(None, -1, None) with pytest.raises(ValueError, match='epsilon must be >= 0'): ApproximateStateActionValueEstimator(None, -1, None, None, None, False, None, None)
def test_off_policy_monte_carlo_with_function_approximation(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = ApproximateStateActionValueEstimator( mdp_environment, 0.05, SKLearnSGD(random_state=random_state, scale_eta0_for_y=False), GridworldFeatureExtractor(mdp_environment), None, False, None, None ) # target agent mdp_agent = ActionValueMdpAgent( 'test', random_state, 1, q_S_A ) # episode generation (behavior) policy off_policy_agent = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None) ) iterate_value_q_pi( agent=mdp_agent, environment=mdp_environment, num_improvements=100, num_episodes_per_improvement=1, update_upon_every_visit=True, planning_environment=None, make_final_policy_greedy=False, off_policy_agent=off_policy_agent ) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_off_policy_monte_carlo_with_function_approximationo.pickle', 'wb') as file: # pickle.dump((mdp_agent.pi, q_S_A), file) with open(f'{os.path.dirname(__file__)}/fixtures/test_off_policy_monte_carlo_with_function_approximationo.pickle', 'rb') as file: pi_fixture, q_S_A_fixture = pickle.load(file) assert mdp_agent.pi == pi_fixture and q_S_A == q_S_A_fixture assert str(mdp_agent.pi.estimator[mdp_environment.SS[5]][mdp_environment.SS[5].AA[1]]).startswith('-2.4305') # make greedy q_S_A.epsilon = 0.0 assert q_S_A.improve_policy(mdp_agent, None, PolicyImprovementEvent.MAKING_POLICY_GREEDY) == -1 assert mdp_agent.pi.estimator.epsilon == 0.0
def test_invalid_iterate_value_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.0, None) # target agent mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) # episode generation (behavior) policy off_policy_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) with pytest.raises( ValueError, match= 'Planning environments are not currently supported for Monte Carlo iteration.' ): iterate_value_q_pi( agent=mdp_agent, environment=mdp_environment, num_improvements=100, num_episodes_per_improvement=1, update_upon_every_visit=True, planning_environment=TrajectorySamplingMdpPlanningEnvironment( 'foo', random_state, StochasticEnvironmentModel(), 100, None), make_final_policy_greedy=False, q_S_A=q_S_A, off_policy_agent=off_policy_agent) # test warning...no off-policy agent with epsilon=0.0 q_S_A.epsilon = 0.0 iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=100, num_episodes_per_improvement=1, update_upon_every_visit=True, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, off_policy_agent=None)
def train_thread_target(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=1000000, num_episodes_per_improvement=10, update_upon_every_visit=False, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A, thread_manager=thread_manager, num_improvements_per_plot=10)
def test_invalid_improve_policy_with_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) epsilon = 0.0 mdp_agent = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, epsilon, None)) with pytest.raises(ValueError, match='Epsilon must be >= 0'): improve_policy_with_q_pi(mdp_agent, {}, -1)
def test_q_learning_iterate_value_q_pi_tabular_policy_ne(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, 20) epsilon = 0.05 q_S_A_1 = TabularStateActionValueEstimator(mdp_environment, epsilon, None) mdp_agent_1 = ActionValueMdpAgent('test', random_state, 1, q_S_A_1) iterate_value_q_pi(agent=mdp_agent_1, environment=mdp_environment, num_improvements=5, num_episodes_per_improvement=10, num_updates_per_improvement=None, alpha=None, mode=Mode.Q_LEARNING, n_steps=None, planning_environment=None, make_final_policy_greedy=True) q_S_A_2 = TabularStateActionValueEstimator(mdp_environment, epsilon, None) mdp_agent_2 = ActionValueMdpAgent('test', random_state, 1, q_S_A_2) iterate_value_q_pi(agent=mdp_agent_2, environment=mdp_environment, num_improvements=5, num_episodes_per_improvement=5, num_updates_per_improvement=None, alpha=None, mode=Mode.Q_LEARNING, n_steps=None, planning_environment=None, make_final_policy_greedy=True) test_state = mdp_environment.SS[5] test_action = test_state.AA[0] assert q_S_A_1 != q_S_A_2 assert q_S_A_1[test_state] != q_S_A_2[test_state] assert q_S_A_1[test_state][test_action] != q_S_A_2[test_state][test_action]
def test_policy_iteration(): # state-value policy iteration random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent_v_pi = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) iterate_policy_v_pi(mdp_agent_v_pi, mdp_environment, 0.001, True) # action-value policy iteration random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent_q_pi = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) iterate_policy_q_pi(mdp_agent_q_pi, mdp_environment, 0.001, True) # should get the same policy assert mdp_agent_v_pi.pi == mdp_agent_q_pi.pi
def test_evaluate_v_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) v_pi, _ = evaluate_v_pi(agent=mdp_agent, environment=mdp_environment, theta=0.001, num_iterations=None, update_in_place=True) random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) v_pi_not_in_place, _ = evaluate_v_pi(agent=mdp_agent, environment=mdp_environment, theta=0.001, num_iterations=None, update_in_place=False) assert list(v_pi.keys()) == list(v_pi_not_in_place.keys()) np.testing.assert_allclose(list(v_pi.values()), list(v_pi_not_in_place.values()), atol=0.01) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_iterative_policy_evaluation_of_state_value.pickle', 'wb') as file: # pickle.dump(v_pi, file) with open( f'{os.path.dirname(__file__)}/fixtures/test_iterative_policy_evaluation_of_state_value.pickle', 'rb') as file: fixture = pickle.load(file) assert v_pi == fixture
def test_evaluate_q_pi_invalid_n_steps(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) epsilon = 0.05 q_S_A = TabularStateActionValueEstimator(mdp_environment, epsilon, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) with pytest.raises(ValueError): evaluate_q_pi(agent=mdp_agent, environment=mdp_environment, num_episodes=5, num_updates_per_improvement=None, alpha=0.1, mode=Mode.Q_LEARNING, n_steps=-1, planning_environment=None, q_S_A=q_S_A)
def test_invalid_epsilon_iterate_value_q_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.0, None) mdp_agent = StochasticMdpAgent('test', random_state, q_S_A.get_initial_policy(), 1) with pytest.raises(ValueError, match='epsilon must be strictly > 0 for TD-learning'): iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=10, num_episodes_per_improvement=100, num_updates_per_improvement=None, alpha=0.1, mode=Mode.Q_LEARNING, n_steps=3, planning_environment=None, make_final_policy_greedy=False, q_S_A=q_S_A)
def train_thread_target(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, 0.1, None)) iterate_value_q_pi(agent=mdp_agent, environment=mdp_environment, num_improvements=1000000, num_episodes_per_improvement=10, num_updates_per_improvement=None, alpha=0.1, mode=Mode.SARSA, n_steps=None, planning_environment=None, make_final_policy_greedy=False, thread_manager=thread_manager, num_improvements_per_plot=10)
def test_gamblers_problem(): random_state = RandomState(12345) mdp_environment: GamblersProblem = GamblersProblem( 'gamblers problem', random_state=random_state, T=None, p_h=0.4) mdp_agent_v_pi_value_iteration = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) v_pi = iterate_value_v_pi(mdp_agent_v_pi_value_iteration, mdp_environment, 0.001, 1, True) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_gamblers_problem.pickle', 'wb') as file: # pickle.dump(v_pi, file) with open( f'{os.path.dirname(__file__)}/fixtures/test_gamblers_problem.pickle', 'rb') as file: fixture = pickle.load(file) assert v_pi == fixture
def test_agent_invalid_action(): random = RandomState() agent = ActionValueMdpAgent( 'foo', random, 1.0, TabularStateActionValueEstimator(Gridworld.example_4_1(random, None), None, None)) # test None action agent.__act__ = lambda t: None with pytest.raises(ValueError, match='Agent returned action of None'): agent.act(0) # test infeasible action action = Action(1, 'foo') agent.__act__ = lambda t: action state = MdpState(1, [], False) agent.sense(state, 0) with pytest.raises( ValueError, match=f'Action {action} is not feasible in state {state}'): agent.act(0)
def test_evaluate_v_pi(): random_state = RandomState(12345) mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None) mdp_agent = ActionValueMdpAgent( 'test', random_state, 1, TabularStateActionValueEstimator(mdp_environment, None, None)) v_pi = evaluate_v_pi(agent=mdp_agent, environment=mdp_environment, num_episodes=1000) # uncomment the following line and run test to update fixture # with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_value.pickle', 'wb') as file: # pickle.dump(v_pi, file) with open( f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_value.pickle', 'rb') as file: fixture = pickle.load(file) assert v_pi == fixture
def __init__( self, random_state: RandomState, T: Optional[int], initial_count: int, player_2: Optional[Agent] ): """ Initialize the game. :param random_state: Random state. :param T: Maximum number of steps to run, or None for no limit. :param initial_count: Initial count for each pit. :param player_2: Agent for player 2, or None to use a random agent. """ super().__init__( name='mancala', random_state=random_state, T=T ) if player_2 is None: player_2 = ActionValueMdpAgent( 'environmental agent', random_state, 1, TabularStateActionValueEstimator(self, None, None) ) self.initial_count = initial_count self.player_2 = player_2 self.r_win = Reward(0, 1.0) self.r_lose = Reward(1, -1.0) self.r_none = Reward(2, 0.0) self.player_1_pockets = [ Pit(True, self.initial_count, False) for _ in range(6) ] self.player_1_store = Pit(True, 0, True) self.player_2_pockets = [ Pit(False, self.initial_count, False) for _ in range(6) ] self.player_2_store = Pit(False, 0, True) self.board = self.player_1_pockets + [self.player_1_store] + self.player_2_pockets + [self.player_2_store] for i, pit in enumerate(self.board): pit.i = i # non-store pit (i.e., pockets) have actions associated with them. Action.i indexes the particular pit # within the board. if not pit.store: pit.action = Action(pit.i) # Action.name indicates the i-th pit from the player's perspective for i, pit in enumerate(self.player_1_pockets): pit.action.name = str(i) for i, pit in enumerate(self.player_2_pockets): pit.action.name = str(i) for player_1_pocket, opposing_player_2_pocket in zip(self.player_1_pockets, reversed(self.player_2_pockets)): player_1_pocket.opposing_pocket = opposing_player_2_pocket opposing_player_2_pocket.opposing_pocket = player_1_pocket