def test_evaluate_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, None, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
evaluated_states, _ = evaluate_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_episodes=1000,
exploring_starts=True,
update_upon_every_visit=False,
q_S_A=q_S_A)
assert len(
q_S_A) == len(evaluated_states) + 2 # terminal states aren't evaluated
assert all(s in q_S_A for s in evaluated_states)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_action_value.pickle', 'wb') as file:
# pickle.dump(q_S_A, file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_action_value.pickle',
'rb') as file:
fixture = pickle.load(file)
assert tabular_estimator_legacy_eq(q_S_A, fixture)
def test_iterate_value_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=3000,
num_episodes_per_improvement=1,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_iteration_of_value_q_pi.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_iteration_of_value_q_pi.pickle',
'rb') as file:
pi_fixture, q_S_A_fixture = pickle.load(file)
assert tabular_pi_legacy_eq(mdp_agent.pi,
pi_fixture) and tabular_estimator_legacy_eq(
q_S_A, q_S_A_fixture)
def test_value_iteration():
# run policy iteration on v_pi
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent_v_pi_policy_iteration = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
iterate_policy_v_pi(mdp_agent_v_pi_policy_iteration, mdp_environment,
0.001, True)
# run value iteration on v_pi
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent_v_pi_value_iteration = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
iterate_value_v_pi(mdp_agent_v_pi_value_iteration, mdp_environment, 0.001,
1, True)
assert mdp_agent_v_pi_policy_iteration.pi == mdp_agent_v_pi_value_iteration.pi
# run value iteration on q_pi
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent_q_pi_value_iteration = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
iterate_value_q_pi(mdp_agent_q_pi_value_iteration, mdp_environment, 0.001,
1, True)
assert mdp_agent_q_pi_value_iteration.pi == mdp_agent_v_pi_policy_iteration.pi
def test_iterate_value_q_pi_with_pdf():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(
agent=mdp_agent,
environment=mdp_environment,
num_improvements=10,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.Q_LEARNING,
n_steps=1,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
num_improvements_per_plot=5,
pdf_save_path=tempfile.NamedTemporaryFile(delete=False).name)
def test_learn():
random_state = RandomState(12345)
gym = Gym(random_state=random_state, T=None, gym_id='CartPole-v1')
q_S_A = TabularStateActionValueEstimator(gym, 0.05, 0.001)
mdp_agent = StochasticMdpAgent('agent', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent,
environment=gym,
num_improvements=10,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.SARSA,
n_steps=1,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_gym.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(f'{os.path.dirname(__file__)}/fixtures/test_gym.pickle',
'rb') as file:
fixture_pi, fixture_q_S_A = pickle.load(file)
assert tabular_pi_legacy_eq(mdp_agent.pi,
fixture_pi) and tabular_estimator_legacy_eq(
q_S_A, fixture_q_S_A)
def test_n_step_q_learning_iterate_value_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=10,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.Q_LEARNING,
n_steps=3,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_td_n_step_q_learning_iteration_of_value_q_pi.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_td_n_step_q_learning_iteration_of_value_q_pi.pickle',
'rb') as file:
fixture_pi, fixture_q_S_A = pickle.load(file)
assert tabular_pi_legacy_eq(mdp_agent.pi,
fixture_pi) and tabular_estimator_legacy_eq(
q_S_A, fixture_q_S_A)
def test_human_player():
random_state = RandomState(12345)
human = Human()
def mock_input(prompt: str) -> str:
s = human.most_recent_state
selected_a = sample_list_item(s.AA,
probs=None,
random_state=random_state)
return selected_a.name
human.get_input = mock_input
mancala: Mancala = Mancala(random_state=random_state,
T=None,
initial_count=4,
player_2=human)
epsilon = 0.05
q_S_A = TabularStateActionValueEstimator(mancala, epsilon, None)
p1 = StochasticMdpAgent('player 1', random_state,
q_S_A.get_initial_policy(), 1)
state = mancala.reset_for_new_run(p1)
p1.reset_for_new_run(state)
a = p1.act(0)
state, reward = mancala.advance(state, 0, a, p1)
assert mancala.board[7].count == 0 and state.i == 1 and reward.i == 2
def test_learn():
random_state = RandomState(12345)
mancala: Mancala = Mancala(random_state=random_state,
T=None,
initial_count=4,
player_2=None)
p1 = ActionValueMdpAgent(
'player 1', random_state, 1,
TabularStateActionValueEstimator(mancala, 0.05, None))
checkpoint_path = iterate_value_q_pi(
agent=p1,
environment=mancala,
num_improvements=3,
num_episodes_per_improvement=100,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
num_improvements_per_checkpoint=3,
checkpoint_path=tempfile.NamedTemporaryFile(delete=False).name)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_mancala.pickle', 'wb') as file:
# pickle.dump(p1.pi, file)
with open(f'{os.path.dirname(__file__)}/fixtures/test_mancala.pickle',
'rb') as file:
fixture = pickle.load(file)
assert tabular_pi_legacy_eq(p1.pi, fixture)
resumed_p1 = resume_from_checkpoint(checkpoint_path=checkpoint_path,
resume_function=iterate_value_q_pi,
num_improvements=2)
# run same number of improvements without checkpoint...result should be the same.
random_state = RandomState(12345)
mancala: Mancala = Mancala(random_state=random_state,
T=None,
initial_count=4,
player_2=None)
no_checkpoint_p1 = ActionValueMdpAgent(
'player 1', random_state, 1,
TabularStateActionValueEstimator(mancala, 0.05, None))
iterate_value_q_pi(agent=no_checkpoint_p1,
environment=mancala,
num_improvements=5,
num_episodes_per_improvement=100,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False)
assert no_checkpoint_p1.pi == resumed_p1.pi
def main():
random = RandomState(12345)
gridworld = Gridworld.example_4_1(random, None)
# the bottom-right corner (3,3) is a goal state. get the states surrounding this goal. these will become the sticky
# states.
sticky_states = [
gridworld.grid[2, 2], gridworld.grid[2, 3], gridworld.grid[3, 2]
]
# amplify all negative rewards in the sticky states by a factor of 10, keeping the probabilities the same.
for sticky_state in sticky_states:
for a in gridworld.p_S_prime_R_given_S_A[sticky_state]:
for s_prime in gridworld.p_S_prime_R_given_S_A[sticky_state][a]:
gridworld.p_S_prime_R_given_S_A[sticky_state][a][s_prime] = {
Reward(r.i, (r.r * 10.0 if r.r < 0.0 else r.r)):
gridworld.p_S_prime_R_given_S_A[sticky_state][a][s_prime]
[r]
for r in gridworld.p_S_prime_R_given_S_A[sticky_state][a]
[s_prime]
}
epsilon = 0.1
q_S_A = TabularStateActionValueEstimator(
environment=gridworld,
epsilon=epsilon,
continuous_state_discretization_resolution=None)
pi = q_S_A.get_initial_policy()
mdp_agent = StochasticMdpAgent(name='agent',
random_state=random,
pi=pi,
gamma=1.0)
# iterate the agents policy using q-learning temporal differencing
iterate_value_q_pi(agent=mdp_agent,
environment=gridworld,
num_improvements=20,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=None,
mode=Mode.Q_LEARNING,
n_steps=None,
planning_environment=None,
make_final_policy_greedy=True,
q_S_A=q_S_A,
num_improvements_per_plot=20)
for s in pi:
print(f'State {s.i}:')
for a in pi[s]:
if pi[s][a] > 0.0:
print(f'\tPr({a.name}): {pi[s][a]}')
def test_iterate_value_q_pi_with_pdf():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(
agent=mdp_agent,
environment=mdp_environment,
num_improvements=3000,
num_episodes_per_improvement=1,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
num_improvements_per_plot=1500,
pdf_save_path=tempfile.NamedTemporaryFile(delete=False).name)
with pytest.raises(ValueError, match='Epsilon must be >= 0'):
q_S_A.epsilon = -1.0
q_S_A.improve_policy(mdp_agent,
states=None,
event=PolicyImprovementEvent.MAKING_POLICY_GREEDY)
q_S_A.epsilon = 0.0
assert q_S_A.improve_policy(
mdp_agent, None, PolicyImprovementEvent.MAKING_POLICY_GREEDY) == 14
def test_run():
random_state = RandomState(12345)
mdp_environment: GamblersProblem = GamblersProblem(
'gamblers problem', random_state=random_state, T=None, p_h=0.4)
agent = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
monitor = Monitor()
state = mdp_environment.reset_for_new_run(agent)
agent.reset_for_new_run(state)
mdp_environment.run(agent, monitor)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_run.pickle', 'wb') as file:
# pickle.dump(monitor, file)
with open(f'{os.path.dirname(__file__)}/fixtures/test_run.pickle',
'rb') as file:
fixture = pickle.load(file)
assert monitor.t_average_reward == fixture.t_average_reward
def test_sarsa_iterate_value_q_pi_with_trajectory_planning():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None)
mdp_agent = ActionValueMdpAgent('test', random_state, 1, q_S_A)
planning_environment = TrajectorySamplingMdpPlanningEnvironment(
'test planning', random_state, StochasticEnvironmentModel(), 10, None)
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=100,
num_episodes_per_improvement=1,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.SARSA,
n_steps=1,
planning_environment=planning_environment,
make_final_policy_greedy=True)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_td_iteration_of_value_q_pi_planning.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_td_iteration_of_value_q_pi_planning.pickle',
'rb') as file:
pi_fixture, q_S_A_fixture = pickle.load(file)
assert tabular_pi_legacy_eq(mdp_agent.pi,
pi_fixture) and tabular_estimator_legacy_eq(
q_S_A, q_S_A_fixture)
def test_invalid_improve_policy_with_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
epsilon = 0.0
q_S_A = TabularStateActionValueEstimator(mdp_environment, epsilon, None)
# target agent
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
with pytest.raises(ValueError, match='Epsilon must be >= 0'):
improve_policy_with_q_pi(mdp_agent, {}, -1)
def test_invalid_epsilon():
with pytest.raises(ValueError, match='epsilon must be >= 0'):
TabularStateActionValueEstimator(None, -1, None)
with pytest.raises(ValueError, match='epsilon must be >= 0'):
ApproximateStateActionValueEstimator(None, -1, None, None, None, False,
None, None)
def test_off_policy_monte_carlo_with_function_approximation():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = ApproximateStateActionValueEstimator(
mdp_environment,
0.05,
SKLearnSGD(random_state=random_state, scale_eta0_for_y=False),
GridworldFeatureExtractor(mdp_environment),
None,
False,
None,
None
)
# target agent
mdp_agent = ActionValueMdpAgent(
'test',
random_state,
1,
q_S_A
)
# episode generation (behavior) policy
off_policy_agent = ActionValueMdpAgent(
'test',
random_state,
1,
TabularStateActionValueEstimator(mdp_environment, None, None)
)
iterate_value_q_pi(
agent=mdp_agent,
environment=mdp_environment,
num_improvements=100,
num_episodes_per_improvement=1,
update_upon_every_visit=True,
planning_environment=None,
make_final_policy_greedy=False,
off_policy_agent=off_policy_agent
)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_off_policy_monte_carlo_with_function_approximationo.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(f'{os.path.dirname(__file__)}/fixtures/test_off_policy_monte_carlo_with_function_approximationo.pickle', 'rb') as file:
pi_fixture, q_S_A_fixture = pickle.load(file)
assert mdp_agent.pi == pi_fixture and q_S_A == q_S_A_fixture
assert str(mdp_agent.pi.estimator[mdp_environment.SS[5]][mdp_environment.SS[5].AA[1]]).startswith('-2.4305')
# make greedy
q_S_A.epsilon = 0.0
assert q_S_A.improve_policy(mdp_agent, None, PolicyImprovementEvent.MAKING_POLICY_GREEDY) == -1
assert mdp_agent.pi.estimator.epsilon == 0.0
def test_invalid_iterate_value_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.0, None)
# target agent
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
# episode generation (behavior) policy
off_policy_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
with pytest.raises(
ValueError,
match=
'Planning environments are not currently supported for Monte Carlo iteration.'
):
iterate_value_q_pi(
agent=mdp_agent,
environment=mdp_environment,
num_improvements=100,
num_episodes_per_improvement=1,
update_upon_every_visit=True,
planning_environment=TrajectorySamplingMdpPlanningEnvironment(
'foo', random_state, StochasticEnvironmentModel(), 100, None),
make_final_policy_greedy=False,
q_S_A=q_S_A,
off_policy_agent=off_policy_agent)
# test warning...no off-policy agent with epsilon=0.0
q_S_A.epsilon = 0.0
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=100,
num_episodes_per_improvement=1,
update_upon_every_visit=True,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
off_policy_agent=None)
def train_thread_target():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=1000000,
num_episodes_per_improvement=10,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
thread_manager=thread_manager,
num_improvements_per_plot=10)
def test_invalid_improve_policy_with_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
epsilon = 0.0
mdp_agent = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, epsilon, None))
with pytest.raises(ValueError, match='Epsilon must be >= 0'):
improve_policy_with_q_pi(mdp_agent, {}, -1)
def test_q_learning_iterate_value_q_pi_tabular_policy_ne():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, 20)
epsilon = 0.05
q_S_A_1 = TabularStateActionValueEstimator(mdp_environment, epsilon, None)
mdp_agent_1 = ActionValueMdpAgent('test', random_state, 1, q_S_A_1)
iterate_value_q_pi(agent=mdp_agent_1,
environment=mdp_environment,
num_improvements=5,
num_episodes_per_improvement=10,
num_updates_per_improvement=None,
alpha=None,
mode=Mode.Q_LEARNING,
n_steps=None,
planning_environment=None,
make_final_policy_greedy=True)
q_S_A_2 = TabularStateActionValueEstimator(mdp_environment, epsilon, None)
mdp_agent_2 = ActionValueMdpAgent('test', random_state, 1, q_S_A_2)
iterate_value_q_pi(agent=mdp_agent_2,
environment=mdp_environment,
num_improvements=5,
num_episodes_per_improvement=5,
num_updates_per_improvement=None,
alpha=None,
mode=Mode.Q_LEARNING,
n_steps=None,
planning_environment=None,
make_final_policy_greedy=True)
test_state = mdp_environment.SS[5]
test_action = test_state.AA[0]
assert q_S_A_1 != q_S_A_2
assert q_S_A_1[test_state] != q_S_A_2[test_state]
assert q_S_A_1[test_state][test_action] != q_S_A_2[test_state][test_action]
def test_policy_iteration():
# state-value policy iteration
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent_v_pi = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
iterate_policy_v_pi(mdp_agent_v_pi, mdp_environment, 0.001, True)
# action-value policy iteration
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent_q_pi = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
iterate_policy_q_pi(mdp_agent_q_pi, mdp_environment, 0.001, True)
# should get the same policy
assert mdp_agent_v_pi.pi == mdp_agent_q_pi.pi
def test_evaluate_v_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
v_pi, _ = evaluate_v_pi(agent=mdp_agent,
environment=mdp_environment,
theta=0.001,
num_iterations=None,
update_in_place=True)
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
v_pi_not_in_place, _ = evaluate_v_pi(agent=mdp_agent,
environment=mdp_environment,
theta=0.001,
num_iterations=None,
update_in_place=False)
assert list(v_pi.keys()) == list(v_pi_not_in_place.keys())
np.testing.assert_allclose(list(v_pi.values()),
list(v_pi_not_in_place.values()),
atol=0.01)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_iterative_policy_evaluation_of_state_value.pickle', 'wb') as file:
# pickle.dump(v_pi, file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_iterative_policy_evaluation_of_state_value.pickle',
'rb') as file:
fixture = pickle.load(file)
assert v_pi == fixture
def test_evaluate_q_pi_invalid_n_steps():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
epsilon = 0.05
q_S_A = TabularStateActionValueEstimator(mdp_environment, epsilon, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
with pytest.raises(ValueError):
evaluate_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_episodes=5,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.Q_LEARNING,
n_steps=-1,
planning_environment=None,
q_S_A=q_S_A)
def test_invalid_epsilon_iterate_value_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.0, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
with pytest.raises(ValueError,
match='epsilon must be strictly > 0 for TD-learning'):
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=10,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.Q_LEARNING,
n_steps=3,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
def train_thread_target():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, 0.1, None))
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=1000000,
num_episodes_per_improvement=10,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.SARSA,
n_steps=None,
planning_environment=None,
make_final_policy_greedy=False,
thread_manager=thread_manager,
num_improvements_per_plot=10)
def test_gamblers_problem():
random_state = RandomState(12345)
mdp_environment: GamblersProblem = GamblersProblem(
'gamblers problem', random_state=random_state, T=None, p_h=0.4)
mdp_agent_v_pi_value_iteration = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
v_pi = iterate_value_v_pi(mdp_agent_v_pi_value_iteration, mdp_environment,
0.001, 1, True)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_gamblers_problem.pickle', 'wb') as file:
# pickle.dump(v_pi, file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_gamblers_problem.pickle',
'rb') as file:
fixture = pickle.load(file)
assert v_pi == fixture
def test_agent_invalid_action():
random = RandomState()
agent = ActionValueMdpAgent(
'foo', random, 1.0,
TabularStateActionValueEstimator(Gridworld.example_4_1(random, None),
None, None))
# test None action
agent.__act__ = lambda t: None
with pytest.raises(ValueError, match='Agent returned action of None'):
agent.act(0)
# test infeasible action
action = Action(1, 'foo')
agent.__act__ = lambda t: action
state = MdpState(1, [], False)
agent.sense(state, 0)
with pytest.raises(
ValueError,
match=f'Action {action} is not feasible in state {state}'):
agent.act(0)
def test_evaluate_v_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
mdp_agent = ActionValueMdpAgent(
'test', random_state, 1,
TabularStateActionValueEstimator(mdp_environment, None, None))
v_pi = evaluate_v_pi(agent=mdp_agent,
environment=mdp_environment,
num_episodes=1000)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_value.pickle', 'wb') as file:
# pickle.dump(v_pi, file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_value.pickle',
'rb') as file:
fixture = pickle.load(file)
assert v_pi == fixture
def __init__(
self,
random_state: RandomState,
T: Optional[int],
initial_count: int,
player_2: Optional[Agent]
):
"""
Initialize the game.
:param random_state: Random state.
:param T: Maximum number of steps to run, or None for no limit.
:param initial_count: Initial count for each pit.
:param player_2: Agent for player 2, or None to use a random agent.
"""
super().__init__(
name='mancala',
random_state=random_state,
T=T
)
if player_2 is None:
player_2 = ActionValueMdpAgent(
'environmental agent',
random_state,
1,
TabularStateActionValueEstimator(self, None, None)
)
self.initial_count = initial_count
self.player_2 = player_2
self.r_win = Reward(0, 1.0)
self.r_lose = Reward(1, -1.0)
self.r_none = Reward(2, 0.0)
self.player_1_pockets = [
Pit(True, self.initial_count, False)
for _ in range(6)
]
self.player_1_store = Pit(True, 0, True)
self.player_2_pockets = [
Pit(False, self.initial_count, False)
for _ in range(6)
]
self.player_2_store = Pit(False, 0, True)
self.board = self.player_1_pockets + [self.player_1_store] + self.player_2_pockets + [self.player_2_store]
for i, pit in enumerate(self.board):
pit.i = i
# non-store pit (i.e., pockets) have actions associated with them. Action.i indexes the particular pit
# within the board.
if not pit.store:
pit.action = Action(pit.i)
# Action.name indicates the i-th pit from the player's perspective
for i, pit in enumerate(self.player_1_pockets):
pit.action.name = str(i)
for i, pit in enumerate(self.player_2_pockets):
pit.action.name = str(i)
for player_1_pocket, opposing_player_2_pocket in zip(self.player_1_pockets, reversed(self.player_2_pockets)):
player_1_pocket.opposing_pocket = opposing_player_2_pocket
opposing_player_2_pocket.opposing_pocket = player_1_pocket
