def test_q_learning_iterate_value_q_pi_tabular_policy_ne():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, 20)
epsilon = 0.05
q_S_A_1 = TabularStateActionValueEstimator(mdp_environment, epsilon, None)
mdp_agent_1 = StochasticMdpAgent('test', random_state,
q_S_A_1.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent_1,
environment=mdp_environment,
num_improvements=5,
num_episodes_per_improvement=10,
num_updates_per_improvement=None,
alpha=None,
mode=Mode.Q_LEARNING,
n_steps=None,
planning_environment=None,
make_final_policy_greedy=True,
q_S_A=q_S_A_1)
q_S_A_2 = TabularStateActionValueEstimator(mdp_environment, epsilon, None)
mdp_agent_2 = StochasticMdpAgent('test', random_state,
q_S_A_2.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent_2,
environment=mdp_environment,
num_improvements=5,
num_episodes_per_improvement=5,
num_updates_per_improvement=None,
alpha=None,
mode=Mode.Q_LEARNING,
n_steps=None,
planning_environment=None,
make_final_policy_greedy=True,
q_S_A=q_S_A_2)
test_state = mdp_environment.SS[5]
test_action = test_state.AA[0]
assert q_S_A_1 != q_S_A_2
assert q_S_A_1[test_state] != q_S_A_2[test_state]
assert q_S_A_1[test_state][test_action] != q_S_A_2[test_state][test_action]
def test_iterate_value_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=3000,
num_episodes_per_improvement=1,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_iteration_of_value_q_pi.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_iteration_of_value_q_pi.pickle',
'rb') as file:
pi_fixture, q_S_A_fixture = pickle.load(file)
assert tabular_pi_legacy_eq(mdp_agent.pi,
pi_fixture) and tabular_estimator_legacy_eq(
q_S_A, q_S_A_fixture)
def test_iterate_value_q_pi_with_pdf():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(
agent=mdp_agent,
environment=mdp_environment,
num_improvements=3000,
num_episodes_per_improvement=1,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
num_improvements_per_plot=1500,
pdf_save_path=tempfile.NamedTemporaryFile(delete=False).name)
with pytest.raises(ValueError, match='Epsilon must be >= 0'):
q_S_A.epsilon = -1.0
q_S_A.improve_policy(mdp_agent,
states=None,
event=PolicyImprovementEvent.MAKING_POLICY_GREEDY)
q_S_A.epsilon = 0.0
assert q_S_A.improve_policy(
mdp_agent, None, PolicyImprovementEvent.MAKING_POLICY_GREEDY) == 14
def test_learn():
random_state = RandomState(12345)
gym = Gym(random_state=random_state, T=None, gym_id='CartPole-v1')
q_S_A = TabularStateActionValueEstimator(gym, 0.05, 0.001)
mdp_agent = StochasticMdpAgent('agent', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent,
environment=gym,
num_improvements=10,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.SARSA,
n_steps=1,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_gym.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(f'{os.path.dirname(__file__)}/fixtures/test_gym.pickle',
'rb') as file:
fixture_pi, fixture_q_S_A = pickle.load(file)
assert tabular_pi_legacy_eq(mdp_agent.pi,
fixture_pi) and tabular_estimator_legacy_eq(
q_S_A, fixture_q_S_A)
def test_evaluate_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, None, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
evaluated_states, _ = evaluate_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_episodes=1000,
exploring_starts=True,
update_upon_every_visit=False,
q_S_A=q_S_A)
assert len(
q_S_A) == len(evaluated_states) + 2 # terminal states aren't evaluated
assert all(s in q_S_A for s in evaluated_states)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_action_value.pickle', 'wb') as file:
# pickle.dump(q_S_A, file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_monte_carlo_evaluation_of_state_action_value.pickle',
'rb') as file:
fixture = pickle.load(file)
assert tabular_estimator_legacy_eq(q_S_A, fixture)
def test_iterate_value_q_pi_with_pdf():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(
agent=mdp_agent,
environment=mdp_environment,
num_improvements=10,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.Q_LEARNING,
n_steps=1,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
num_improvements_per_plot=5,
pdf_save_path=tempfile.NamedTemporaryFile(delete=False).name)
def test_n_step_q_learning_iterate_value_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=10,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.Q_LEARNING,
n_steps=3,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_td_n_step_q_learning_iteration_of_value_q_pi.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_td_n_step_q_learning_iteration_of_value_q_pi.pickle',
'rb') as file:
fixture_pi, fixture_q_S_A = pickle.load(file)
assert tabular_pi_legacy_eq(mdp_agent.pi,
fixture_pi) and tabular_estimator_legacy_eq(
q_S_A, fixture_q_S_A)
def test_human_player():
random_state = RandomState(12345)
human = Human()
def mock_input(prompt: str) -> str:
s = human.most_recent_state
selected_a = sample_list_item(s.AA,
probs=None,
random_state=random_state)
return selected_a.name
human.get_input = mock_input
mancala: Mancala = Mancala(random_state=random_state,
T=None,
initial_count=4,
player_2=human)
epsilon = 0.05
q_S_A = TabularStateActionValueEstimator(mancala, epsilon, None)
p1 = StochasticMdpAgent('player 1', random_state,
q_S_A.get_initial_policy(), 1)
state = mancala.reset_for_new_run(p1)
p1.reset_for_new_run(state)
a = p1.act(0)
state, reward = mancala.advance(state, 0, a, p1)
assert mancala.board[7].count == 0 and state.i == 1 and reward.i == 2
def test_invalid_iterate_value_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.0, None)
# target agent
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
# episode generation (behavior) policy
off_policy_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
with pytest.raises(
ValueError,
match=
'Planning environments are not currently supported for Monte Carlo iteration.'
):
iterate_value_q_pi(
agent=mdp_agent,
environment=mdp_environment,
num_improvements=100,
num_episodes_per_improvement=1,
update_upon_every_visit=True,
planning_environment=TrajectorySamplingMdpPlanningEnvironment(
'foo', random_state, StochasticEnvironmentModel(), 100, None),
make_final_policy_greedy=False,
q_S_A=q_S_A,
off_policy_agent=off_policy_agent)
# test warning...no off-policy agent with epsilon=0.0
q_S_A.epsilon = 0.0
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=100,
num_episodes_per_improvement=1,
update_upon_every_visit=True,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
off_policy_agent=None)
def main():
random = RandomState(12345)
gridworld = Gridworld.example_4_1(random, None)
# the bottom-right corner (3,3) is a goal state. get the states surrounding this goal. these will become the sticky
# states.
sticky_states = [
gridworld.grid[2, 2], gridworld.grid[2, 3], gridworld.grid[3, 2]
]
# amplify all negative rewards in the sticky states by a factor of 10, keeping the probabilities the same.
for sticky_state in sticky_states:
for a in gridworld.p_S_prime_R_given_S_A[sticky_state]:
for s_prime in gridworld.p_S_prime_R_given_S_A[sticky_state][a]:
gridworld.p_S_prime_R_given_S_A[sticky_state][a][s_prime] = {
Reward(r.i, (r.r * 10.0 if r.r < 0.0 else r.r)):
gridworld.p_S_prime_R_given_S_A[sticky_state][a][s_prime]
[r]
for r in gridworld.p_S_prime_R_given_S_A[sticky_state][a]
[s_prime]
}
epsilon = 0.1
q_S_A = TabularStateActionValueEstimator(
environment=gridworld,
epsilon=epsilon,
continuous_state_discretization_resolution=None)
pi = q_S_A.get_initial_policy()
mdp_agent = StochasticMdpAgent(name='agent',
random_state=random,
pi=pi,
gamma=1.0)
# iterate the agents policy using q-learning temporal differencing
iterate_value_q_pi(agent=mdp_agent,
environment=gridworld,
num_improvements=20,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=None,
mode=Mode.Q_LEARNING,
n_steps=None,
planning_environment=None,
make_final_policy_greedy=True,
q_S_A=q_S_A,
num_improvements_per_plot=20)
for s in pi:
print(f'State {s.i}:')
for a in pi[s]:
if pi[s][a] > 0.0:
print(f'\tPr({a.name}): {pi[s][a]}')
def test_invalid_improve_policy_with_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
epsilon = 0.0
q_S_A = TabularStateActionValueEstimator(mdp_environment, epsilon, None)
# target agent
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
with pytest.raises(ValueError, match='Epsilon must be >= 0'):
improve_policy_with_q_pi(mdp_agent, {}, -1)
def train_thread_target():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.1, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=1000000,
num_episodes_per_improvement=10,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
thread_manager=thread_manager,
num_improvements_per_plot=10)
def test_sarsa_iterate_value_q_pi_with_trajectory_planning():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.05, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
planning_environment = TrajectorySamplingMdpPlanningEnvironment(
'test planning', random_state, StochasticEnvironmentModel(), 10, None)
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=100,
num_episodes_per_improvement=1,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.SARSA,
n_steps=1,
planning_environment=planning_environment,
make_final_policy_greedy=True,
q_S_A=q_S_A)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_td_iteration_of_value_q_pi_planning.pickle', 'wb') as file:
# pickle.dump((mdp_agent.pi, q_S_A), file)
with open(
f'{os.path.dirname(__file__)}/fixtures/test_td_iteration_of_value_q_pi_planning.pickle',
'rb') as file:
pi_fixture, q_S_A_fixture = pickle.load(file)
assert tabular_pi_legacy_eq(mdp_agent.pi,
pi_fixture) and tabular_estimator_legacy_eq(
q_S_A, q_S_A_fixture)
def test_evaluate_q_pi_invalid_n_steps():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
epsilon = 0.05
q_S_A = TabularStateActionValueEstimator(mdp_environment, epsilon, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
with pytest.raises(ValueError):
evaluate_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_episodes=5,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.Q_LEARNING,
n_steps=-1,
planning_environment=None,
q_S_A=q_S_A)
def test_invalid_epsilon_iterate_value_q_pi():
random_state = RandomState(12345)
mdp_environment: Gridworld = Gridworld.example_4_1(random_state, None)
q_S_A = TabularStateActionValueEstimator(mdp_environment, 0.0, None)
mdp_agent = StochasticMdpAgent('test', random_state,
q_S_A.get_initial_policy(), 1)
with pytest.raises(ValueError,
match='epsilon must be strictly > 0 for TD-learning'):
iterate_value_q_pi(agent=mdp_agent,
environment=mdp_environment,
num_improvements=10,
num_episodes_per_improvement=100,
num_updates_per_improvement=None,
alpha=0.1,
mode=Mode.Q_LEARNING,
n_steps=3,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
def test_learn():
random_state = RandomState(12345)
mancala: Mancala = Mancala(random_state=random_state,
T=None,
initial_count=4,
player_2=StochasticMdpAgent(
'player 2', random_state,
TabularPolicy(None, None), 1))
q_S_A = TabularStateActionValueEstimator(mancala, 0.05, None)
p1 = StochasticMdpAgent('player 1', random_state,
q_S_A.get_initial_policy(), 1)
checkpoint_path = iterate_value_q_pi(
agent=p1,
environment=mancala,
num_improvements=3,
num_episodes_per_improvement=100,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A,
num_improvements_per_checkpoint=3,
checkpoint_path=tempfile.NamedTemporaryFile(delete=False).name)
# uncomment the following line and run test to update fixture
# with open(f'{os.path.dirname(__file__)}/fixtures/test_mancala.pickle', 'wb') as file:
# pickle.dump(p1.pi, file)
with open(f'{os.path.dirname(__file__)}/fixtures/test_mancala.pickle',
'rb') as file:
fixture = pickle.load(file)
assert tabular_pi_legacy_eq(p1.pi, fixture)
resumed_p1 = resume_from_checkpoint(checkpoint_path=checkpoint_path,
resume_function=iterate_value_q_pi,
num_improvements=2)
# run same number of improvements without checkpoint...result should be the same.
random_state = RandomState(12345)
mancala: Mancala = Mancala(random_state=random_state,
T=None,
initial_count=4,
player_2=StochasticMdpAgent(
'player 2', random_state,
TabularPolicy(None, None), 1))
q_S_A = TabularStateActionValueEstimator(mancala, 0.05, None)
no_checkpoint_p1 = StochasticMdpAgent('player 1', random_state,
q_S_A.get_initial_policy(), 1)
iterate_value_q_pi(agent=no_checkpoint_p1,
environment=mancala,
num_improvements=5,
num_episodes_per_improvement=100,
update_upon_every_visit=False,
planning_environment=None,
make_final_policy_greedy=False,
q_S_A=q_S_A)
assert no_checkpoint_p1.pi == resumed_p1.pi
