def mc_control_scratch(
#traces: Iterable[Iterable[mp.TransitionStep[S]]],
mdp_to_sample: FiniteMarkovDecisionProcess,
states: List[S],
actions: Mapping[S, List[A]],
γ: float,
tolerance: float = 1e-6,
num_episodes: float = 10000) -> Mapping[Tuple[S, A], float]:
q: Mapping[Tuple[S, A], float] = {}
counts_per_state_act: Mapping[Tuple[S, A], int] = {}
for state in states:
for action in actions[state]:
q[(state, action)] = 0.
counts_per_state_act[(state, action)] = 0
policy_map: Mapping[S, Optional[Categorical[A]]] = {}
for state in states:
if actions[state] is None:
policy_map[state] = None
else:
policy_map[state] = Categorical(
{action: 1
for action in actions[state]})
Pi: FinitePolicy[S, A] = FinitePolicy(policy_map)
start_state_distrib = Categorical({state: 1 for state in states})
for i in range(num_episodes):
trace: Iterable[TransitionStep[S, A]] = mdp_to_sample.simulate_actions(
start_state_distrib, Pi)
episode = returns(trace, γ, tolerance)
#print(episode)
for step in episode:
state = step.state
action = step.action
return_ = step.return_
counts_per_state_act[(state, action)] += 1
q[(state, action)] += 1 / counts_per_state_act[
(state, action)] * (return_ - q[(state, action)])
eps = 1 / (i + 1)
new_pol: Mapping[S, Optional[Categorical[A]]] = {}
for state in states:
if actions[state] is None:
new_pol[state] = None
policy_map = {
action: eps / len(actions[state])
for action in actions[state]
}
best_action = actions[state][0]
for action in actions[state]:
if q[(state, best_action)] <= q[(state, action)]:
best_action = action
policy_map[best_action] += 1 - eps
new_pol[state] = Categorical(policy_map)
Pi = FinitePolicy(new_pol)
return q
class TestEvaluate(unittest.TestCase):
def setUp(self):
random.seed(42)
self.finite_flip_flop = FlipFlop(0.7)
self.finite_mdp = FiniteMarkovDecisionProcess({
True: {
True: Categorical({
(True, 1.0): 0.7,
(False, 2.0): 0.3
}),
False: Categorical({
(True, 1.0): 0.3,
(False, 2.0): 0.7
}),
},
False: {
True: Categorical({
(False, 1.0): 0.7,
(True, 2.0): 0.3
}),
False: Categorical({
(False, 1.0): 0.3,
(True, 2.0): 0.7
}),
},
})
def test_evaluate_finite_mrp(self) -> None:
start = Tabular(
{s: 0.0
for s in self.finite_flip_flop.states()},
count_to_weight_func=lambda _: 0.1,
)
episode_length = 20
episodes: Iterable[Iterable[
mp.TransitionStep[bool]]] = self.finite_flip_flop.reward_traces(
Choose({True, False}))
transitions: Iterable[
mp.TransitionStep[bool]] = itertools.chain.from_iterable(
itertools.islice(episode, episode_length)
for episode in episodes)
vs = td.td_prediction(transitions, γ=0.99, approx_0=start)
v: Optional[Tabular[bool]] = iterate.last(
itertools.islice(cast(Iterator[Tabular[bool]], vs), 10000))
if v is not None:
self.assertEqual(len(v.values_map), 2)
for s in v.values_map:
# Intentionally loose bound—otherwise test is too slow.
# Takes >1s on my machine otherwise.
self.assertLess(abs(v(s) - 170), 3.0)
else:
assert False
def test_evaluate_finite_mdp(self) -> None:
q_0: Tabular[Tuple[bool, bool]] = Tabular(
{(s, a): 0.0
for s in self.finite_mdp.states()
for a in self.finite_mdp.actions(s)},
count_to_weight_func=lambda _: 0.1,
)
uniform_policy: mdp.Policy[bool, bool] = mdp.FinitePolicy({
s: Choose(self.finite_mdp.actions(s))
for s in self.finite_mdp.states()
})
transitions: Iterable[mdp.TransitionStep[
bool, bool]] = self.finite_mdp.simulate_actions(
Choose(self.finite_mdp.states()), uniform_policy)
qs = td.td_control(transitions, self.finite_mdp.actions, q_0, γ=0.99)
q: Optional[Tabular[Tuple[bool, bool]]] = iterate.last(
cast(Iterator[Tabular[Tuple[bool, bool]]],
itertools.islice(qs, 20000)))
if q is not None:
self.assertEqual(len(q.values_map), 4)
for s in [True, False]:
self.assertLess(abs(q((s, False)) - 170.0), 2)
self.assertGreater(q((s, False)), q((s, True)))
else:
assert False