def test():
mdp = MDP.generate(n_states=4, n_actions=2)
pi_list = mdp.all_policies()
v_list = [vi(mdp, pi)[0] for pi in pi_list]
v_ranks = sorted_order(v_list)
sorted_v = [v for _, v in sorted(zip(v_ranks, v_list))]
for v1, v2 in zip(sorted_v[:-1], sorted_v[1:]):
assert compare_value_fns(v1, v2) != '<'
# for pi1, v1 in zip(pi_list, v_list):
# for pi2, v2 in zip(pi_list, v_list):
# print(v1.round(4))
# print(compare_value_fns(v1, v2), v2.round(4))
# print()
v_star, _, pi_star = vi(mdp)
assert compare_value_fns(v_star, sorted_v[0]) == '='
def main():
mdp = BlockMDP(MDP.generate(n_states=5, n_actions=6), n_obs_per_block=3)
v, q, pi = vi(mdp)
v_alt = np.zeros_like(v)
for s in range(mdp.n_states):
v_alt[s] = q[pi[s]][s]
v_alt = v_alt.squeeze()
assert np.allclose(v_alt, v)
v_pi = vi(mdp, pi)[0]
assert np.allclose(v_pi, v)
m_phi = mdp.base_mdp
v_phi, q_phi, pi_phi = vi(m_phi)
pi_phi_grounded = np.kron(pi_phi,
np.ones((1, mdp.n_states // m_phi.n_states)))
assert np.allclose(pi_phi_grounded, pi)
print('All tests passed.')
R = np.array([[0, 0.5, 0, 0], [0, 0, 0, 1], [0, 0, 0, 2], [0, 0, 0, 0]])
phi = np.array([
[1, 0, 0],
[0, 1, 0],
[0, 1, 0],
[0, 0, 1],
])
mdp1 = MDP(T_list, [R, R], gamma=0.9)
mdp2 = AbstractMDP(mdp1, phi, p0=np.array([1, 0, 0, 0]), t=1)
mdp2 = AbstractMDP(mdp1, phi)
is_markov(mdp2)
pi_g_list = mdp2.piecewise_constant_policies()
pi_a_list = mdp2.abstract_policies()
v_g_list = [vi(mdp1, pi)[0] for pi in pi_g_list]
v_a_list = [vi(mdp2, pi)[0] for pi in pi_a_list]
order_v_g = np.stack(sort_value_fns(v_g_list)).round(4)
order_v_a = np.stack(sort_value_fns(v_a_list)).round(4)
mdp2.p0
agg_state = mdp2.phi.sum(axis=0) > 1
np.stack([mdp2.B(pi, t=1)[agg_state] for pi in pi_g_list])
v_phi_pi_phi_star, _, pi_phi_star = vi(mdp2)
v_pi_phi_star = vi(mdp1, mdp2.get_ground_policy(pi_phi_star))[0]
# Look for examples of v_pi_phi_star < v
for v in v_g_list:
if compare_value_fns(v_pi_phi_star, v) == "<":
[3, 0, 0, 0, 0, 0],
[4, 0, 0, 0, 0, 0]
])/4
mdp1 = MDP([T, T], [R, R], gamma=0.9)
phi = np.array([
[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
[0, 0, 0, 1],
[0, 0, 0, 1],
])
mdp2 = AbstractMDP(mdp1, phi)
v_star, q_star, pi_star = vi(mdp1)
v_star, pi_star
pi_g_list = mdp2.piecewise_constant_policies()
pi_a_list = mdp2.abstract_policies()
v_g_list = [vi(mdp1, pi)[0] for pi in pi_g_list]
v_a_list = [vi(mdp2, pi)[0] for pi in pi_a_list]
np.allclose(v_g_list, v_g_list[0])
order_v_g = sorted_order(v_g_list)
order_v_a = sorted_order(v_a_list)
assert np.allclose(order_v_a, order_v_g)
graph_value_fns(v_a_list)
graph_value_fns(v_g_list)
[0, 3/4, 1/4],
[2/3, 1/4, 1/12],
[0, 3/4, 1/4],
])
# T_alt = np.array([
# [1/2, 3/8, 1/8],
# [1, 0, 0],
# [1, 0, 0],
# ])
R = np.array([
[0, 1, 1],
[1, 0, 0],
[1, 0, 0],
])
mdp1 = MDP([T1, T2], [R, R], gamma=0.9)
v_star, q_star, pi_star = vi(mdp1)
v_star, pi_star
phi = np.array([
[1, 0],
[0, 1],
[0, 1],
])
mdp2 = AbstractMDP(mdp1, phi)
v_phi_star, q_phi_star, pi_phi_star = vi(mdp2)
v_phi_star
# for each ground-state policy
n_policies = mdp1.n_actions**mdp1.n_states
for i in range(n_policies):
[2/3, 1/4, 1/12],
[0, 3/4, 1/4],
])
# T_alt = np.array([
# [1/2, 3/8, 1/8],
# [1, 0, 0],
# [1, 0, 0],
# ])
R = np.array([
[0, 1, 1],
[1, 0, 0],
[1, 0, 0],
])
mdp1 = MDP([T1, T2], [R, R], gamma=0.9)
mdp2 = AbstractMDP(MDP([T0, T1], [R, R], gamma=0.9), np.array([[1,0],[0,1],[0,1]]))
is_hutter_markov(mdp2)
is_markov(mdp2)
v_star, q_star, pi_star = vi(mdp1)
v_star, pi_star
phi = np.array([
[1, 0],
[0, 1],
[0, 1],
])
mdp2 = AbstractMDP(mdp1, phi)
assert is_markov(mdp2)
assert has_block_dynamics(mdp2)
assert not is_hutter_markov(mdp2)
# pi_abs = pi_a_list[i]
# mdp2.B(pi_gnd)
# phi = mdp2.phi
# N_gnd = mdp1.get_N(pi_gnd)
# phi.transpose() @ phi
# Px = mdp1.stationary_distribution(pi_gnd)
# N_abs = mdp2.get_N(pi_abs)
# Pz = mdp2.stationary_distribution(pi_abs)
#
# ratio_abs = np.divide(N_abs, Pz[None,:], out=np.zeros_like(N_abs), where=Pz!=0)
# ratio_gnd = np.divide(N_gnd, Px[None,:], out=np.zeros_like(N_gnd), where=Px!=0)
# mdp2.B(pi_gnd) @ ratio_gnd
# ratio_abs @ phi.transpose()
# is_markov(mdp2)
v_g_list = [vi(mdp1, pi)[0] for pi in pi_g_list]
v_a_list = [vi(mdp2, pi)[0] for pi in pi_a_list]
order_v_g = sort_value_fns(v_g_list)
order_v_a = sort_value_fns(v_a_list)
agg_state = mdp2.phi.sum(axis=0) > 1
[mdp2.B(pi, t=0)[agg_state][0] for pi in pi_g_list]
[mdp2.B(pi, t=1)[agg_state][0] for pi in pi_g_list]
[mdp2.B(pi, t=3)[agg_state][0] for pi in pi_g_list]
v_star, _, pi_star = vi(mdp1)
v_phi_pi_phi_star, _, pi_phi_star = vi(mdp2)
pi_phi_star_gnd = mdp2.get_ground_policy(pi_phi_star)
v_pi_phi_star = vi(mdp1, pi_phi_star_gnd)[0]
