(actions_377_train, actions_377_test),
axis = 0)
rewards_377 = np.concatenate(
(rewards_377_train, rewards_377_test),
axis = 0)
trajectories_377 = trajectories_377_train + trajectories_377_test
print '-------------------Evaluation for Q377--------------------------------'
# evaluation for q377
state_rewards_377 = ql.estimate_rewards(next_states_377_train, actions_377_train, rewards_377_train, action_q377)
discounted_rewards_377 = ql.discount_rewards(state_rewards_377, discount)
discounted_max_states_377 = ql.get_max_reward_states(discounted_rewards_377)
max_states_377 = ql.get_max_reward_states(state_rewards_377)
q377_policy = QLearnedPolicy(discounted_max_states_377[0], q377_labels)
print 'Reward of max state = {a}, discounted max state = {b}'.format(
a = state_rewards_377[max_states_377[0]], b = state_rewards_377[discounted_max_states_377[0]])
print 'Discounted reward of max state = {a}, discounted max state = {b}'.format(
a = discounted_rewards_377[max_states_377[0]], b = discounted_rewards_377[discounted_max_states_377[0]])
print 'Max state actions: {a} \nDiscounted max state actions: {b}'.format(
a = q377_labels[np.array(max_states_377[0]).astype(int) == 1], b = q377_labels[np.array(discounted_max_states_377[0]).astype(int) == 1])
action_counts_377 = defaultdict(lambda: defaultdict(int))
for s,a in zip(states_377,actions_377):
action_counts_377[tuple(s)][a] += 1
sample_policy_377 = SamplePolicy(action_counts_377)
expected_reward_377 = estimate_utility(sample_policy_377, q377_policy, trajectories_377_test, discount)
lower_bound_377 = hcope(sample_policy_377, q377_policy, trajectories_377_test, discount, delta)
print 'Expected Reward = {r}, Lower bound = {l}'.format(r = expected_reward_377, l = lower_bound_377)
print 'Expected Reward sample = {r}'.format(r = estimate_utility(sample_policy_377, sample_policy_377, trajectories_377_test, discount))
(actions_315_train, actions_315_test),
axis = 0)
rewards_315 = np.concatenate(
(rewards_315_train, rewards_315_test),
axis = 0)
trajectories_315 = trajectories_315_train + trajectories_315_test
# evaluation for q315
print '-------------------Evaluation for Q315--------------------------------'
state_rewards_315 = ql.estimate_rewards(next_states_315_train, actions_315_train, rewards_315_train, action_q315)
discounted_rewards_315 = ql.discount_rewards(state_rewards_315, discount)
discounted_max_states_315 = ql.get_max_reward_states(discounted_rewards_315)
max_states_315 = ql.get_max_reward_states(state_rewards_315)
q315_policy = QLearnedPolicy(discounted_max_states_315[0], q315_labels)
print 'Reward of max state = {a}, discounted max state = {b}'.format(
a = state_rewards_315[max_states_315[0]], b = state_rewards_315[discounted_max_states_315[0]])
print 'Discounted reward of max state = {a}, discounted max state = {b}'.format(
a = discounted_rewards_315[max_states_315[0]], b = discounted_rewards_315[discounted_max_states_315[0]])
print 'Max state actions: {a} \nDiscounted max state actions: {b}'.format(
a = q315_labels[np.array(max_states_315[0]).astype(int) == 1], b = q315_labels[np.array(discounted_max_states_315[0]).astype(int) == 1])
action_counts_315 = defaultdict(lambda: defaultdict(int))
for s,a in zip(states_315,actions_315):
action_counts_315[tuple(s)][a] += 1
sample_policy_315 = SamplePolicy(action_counts_315)
expected_reward_315 = estimate_utility(sample_policy_315, q315_policy, trajectories_315_test, discount)
lower_bound_315 = hcope(sample_policy_315, q315_policy, trajectories_315_test, discount, delta)
print 'Expected Reward = {r}, Lower bound = {l}'.format(r = expected_reward_315, l = lower_bound_315)
print 'Expected Reward sample = {r}'.format(r = estimate_utility(sample_policy_315, sample_policy_315, trajectories_315_test, discount))
