def main(open_plot=True):
# Make MDP distribution, agents.
mdp_distr = make_mdp.make_mdp_distr(mdp_class="four_room")
ql_agent = QLearnerAgent(actions=mdp_distr.get_actions())
rand_agent = RandomAgent(actions=mdp_distr.get_actions())
# Run experiment and make plot.
run_agents_multi_task([ql_agent, rand_agent], mdp_distr, task_samples=50, episodes=1, steps=1500, reset_at_terminal=True, open_plot=open_plot)
def main():
import OptimalBeliefAgentClass
# Setup multitask setting.
# R ~ D : Puddle, Rock Sample
# G ~ D : octo, four_room
# T ~ D : grid
mdp_class, is_goal_terminal, samples = parse_args()
mdp_distr = make_mdp_distr(mdp_class=mdp_class,
is_goal_terminal=is_goal_terminal)
mdp_distr.set_gamma(0.99)
actions = mdp_distr.get_actions()
# Compute average MDP.
print "Making and solving avg MDP...",
sys.stdout.flush()
avg_mdp = compute_avg_mdp(mdp_distr)
avg_mdp_vi = ValueIteration(avg_mdp,
delta=0.001,
max_iterations=1000,
sample_rate=5)
iters, value = avg_mdp_vi.run_vi()
print "done." #, iters, value
sys.stdout.flush()
# Agents.
print "Making agents...",
sys.stdout.flush()
mdp_distr_copy = copy.deepcopy(mdp_distr)
opt_stoch_policy = compute_optimal_stoch_policy(mdp_distr_copy)
opt_stoch_policy_agent = FixedPolicyAgent(opt_stoch_policy,
name="$\pi_{prior}$")
opt_belief_agent = OptimalBeliefAgentClass.OptimalBeliefAgent(
mdp_distr, actions)
vi_agent = FixedPolicyAgent(avg_mdp_vi.policy, name="$\pi_{avg}$")
rand_agent = RandomAgent(actions, name="$\pi^u$")
ql_agent = QLearningAgent(actions)
print "done."
agents = [vi_agent, opt_stoch_policy_agent, rand_agent, opt_belief_agent]
# Run task.
run_agents_multi_task(agents,
mdp_distr,
task_samples=samples,
episodes=1,
steps=100,
reset_at_terminal=False,
track_disc_reward=False,
cumulative_plot=True)
def main():
# Make MDP distribution, agents.
mdp_distr = make_mdp_distr(mdp_class="grid")
ql_agent = QLearnerAgent(actions=mdp_distr.get_actions())
rand_agent = RandomAgent(actions=mdp_distr.get_actions())
# Run experiment and make plot.
run_agents_multi_task([ql_agent, rand_agent],
mdp_distr,
task_samples=30,
episodes=100,
steps=50,
reset_at_terminal=True,
include_optimal=True)
def main():
# ========================
# === Make Environment ===
# ========================
mdp_class = "four_room"
gamma = 1.0
environment = make_mdp.make_mdp_distr(mdp_class=mdp_class, step_cost=0.01, grid_dim=15, gamma=gamma)
actions = environment.get_actions()
# ==========================
# === Make SA, AA Stacks ===
# ==========================
sa_stack, aa_stack = hierarchy_helpers.make_hierarchy(environment, num_levels=2)
# Debug.
print "\n" + ("=" * 30) + "\n== Done making abstraction. ==\n" + ("=" * 30) + "\n"
sa_stack.print_state_space_sizes()
aa_stack.print_action_spaces_sizes()
# ===================
# === Make Agents ===
# ===================
# baseline_agent = QLearnerAgent(actions)
agent_class = QLearnerAgent
baseline_agent = agent_class(actions, gamma=gamma)
rand_agent = RandomAgent(actions)
# hierarch_r_max = HRMaxAgent(actions, sa_stack=sa_stack, aa_stack=aa_stack)
l0_hierarch_agent = HierarchyAgent(agent_class, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=0, name_ext="-$l_0$")
l1_hierarch_agent = HierarchyAgent(agent_class, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$l_1$")
# l2_hierarch_agent = HierarchyAgent(agent_class, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=2, name_ext="-$l_2$")
dynamic_hierarch_agent = DynamicHierarchyAgent(agent_class, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$d$")
# dynamic_rmax_hierarch_agent = DynamicHierarchyAgent(RMaxAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$d$")
print "\n" + ("=" * 26)
print "== Running experiments. =="
print "=" * 26 + "\n"
# ======================
# === Run Experiment ===
# ======================
agents = [dynamic_hierarch_agent, baseline_agent]
run_agents_multi_task(agents, environment, task_samples=10, steps=20000, episodes=1, reset_at_terminal=True)
def main():
# ========================
# === Make Environment ===
# ========================
mdp_class = "hrooms"
environment = make_mdp.make_mdp_distr(mdp_class=mdp_class)
actions = environment.get_actions()
# ==========================
# === Make SA, AA Stacks ===
# ==========================
# sa_stack, aa_stack = aa_stack_h.make_random_sa_diropt_aa_stack(environment, max_num_levels=3)
sa_stack, aa_stack = hierarchy_helpers.make_hierarchy(environment, num_levels=3)
# Debug.
print("\n" + ("=" * 30))
print("== Done making abstraction. ==")
print("=" * 30 + "\n")
sa_stack.print_state_space_sizes()
print("Num Action Abstractions:", len(aa_stack.get_aa_list()))
# ===================
# === Make Agents ===
# ===================
baseline_agent = QLearningAgent(actions)
rmax_agent = RMaxAgent(actions)
rand_agent = RandomAgent(actions)
l0_hierarch_agent = HierarchyAgent(QLearningAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=0, name_ext="-$l_0$")
l1_hierarch_agent = HierarchyAgent(QLearningAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$l_1$")
# l2_hierarch_agent = HierarchyAgent(QLearningAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=2, name_ext="-$l_2$")
dynamic_hierarch_agent = DynamicHierarchyAgent(QLearningAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$d$")
# dynamic_rmax_hierarch_agent = DynamicHierarchyAgent(RMaxAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$d$")
print("\n" + ("=" * 26))
print("== Running experiments. ==")
print("=" * 26 + "\n")
# ======================
# === Run Experiment ===
# ======================
agents = [l1_hierarch_agent, dynamic_hierarch_agent, baseline_agent]
run_agents_multi_task(agents, environment, task_samples=10, steps=1500, episodes=1, reset_at_terminal=True)
def main(open_plot=True):
# Setup MDP, Agents.
mdp_distr = make_mdp.make_mdp_distr(mdp_class="four_room")
ql_agent = QLearnerAgent(actions=mdp_distr.get_actions())
rand_agent = RandomAgent(actions=mdp_distr.get_actions())
# Make goal-based option agent.
goal_based_options = aa_helpers.make_goal_based_options(mdp_distr)
goal_based_aa = ActionAbstraction(prim_actions=mdp_distr.get_actions(),
options=goal_based_options)
option_agent = AbstractionWrapper(QLearnerAgent,
actions=mdp_distr.get_actions(),
action_abstr=goal_based_aa)
# Run experiment and make plot.
run_agents_multi_task([ql_agent, rand_agent, option_agent],
mdp_distr,
task_samples=10,
episodes=100,
steps=150,
open_plot=open_plot)
def main(eps=0.1, open_plot=True):
mdp_class, is_goal_terminal, samples, alg = parse_args()
# Setup multitask setting.
mdp_distr = make_mdp.make_mdp_distr(mdp_class=mdp_class)
actions = mdp_distr.get_actions()
# Compute average MDP.
print "Making and solving avg MDP...",
sys.stdout.flush()
avg_mdp = compute_avg_mdp(mdp_distr)
avg_mdp_vi = ValueIteration(avg_mdp,
delta=0.001,
max_iterations=1000,
sample_rate=5)
iters, value = avg_mdp_vi.run_vi()
### Yuu
transfer_fixed_agent = FixedPolicyAgent(avg_mdp_vi.policy,
name="transferFixed")
rand_agent = RandomAgent(actions, name="$\pi^u$")
opt_q_func = compute_optimistic_q_function(mdp_distr)
avg_q_func = avg_mdp_vi.get_q_function()
if alg == "q":
pure_ql_agent = QLearnerAgent(actions, epsilon=eps, name="Q-0")
qmax = 1.0 * (1 - 0.99)
# qmax = 1.0
pure_ql_agent_opt = QLearnerAgent(actions,
epsilon=eps,
default_q=qmax,
name="Q-vmax")
transfer_ql_agent_optq = QLearnerAgent(actions,
epsilon=eps,
name="Q-trans-max")
transfer_ql_agent_optq.set_init_q_function(opt_q_func)
transfer_ql_agent_avgq = QLearnerAgent(actions,
epsilon=eps,
name="Q-trans-avg")
transfer_ql_agent_avgq.set_init_q_function(avg_q_func)
agents = [
pure_ql_agent, pure_ql_agent_opt, transfer_ql_agent_optq,
transfer_ql_agent_avgq
]
elif alg == "rmax":
pure_rmax_agent = RMaxAgent(actions, name="RMAX-vmax")
updating_trans_rmax_agent = UpdatingRMaxAgent(actions,
name="RMAX-updating_max")
trans_rmax_agent = RMaxAgent(actions, name="RMAX-trans_max")
trans_rmax_agent.set_init_q_function(opt_q_func)
agents = [pure_rmax_agent, updating_trans_rmax_agent, trans_rmax_agent]
elif alg == "delayed-q":
pure_delayed_ql_agent = DelayedQLearnerAgent(actions,
opt_q_func,
name="DelayedQ-vmax")
pure_delayed_ql_agent.set_vmax()
updating_delayed_ql_agent = UpdatingDelayedQLearnerAgent(
actions, name="DelayedQ-updating_max")
trans_delayed_ql_agent = DelayedQLearnerAgent(
actions, opt_q_func, name="DelayedQ-trans-max")
agents = [
pure_delayed_ql_agent, updating_delayed_ql_agent,
trans_delayed_ql_agent
]
else:
print "Unknown type of agents:", alg
print "(q, rmax, delayed-q)"
assert (False)
# Run task.
# TODO: Function for Learning on each MDP
run_agents_multi_task(agents,
mdp_distr,
task_samples=samples,
episodes=1,
steps=100,
reset_at_terminal=is_goal_terminal,
is_rec_disc_reward=False,
cumulative_plot=True,
open_plot=open_plot)
def main():
# Setup environment.
mdp_class, agent_type, samples = parse_args()
is_goal_terminal = False
mdp_distr = make_mdp_distr(mdp_class=mdp_class,
is_goal_terminal=is_goal_terminal)
mdp_distr.set_gamma(0.99)
actions = mdp_distr.get_actions()
# Compute priors.
# Stochastic mixture.
mdp_distr_copy = copy.deepcopy(mdp_distr)
opt_stoch_policy = ape.compute_optimal_stoch_policy(mdp_distr_copy)
# Avg MDP
avg_mdp = ape.compute_avg_mdp(mdp_distr)
avg_mdp_vi = ValueIteration(avg_mdp,
delta=0.001,
max_iterations=1000,
sample_rate=5)
iters, value = avg_mdp_vi.run_vi()
# Make agents.
# Q Learning
ql_agent = QLearnerAgent(actions)
shaped_ql_agent_prior = ShapedQAgent(shaping_policy=opt_stoch_policy,
actions=actions,
name="Prior-QLearning")
shaped_ql_agent_avgmdp = ShapedQAgent(shaping_policy=avg_mdp_vi.policy,
actions=actions,
name="AvgMDP-QLearning")
# RMax
rmax_agent = RMaxAgent(actions)
shaped_rmax_agent_prior = ShapedRMaxAgent(
shaping_policy=opt_stoch_policy,
state_space=avg_mdp_vi.get_states(),
actions=actions,
name="Prior-RMax")
shaped_rmax_agent_avgmdp = ShapedRMaxAgent(
shaping_policy=avg_mdp_vi.policy,
state_space=avg_mdp_vi.get_states(),
actions=actions,
name="AvgMDP-RMax")
prune_rmax_agent = PruneRMaxAgent(mdp_distr=mdp_distr)
if agent_type == "rmax":
agents = [
rmax_agent, shaped_rmax_agent_prior, shaped_rmax_agent_avgmdp,
prune_rmax_agent
]
else:
agents = [ql_agent, shaped_ql_agent_prior, shaped_ql_agent_avgmdp]
# Run task.
run_agents_multi_task(agents,
mdp_distr,
task_samples=samples,
episodes=1,
steps=200,
is_rec_disc_reward=False,
verbose=True)