def main(): # Make MDP Distribution. mdp_class = "four_room" environment = make_mdp.make_mdp_distr(mdp_class=mdp_class, grid_dim=10) sa_stack = make_random_sa_stack(environment, max_num_levels=5) sa_stack.print_state_space_sizes()
def main(): # ======================== # === Make Environment === # ======================== mdp_class = "four_room" environment = make_mdp.make_mdp_distr(mdp_class=mdp_class, grid_dim=10) actions = environment.get_actions() # ========================== # === Make SA, AA Stacks === # ========================== # sa_stack, aa_stack = aa_stack_h.make_random_sa_diropt_aa_stack(environment, max_num_levels=3) sa_stack, aa_stack = hierarchy_helpers.make_hierarchy(environment, num_levels=3) mdp = environment.sample() HVI = HierarchicalValueIteration(mdp, sa_stack, aa_stack) VI = ValueIteration(mdp) h_iters, h_val = HVI.run_vi() iters, val = VI.run_vi() print "H:", h_iters, h_val print "V:", iters, val
def main(open_plot=True): # Make MDP distribution, agents. mdp_distr = make_mdp.make_mdp_distr(mdp_class="four_room") ql_agent = QLearnerAgent(actions=mdp_distr.get_actions()) rand_agent = RandomAgent(actions=mdp_distr.get_actions()) # Run experiment and make plot. run_agents_multi_task([ql_agent, rand_agent], mdp_distr, task_samples=50, episodes=1, steps=1500, reset_at_terminal=True, open_plot=open_plot)
def main(): # ====================== # == Make Environment == # ====================== mdp_class = "four_room" environment = make_mdp.make_mdp_distr(mdp_class=mdp_class, grid_dim=7) actions = environment.get_actions() # ==================== # == Make Hierarchy == # ==================== sa_stack, aa_stack = make_hierarchy(environment, num_levels=3)
def main(open_plot=True): # Setup MDP, Agents. mdp_distr = make_mdp.make_mdp_distr(mdp_class="four_room") ql_agent = QLearningAgent(actions=mdp_distr.get_actions()) rand_agent = RandomAgent(actions=mdp_distr.get_actions()) # Make goal-based option agent. goal_based_options = aa_helpers.make_goal_based_options(mdp_distr) goal_based_aa = ActionAbstraction(prim_actions=mdp_distr.get_actions(), options=goal_based_options) option_agent = AbstractionWrapper(QLearningAgent, agent_params={"actions":mdp_distr.get_actions()}, action_abstr=goal_based_aa) # Run experiment and make plot. run_agents_lifelong([ql_agent, rand_agent, option_agent], mdp_distr, samples=10, episodes=100, steps=150, open_plot=open_plot)
def main(): # ======================== # === Make Environment === # ======================== mdp_class = "four_room" gamma = 1.0 environment = make_mdp.make_mdp_distr(mdp_class=mdp_class, step_cost=0.01, grid_dim=15, gamma=gamma) actions = environment.get_actions() # ========================== # === Make SA, AA Stacks === # ========================== sa_stack, aa_stack = hierarchy_helpers.make_hierarchy(environment, num_levels=2) # Debug. print "\n" + ("=" * 30) + "\n== Done making abstraction. ==\n" + ("=" * 30) + "\n" sa_stack.print_state_space_sizes() aa_stack.print_action_spaces_sizes() # =================== # === Make Agents === # =================== # baseline_agent = QLearnerAgent(actions) agent_class = QLearnerAgent baseline_agent = agent_class(actions, gamma=gamma) rand_agent = RandomAgent(actions) # hierarch_r_max = HRMaxAgent(actions, sa_stack=sa_stack, aa_stack=aa_stack) l0_hierarch_agent = HierarchyAgent(agent_class, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=0, name_ext="-$l_0$") l1_hierarch_agent = HierarchyAgent(agent_class, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$l_1$") # l2_hierarch_agent = HierarchyAgent(agent_class, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=2, name_ext="-$l_2$") dynamic_hierarch_agent = DynamicHierarchyAgent(agent_class, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$d$") # dynamic_rmax_hierarch_agent = DynamicHierarchyAgent(RMaxAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$d$") print "\n" + ("=" * 26) print "== Running experiments. ==" print "=" * 26 + "\n" # ====================== # === Run Experiment === # ====================== agents = [dynamic_hierarch_agent, baseline_agent] run_agents_multi_task(agents, environment, task_samples=10, steps=20000, episodes=1, reset_at_terminal=True)
def main(): # ======================== # === Make Environment === # ======================== mdp_class = "hrooms" environment = make_mdp.make_mdp_distr(mdp_class=mdp_class) actions = environment.get_actions() # ========================== # === Make SA, AA Stacks === # ========================== # sa_stack, aa_stack = aa_stack_h.make_random_sa_diropt_aa_stack(environment, max_num_levels=3) sa_stack, aa_stack = hierarchy_helpers.make_hierarchy(environment, num_levels=3) # Debug. print("\n" + ("=" * 30)) print("== Done making abstraction. ==") print("=" * 30 + "\n") sa_stack.print_state_space_sizes() print("Num Action Abstractions:", len(aa_stack.get_aa_list())) # =================== # === Make Agents === # =================== baseline_agent = QLearningAgent(actions) rmax_agent = RMaxAgent(actions) rand_agent = RandomAgent(actions) l0_hierarch_agent = HierarchyAgent(QLearningAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=0, name_ext="-$l_0$") l1_hierarch_agent = HierarchyAgent(QLearningAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$l_1$") # l2_hierarch_agent = HierarchyAgent(QLearningAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=2, name_ext="-$l_2$") dynamic_hierarch_agent = DynamicHierarchyAgent(QLearningAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$d$") # dynamic_rmax_hierarch_agent = DynamicHierarchyAgent(RMaxAgent, sa_stack=sa_stack, aa_stack=aa_stack, cur_level=1, name_ext="-$d$") print("\n" + ("=" * 26)) print("== Running experiments. ==") print("=" * 26 + "\n") # ====================== # === Run Experiment === # ====================== agents = [l1_hierarch_agent, dynamic_hierarch_agent, baseline_agent] run_agents_multi_task(agents, environment, task_samples=10, steps=1500, episodes=1, reset_at_terminal=True)
def main(): # MDP Setting. lifelong = True mdp_class = "four_room" grid_dim = 11 # Make MDP. mdp_distr = make_mdp.make_mdp_distr(mdp_class=mdp_class, grid_dim=grid_dim) actions = mdp_distr.get_actions() experiment_type = "aa" goal_based_options = aa_helpers.make_goal_based_options(mdp_distr) goal_based_aa = ActionAbstraction(prim_actions=actions, options=goal_based_options) # Visualize Action Abstractions. visualize_options_grid(mdp_distr, goal_based_aa) input("Press any key to quit ") quit()
def main(open_plot=True): # Setup MDP, Agents. mdp_distr = make_mdp.make_mdp_distr(mdp_class="four_room") ql_agent = QLearningAgent(actions=mdp_distr.get_actions()) rand_agent = RandomAgent(actions=mdp_distr.get_actions()) # Make goal-based option agent. goal_based_options = aa_helpers.make_goal_based_options(mdp_distr) goal_based_aa = ActionAbstraction(prim_actions=mdp_distr.get_actions(), options=goal_based_options) option_agent = AbstractionWrapper(QLearningAgent, actions=mdp_distr.get_actions(), action_abstr=goal_based_aa) # Run experiment and make plot. run_agents_lifelong([ql_agent, rand_agent, option_agent], mdp_distr, samples=10, episodes=100, steps=150, open_plot=open_plot)
def main(): # MDP Setting. lifelong = True mdp_class = "four_room" grid_dim = 11 # Make MDP. mdp_distr = make_mdp.make_mdp_distr(mdp_class=mdp_class, grid_dim=grid_dim) actions = mdp_distr.get_actions() experiment_type = "aa" goal_based_options = aa_helpers.make_goal_based_options(mdp_distr) goal_based_aa = ActionAbstraction(prim_actions=actions, options=goal_based_options) # Visualize Action Abstractions. visualize_options_grid(mdp_distr, goal_based_aa) input("Press any key to quit ") quit()
def main(eps=0.1, open_plot=True): mdp_class, is_goal_terminal, samples, alg = parse_args() # Setup multitask setting. mdp_distr = make_mdp.make_mdp_distr(mdp_class=mdp_class) actions = mdp_distr.get_actions() # Compute average MDP. print "Making and solving avg MDP...", sys.stdout.flush() avg_mdp = compute_avg_mdp(mdp_distr) avg_mdp_vi = ValueIteration(avg_mdp, delta=0.001, max_iterations=1000, sample_rate=5) iters, value = avg_mdp_vi.run_vi() ### Yuu transfer_fixed_agent = FixedPolicyAgent(avg_mdp_vi.policy, name="transferFixed") rand_agent = RandomAgent(actions, name="$\pi^u$") opt_q_func = compute_optimistic_q_function(mdp_distr) avg_q_func = avg_mdp_vi.get_q_function() if alg == "q": pure_ql_agent = QLearnerAgent(actions, epsilon=eps, name="Q-0") qmax = 1.0 * (1 - 0.99) # qmax = 1.0 pure_ql_agent_opt = QLearnerAgent(actions, epsilon=eps, default_q=qmax, name="Q-vmax") transfer_ql_agent_optq = QLearnerAgent(actions, epsilon=eps, name="Q-trans-max") transfer_ql_agent_optq.set_init_q_function(opt_q_func) transfer_ql_agent_avgq = QLearnerAgent(actions, epsilon=eps, name="Q-trans-avg") transfer_ql_agent_avgq.set_init_q_function(avg_q_func) agents = [ pure_ql_agent, pure_ql_agent_opt, transfer_ql_agent_optq, transfer_ql_agent_avgq ] elif alg == "rmax": pure_rmax_agent = RMaxAgent(actions, name="RMAX-vmax") updating_trans_rmax_agent = UpdatingRMaxAgent(actions, name="RMAX-updating_max") trans_rmax_agent = RMaxAgent(actions, name="RMAX-trans_max") trans_rmax_agent.set_init_q_function(opt_q_func) agents = [pure_rmax_agent, updating_trans_rmax_agent, trans_rmax_agent] elif alg == "delayed-q": pure_delayed_ql_agent = DelayedQLearnerAgent(actions, opt_q_func, name="DelayedQ-vmax") pure_delayed_ql_agent.set_vmax() updating_delayed_ql_agent = UpdatingDelayedQLearnerAgent( actions, name="DelayedQ-updating_max") trans_delayed_ql_agent = DelayedQLearnerAgent( actions, opt_q_func, name="DelayedQ-trans-max") agents = [ pure_delayed_ql_agent, updating_delayed_ql_agent, trans_delayed_ql_agent ] else: print "Unknown type of agents:", alg print "(q, rmax, delayed-q)" assert (False) # Run task. # TODO: Function for Learning on each MDP run_agents_multi_task(agents, mdp_distr, task_samples=samples, episodes=1, steps=100, reset_at_terminal=is_goal_terminal, is_rec_disc_reward=False, cumulative_plot=True, open_plot=open_plot)
def main(): # Grab experiment params. mdp_class, task_samples, episodes, steps, grid_dim, x_axis_num_options, agent_class_str, max_options, exp_type = parse_args( ) gamma = 0.9 # ======================== # === Make Environment === # ======================== multi_task = True max_option_steps = 50 if x_axis_num_options else 0 environment = make_mdp.make_mdp_distr( mdp_class=mdp_class, grid_dim=grid_dim) if multi_task else make_mdp.make_mdp( mdp_class=mdp_class) actions = environment.get_actions() environment.set_gamma(gamma) # Indicator functions. v_indic = ind_funcs._v_approx_indicator q_indic = ind_funcs._q_eps_approx_indicator v_disc_indic = ind_funcs._v_disc_approx_indicator rand_indic = ind_funcs._random # ========================= # === Make Abstractions === # ========================= # Directed Variants. v_directed_sa, v_directed_aa = get_abstractions(environment, v_disc_indic, directed=True, max_options=max_options) # v_directed_sa, v_directed_aa = get_abstractions(environment, v_indic, directed=True, max_options=max_options) # Identity action abstraction. identity_sa, identity_aa = get_sa(environment, default=True), get_aa(environment, default=True) if exp_type == "core": # Core only abstraction types. q_directed_sa, q_directed_aa = get_abstractions( environment, q_indic, directed=True, max_options=max_options) rand_directed_sa, rand_directed_aa = get_abstractions( environment, rand_indic, directed=True, max_options=max_options) pblocks_sa, pblocks_aa = get_sa( environment, default=True), action_abs.aa_baselines.get_policy_blocks_aa( environment, incl_prim_actions=True, num_options=max_options) # =================== # === Make Agents === # =================== # Base Agents. agent_class = QLearningAgent if agent_class_str == "ql" else RMaxAgent rand_agent = RandomAgent(actions) baseline_agent = agent_class(actions, gamma=gamma) if mdp_class == "pblocks": baseline_agent.epsilon = 0.01 # Abstraction Extensions. agents = [] vabs_agent_directed = AbstractionWrapper(agent_class, actions, str(environment), max_option_steps=max_option_steps, state_abstr=v_directed_sa, action_abstr=v_directed_aa, name_ext="v-sa+aa") if exp_type == "core": # Core only agents. qabs_agent_directed = AbstractionWrapper( agent_class, actions, str(environment), max_option_steps=max_option_steps, state_abstr=q_directed_sa, action_abstr=q_directed_aa, name_ext="q-sa+aa") rabs_agent_directed = AbstractionWrapper( agent_class, actions, str(environment), max_option_steps=max_option_steps, state_abstr=rand_directed_sa, action_abstr=rand_directed_aa, name_ext="rand-sa+aa") pblocks_agent = AbstractionWrapper(agent_class, actions, str(environment), max_option_steps=max_option_steps, state_abstr=pblocks_sa, action_abstr=pblocks_aa, name_ext="pblocks") agents = [ vabs_agent_directed, qabs_agent_directed, rabs_agent_directed, pblocks_agent, baseline_agent ] elif exp_type == "combo": # Combo only agents. aa_agent = AbstractionWrapper(agent_class, actions, str(environment), max_option_steps=max_option_steps, state_abstr=identity_sa, action_abstr=v_directed_aa, name_ext="aa") sa_agent = AbstractionWrapper(agent_class, actions, str(environment), max_option_steps=max_option_steps, state_abstr=v_directed_sa, action_abstr=identity_aa, name_ext="sa") agents = [vabs_agent_directed, sa_agent, aa_agent, baseline_agent] # Run experiments. if multi_task: steps = 999999 if x_axis_num_options else steps run_agents_multi_task(agents, environment, task_samples=task_samples, steps=steps, episodes=episodes, reset_at_terminal=True) else: run_agents_on_mdp(agents, environment, instances=20, episodes=30, reset_at_terminal=True)
def main(): # Make MDP Distribution. mdp_class = "four_room" environment = make_mdp.make_mdp_distr(mdp_class=mdp_class, grid_dim=10) make_random_sa_diropt_aa_stack(environment, max_num_levels=3)