def __init__(self, env, max_layer): self.env = env self.modifications = [] self.counter = 0 self.nodes = [] # List of all possible modifications: tree.modifications for wall in env.walls: self.modifications.append((0, wall[0], wall[1])) for row in range(env.width): for col in range(env.length): self.modifications.append((1, row, col)) self.num_nodes = 0 self.root = None self.threshold = 8 # Train original agent self.agent = QAgent(self.env) self.agent.qlearn(500, show=False, render=False) self.env.reset() self.max_layer = max_layer # Storing tree's max reward with corresponding environment self.max_reward = float("-inf") self.opt_env = None
def potency(mutual_ls, agent, modified, num_episodes, index): # This function tests the potency of the connected q-learning paradigm. # Parameters # ============================================== # agent: pre-trained agent in some environment # modified: new environment # num_episodes: number of episodes trained in connected q-learning paradigm # ============================================== series = agent.env.resettable_states() conn_agent = connected_qlearn(agent, modified, num_episodes) l_vals = [] for state in series: res = conn_agent.eval(fixed=state, show=False)[1] l_vals.append(res) new_agent = QAgent(modified) new_agent.qlearn(600, show=False, render=False) n_vals = [] for state in series: res = new_agent.eval(fixed=state, show=False)[1] n_vals.append(res) l_vals = np.array(l_vals) n_vals = np.array(n_vals) a = abs(np.sum(l_vals) - np.sum(n_vals)) mutual_ls[index] = a
def __init__(self, env, max_layer): self.env = env self.modifications = [] self.counter = 0 self.nodes = [] agent = QAgent(env) agent.qlearn(600, show=False) wall_dict = wall_interference(agent) cell_dict = cell_frequency(agent) for element in wall_dict: self.modifications.append((0, element[0][0], element[0][1])) for element in cell_dict[0:14]: self.modifications.append((1, element[0][0], element[0][1])) self.num_nodes = 0 self.root = None self.max_layer = max_layer self.threshold = 8 # Storing best reward and corresponding environment self.max_reward = float("-inf") self.opt_env = None
def default_policy(self, node_index): start = node_index simulate_env = copy.deepcopy(self.nodes[start].env) num_modifications_applied = len(self.env.walls) - len( simulate_env.walls) + len(simulate_env.special) - len( self.env.special) mods_left = self.max_layer - num_modifications_applied # Choose from unused modifications, from start node # We know that tree.nodes[start] is a leaf, so there is no used modifications at start yet. ls = [] for i in range(self.nodes[start].modification + 1, len(self.modifications)): ls.append(i) try: a = random.sample(ls, k=mods_left) except: print(ls) print(num_modifications_applied) raise ValueError a = sorted(a) for element in a: mod = self.modifications[element] if mod[0] == 0: simulate_env = simulate_env.transition([(mod[1], mod[2])]) elif mod[0] == 1: simulate_env.special.append((mod[1], mod[2])) # Training agent = QAgent(simulate_env) agent.qlearn(600, render=False) reward = utility(agent) if reward > self.threshold + 0.5: print(colored(a, "red")) print(colored(reward, "red")) for element in a: start = self.add_node(element, start).index # Update tree's max reward if possible if reward > self.max_reward: self.max_reward = reward self.opt_env = simulate_env return [self.scale(reward), start] return self.scale(reward)
def best_observed_choice(self): vector = [] for wall in self.env.walls: if wall not in self.opt_env.walls: tup = (0, wall[0], wall[1]) vector.append(tup) for cell in self.opt_env.special: if cell not in self.env.special: tup = (1, cell[0], cell[1]) vector.append(tup) # Training to prevent errors arising from connected training agent = QAgent(self.opt_env) agent.qlearn(600) rews = utility(agent) return (vector, rews)
def greedy(self): walk = [] start = 0 while self.nodes[start].layer < self.max_layer: if len(self.nodes[start].visited_children) != 0: start = self.best_child(start, 0, 0, expanded=False) mod_index = self.nodes[start].modification walk.append(self.modifications[mod_index]) if len(walk) < self.max_layer: print("MCTS insufficient to get {} modifications!".format(self.max_layer)) return (walk, None) else: modified = make_env(self.env, walk) agent = QAgent(modified) agent.qlearn(600, render=False) rews = utility(agent) return (walk, rews)
def path_based(env, num_mods): # Train agent in original environment agent = QAgent(env) agent.qlearn(600, render=False) cell_dict = cell_frequency(agent) wall_dict = wall_interference(agent) opt_seq = None opt_val = float("-inf") for k in range(num_mods): seq = [] ref = copy.deepcopy(env) # Pick k modifications on walls walls_to_remove = [wall_dict[i][0] for i in range(k)] for wall in walls_to_remove: ref = ref.transition([(wall)]) seq.append((0, wall[0], wall[1])) num_special_cells = num_mods - k cells_to_assign = [cell_dict[i][0] for i in range(num_special_cells)] for cell in cells_to_assign: ref.special.append(cell) seq.append((1, cell[0], cell[1])) agent_k = QAgent(ref) print(colored("Iteration {} begins!".format(k), "red")) print(ref.walls, ref.special) agent_k.qlearn(600, render=False) rews = utility(agent_k) if rews > opt_val: opt_val = rews opt_seq = seq return (opt_seq, opt_val)
def connected_qlearn(agent, new_env, num_episodes): # Parameters # ============================================== # agent: pre-trained agent in some environment # new_env: new environment # ============================================== # We will use the pre-trained agent to train it in the new environment # Intuition is that the q-values only need slight changes, so it will be computationally wasteful to calculate from scratch linked_agent = QAgent(new_env) linked_agent.q = copy.deepcopy(agent.q) # linking the q-values together linked_agent.epsilon = 0.75 linked_agent.qlearn(num_episodes, render=False) return linked_agent
def check(input_data, index, output_data): modified = make_env(env, input_data, index) agent = QAgent(modified) agent.qlearn(600, show=False) rews = utility(agent) return (rews == output_data[index])
mutual_ls[index] = a if __name__ == "__main__": processes = [] mp.set_start_method = "spawn" num_processes = 10 manager = Manager() mutual = manager.list() for _ in range(N * num_processes): mutual.append(0) # Create default agent agent = QAgent(env) agent.qlearn(650) for iter in range(N): for i in range(num_processes): p = mp.Process(target=potency, args=(mutual, agent, modified, 400, i + iter * num_processes)) p.start() processes.append(p) for process in processes: process.join() for process in processes: process.terminate()
class Tree(): def __init__(self, env, max_layer): self.env = env self.modifications = [] self.counter = 0 self.nodes = [] # List of all possible modifications: tree.modifications for wall in env.walls: self.modifications.append((0, wall[0], wall[1])) for row in range(env.width): for col in range(env.length): self.modifications.append((1, row, col)) self.num_nodes = 0 self.root = None self.threshold = 8 # Train original agent self.agent = QAgent(self.env) self.agent.qlearn(500, show=False, render=False) self.env.reset() self.max_layer = max_layer # Storing tree's max reward with corresponding environment self.max_reward = float("-inf") self.opt_env = None def scale(self, x): # Scale the utility of the agent for backpropagation return max(0, x - self.threshold) def initialize(self): # Initialize the tree with the root # Assert that the tree is null (no root beforehand) assert self.root == None # Create root root = Node(None, self.num_nodes, None, self.env) # Add root to the list of nodes of the tree and increase the number of nodes self.nodes.append(root) self.num_nodes += 1 # Assign the root to the root field of the tree self.root = root # Update the environment configuration of the root self.root.update_walls_special(self, parent_bool=False) # Update the layer of the root (the tree's version) self.root.update_layer(self) def add_node(self, mod_index, parent_index): # This function adds a node with a modification index and a parent index into the MCTS tree. assert parent_index < self.num_nodes assert mod_index in self.nodes[parent_index].get_unused_modifications(self) # Create a node from the modification index (in tree.modifications) and the parent index on the tree node = Node(mod_index, self.num_nodes, parent_index, self.env) # Append the node to the list of nodes of the tree and increase the number of nodes self.nodes.append(node) self.num_nodes += 1 # Changing the boolean leaf status of the parent_index to False. self.nodes[parent_index].leaf = False # Update the layer of the newly added node self.nodes[node.index].update_layer(self) # Update the environment configuration of the newly added node self.nodes[node.index].update_walls_special(self, parent_bool=True) # Mark the newly added node as a visited child, with respect to its parent node self.nodes[parent_index].visited_children.append(node.index) # Return the node as output return self.nodes[node.index] def expand(self, node_index): # return index of an expanded node # Get the list of unused modifications on the node (given by the tree and the node index) ls = self.nodes[node_index].get_unused_modifications(self) # Assert that the node still has unvisited children assert len(ls) > 0 # Choose a random modification index from this list mod_index = random.choice(ls) # Add the node of this modification index into the tree node = self.add_node(mod_index, node_index) # Return the index of this newly created node as output return node.index def best_child(self, node_index, const, const_2=1, expanded=True): # Find the best child of a node based on the UCB heuristic # If node is not fully expanded, use the heuristic only on visited children assert self.num_nodes > node_index ls = self.nodes[node_index].get_unused_modifications(self) # If expanded boolean is True, the length of unused modifications list must be 0 if expanded: assert len(ls) == 0 # Find the best child with largest UCB value opt = float("-inf") child = None for c in self.nodes[node_index].visited_children: # Calculate the term for child c scaled_reward = self.nodes[c].sum_reward / self.nodes[c].count exploration_term = const * math.sqrt(2 * math.log(self.nodes[node_index].count) / self.nodes[c].count) extra = 0 if len(self.nodes[c].simulation_history) != 0: extra = const_2 * math.sqrt(np.var(self.nodes[c].simulation_history) + 1 / self.nodes[c].count) result = scaled_reward + exploration_term + extra # Schadd SP-MCTS added term # Compare to running maximum if result > opt: opt = result child = c chosen_mod = self.modifications[self.nodes[child].modification] print(colored("Chosen child's modification: {}".format(chosen_mod), "red")) return child def default_policy(self, node_index): start = node_index simulate_env = copy.deepcopy(self.nodes[start].env) num_modifications_applied = len(self.env.walls) - len(simulate_env.walls) + len(simulate_env.special) mods_left = self.max_layer - num_modifications_applied # Choose from unused modifications, from start node # We know that tree.nodes[start] is a leaf, so there is no used modifications at start yet. ls = [] for i in range(self.nodes[start].modification + 1, len(self.modifications)): ls.append(i) # Following a completely random policy a = random.sample(ls, k=mods_left) a = sorted(a) for element in a: mod = self.modifications[element] if mod[0] == 0: simulate_env = simulate_env.transition([(mod[1], mod[2])]) elif mod[0] == 1: simulate_env.special.append((mod[1], mod[2])) # Training agent = connected_qlearn(self.agent, simulate_env, 400) reward = utility(agent) if reward > self.threshold + 0.5: print(colored(a, "red")) print(colored(reward, "red")) for element in a: start = self.add_node(element, start).index if reward > self.max_reward: self.max_reward = reward self.opt_env = simulate_env return [self.scale(reward), start] return self.scale(reward) def tree_policy(self, node_index, c1, c2): iter_index = node_index while not self.nodes[iter_index].terminal(self): if not self.nodes[iter_index].fully_expanded(self): return self.expand(iter_index) else: iter_index = self.best_child(iter_index, c1, c2) return iter_index def backup(self, node_index, reward): iter_index = node_index while iter_index is not None: self.nodes[iter_index].sum_reward += reward self.nodes[iter_index].simulation_history.append(reward) self.nodes[iter_index].count += 1 iter_index = self.nodes[iter_index].parent def ucb_search(self, iterations): root_index = self.nodes[0].index c1 = 1 c2 = 1 for i in range(iterations): print(colored("Iteration {} begins!".format(i), "red")) # Return the index of the node newly expanded and play out to the terminal (or node at last layer) leaf_index = self.tree_policy(root_index, c1, c2) a = self.default_policy(leaf_index) if isinstance(a, list): leaf_index = a[1] reward = a[0] else: reward = a self.backup(leaf_index, reward) print(colored("Number of nodes so far: {}".format(len(self.nodes)), "green")) print(colored("Maximum reward seen so far: {}".format(self.max_reward), "green")) print("Iteration {} ends!".format(i)) print() def greedy(self): walk = [] start = 0 while self.nodes[start].layer < self.max_layer: if len(self.nodes[start].visited_children) != 0: start = self.best_child(start, 0, 0, expanded=False) mod_index = self.nodes[start].modification walk.append(self.modifications[mod_index]) if len(walk) < self.max_layer: print("MCTS insufficient to get {} modifications!".format(self.max_layer)) return (walk, None) else: modified = make_env(self.env, walk) agent = QAgent(modified) agent.qlearn(600, render=False) rews = utility(agent) return (walk, rews) def best_observed_choice(self): vector = [] for wall in self.env.walls: if wall not in self.opt_env.walls: tup = (0, wall[0], wall[1]) vector.append(tup) for cell in self.opt_env.special: if cell not in self.env.special: tup = (1, cell[0], cell[1]) vector.append(tup) # Training to prevent errors arising from connected training agent = QAgent(self.opt_env) agent.qlearn(600) rews = utility(agent) x = max(rews, self.max_reward) return (vector, x) def info(self, node_index): dict_return = {} for key in vars(self.nodes[node_index]): if key != "simulation_history": if key != "env": dict_return[key] = vars(self.nodes[node_index])[key] else: dict_return["walls"] = self.nodes[node_index].env.walls dict_return["special_cells"] = self.nodes[node_index].env.special return dict_return
rounds = 5 mp.set_start_method = "spawn" num_processes = 10 processes = [] manager = Manager() agents = manager.list() for i in range(rounds * num_processes): agents.append(0) # keeper categories = [] num_mods = 1 map_to_numpy = np.asarray(map, dtype="c") env = TaxiEnv(map_to_numpy) # reference environment orig_agent = QAgent(env) orig_agent.qlearn(600, show=False) cell_dict = cell_frequency(orig_agent) wall_dict = wall_interference(orig_agent) modifications = [] for element in wall_dict: modifications.append((0, element[0])) for element in cell_dict[0:14]: row, col = element[0] modifications.append((1, (row, col))) for iter in range(rounds): print(colored("Data addition round {} begins!".format(iter), "red")) for i in range(num_processes): results = simulate_env(env, num_mods)
def greedy(env, num_mods): # This function returns the sequence of modifications based on the wall and cell heuristics # Parameters # =============================================================== # env: the original environment # num_mods: the number of modifications # =============================================================== greedy_seq = [] ref = copy.deepcopy(env) agent = None for i in range(num_mods): # For each iteration, find out the wall that most interferes and the cell that is crossed the most. Try out all options. if i == 0: agent = QAgent(ref) agent.qlearn(600, render=False) else: agent = connected_qlearn(agent, ref, 300) # Take out the lists from the heuristics. wall_dict = wall_interference(agent) cell_dict = cell_frequency(agent) # Take out the max values, and the options to try out. wall_nums = [elem[1] for elem in wall_dict] max_wall = max(wall_nums) cell_nums = [elem[1] for elem in cell_dict] max_cell = max(cell_nums) wall_options = [elem[0] for elem in wall_dict if elem[1] == max_wall] cell_options = [elem[0] for elem in cell_dict if elem[1] == max_cell] # Test out all the options, get optimal modification opt_value = float("-inf") opt_choice = None category = -1 for wall in wall_options: print(colored("Testing environment", "red")) e = ref.transition([wall]) new_agent = connected_qlearn(agent, e, 300) # Get utility val = utility(new_agent) if val > opt_value: opt_value = val opt_choice = wall category = 0 for cell in cell_options: print(colored("Testing environment", "red")) e = copy.deepcopy(ref) e.special.append(cell) new_agent = connected_qlearn(agent, e, 300) # Get utility val = utility(new_agent) if val > opt_value: opt_value = val opt_choice = cell category = 1 assert (category != -1) # Store found modification and change the reference environment if category == 0: mod = (0, opt_choice[0], opt_choice[1]) greedy_seq.append(mod) ref = ref.transition([opt_choice]) elif category == 1: mod = (1, opt_choice[0], opt_choice[1]) greedy_seq.append(mod) ref.special.append(opt_choice) # Evaluate utility total_agent = QAgent(ref) total_agent.qlearn(600, render=False) result = utility(total_agent) # print(colored(result, "red")) return greedy_seq, result
class Tree(): def __init__(self, env, max_layer): self.env = env self.modifications = [] self.counter = 0 self.nodes = [] agent = QAgent(env) agent.qlearn(600, show=False) wall_dict = wall_interference(agent) cell_dict = cell_frequency(agent) for element in wall_dict: self.modifications.append((0, element[0][0], element[0][1])) for element in cell_dict[0:14]: self.modifications.append((1, element[0][0], element[0][1])) self.num_nodes = 0 self.root = None self.max_layer = max_layer self.threshold = 8 self.agent = QAgent(self.env) self.agent.qlearn(500, show=False, render=False) self.env.reset() # Storing best reward score and corresponding environment seen so far self.max_reward = float("-inf") self.opt_env = None def scale(self, x): return max(0, x - self.threshold) def initialize(self): assert self.root == None root = Node(None, self.num_nodes, None, self.env) self.nodes.append(root) self.num_nodes += 1 self.root = root self.root.update_walls_special(self, parent_bool=False) self.root.update_layer(self) def add_node(self, mod_index, parent_index): assert parent_index < self.num_nodes assert mod_index in self.nodes[parent_index].get_unused_modifications( self) node = Node(mod_index, self.num_nodes, parent_index, self.env) self.nodes.append(node) self.num_nodes += 1 self.nodes[parent_index].leaf = False self.nodes[node.index].update_layer(self) self.nodes[node.index].update_walls_special(self, parent_bool=True) self.nodes[parent_index].visited_children.append(node.index) return self.nodes[node.index] def expand(self, node_index): # return index of an expanded node assert self.num_nodes > node_index ls = self.nodes[node_index].get_unused_modifications(self) assert len(ls) > 0 # still have an unvisited child mod_index = random.choice(ls) node = self.add_node(mod_index, node_index) return node.index def best_child(self, node_index, const, const_2=1, expanded=True ): # return index of best child according to ucb heuristic assert self.num_nodes > node_index ls = self.nodes[node_index].get_unused_modifications(self) if expanded: assert len(ls) == 0 opt = float("-inf") child = None for c in self.nodes[node_index].visited_children: scaled_reward = self.nodes[c].sum_reward / self.nodes[c].count exploration_term = const * math.sqrt(2 * math.log( self.nodes[node_index].count) / self.nodes[c].count) extra = 0 if len(self.nodes[c].simulation_history) != 0: extra = const_2 * math.sqrt( np.var(self.nodes[c].simulation_history) + 1 / self.nodes[c].count) result = scaled_reward + exploration_term + extra # Schadd SP-MCTS added term if result > opt: opt = result child = c chosen_mod = self.modifications[self.nodes[child].modification] print( colored("Chosen child's modification: {}".format(chosen_mod), "red")) return child def default_policy(self, node_index): start = node_index simulate_env = copy.deepcopy(self.nodes[start].env) num_modifications_applied = len(self.env.walls) - len( simulate_env.walls) + len(simulate_env.special) - len( self.env.special) mods_left = self.max_layer - num_modifications_applied # Choose from unused modifications, from start node # We know that tree.nodes[start] is a leaf, so there is no used modifications at start yet. ls = [] for i in range(self.nodes[start].modification + 1, len(self.modifications)): ls.append(i) a = random.sample(ls, k=mods_left) a = sorted(a) for element in a: mod = self.modifications[element] if mod[0] == 0: simulate_env = simulate_env.transition([(mod[1], mod[2])]) elif mod[0] == 1: simulate_env.special.append((mod[1], mod[2])) # Training agent = connected_qlearn(self.agent, simulate_env, 425) reward = utility(agent) if reward > self.threshold + 0.5: print(colored(a, "red")) print(colored(reward, "red")) for element in a: start = self.add_node(element, start).index # Update tree's max reward environment if possible if reward > self.max_reward: self.max_reward = reward self.opt_env = copy.deepcopy(simulate_env) return [self.scale(reward), start] return self.scale(reward) def tree_policy(self, node_index, c1, c2): iter_index = node_index while not self.nodes[iter_index].terminal(self): if not self.nodes[iter_index].fully_expanded(self): return self.expand(iter_index) else: iter_index = self.best_child(iter_index, c1, c2) return iter_index def backup(self, node_index, reward): iter_index = node_index while iter_index is not None: self.nodes[iter_index].sum_reward += reward self.nodes[iter_index].simulation_history.append(reward) self.nodes[iter_index].count += 1 iter_index = self.nodes[iter_index].parent def ucb_search(self, iterations): root_index = self.nodes[0].index c1 = 1 c2 = 1 for i in range(iterations): print(colored("Iteration {} begins!".format(i), "red")) leaf_index = self.tree_policy(root_index, c1, c2) a = self.default_policy(leaf_index) if isinstance(a, list): leaf_index = a[1] reward = a[0] else: reward = a self.backup(leaf_index, reward) print( colored("Number of nodes so far: {}".format(len(self.nodes)), "green")) print( colored( "Maximum reward seen so far: {}".format(self.max_reward), "green")) print("Iteration {} ends!".format(i)) print() def greedy(self): walk = [] start = 0 while self.nodes[start].layer < self.max_layer: if len(self.nodes[start].visited_children) != 0: start = self.best_child(start, 0, 0, expanded=False) mod_index = self.nodes[start].modification walk.append(self.modifications[mod_index]) if len(walk) < self.max_layer: print("MCTS insufficient to get {} modifications".format( self.max_layer)) return (walk, None) else: modified = make_env(self.env, walk) agent = QAgent(modified) agent.qlearn(600, render=False) rews = utility(agent) return (walk, rews) def best_observed_choice(self): vector = [] for wall in self.env.walls: if wall not in self.opt_env.walls: tup = (0, wall[0], wall[1]) vector.append(tup) for cell in self.opt_env.special: if cell not in self.env.special: tup = (1, cell[0], cell[1]) vector.append(tup) # Training to prevent errors arising from connected training agent = QAgent(self.opt_env) agent.qlearn(600) rews = utility(agent) return (vector, rews) def info(self, node_index): dict_return = {} for key in vars(self.nodes[node_index]): if key != "simulation_history": if key != "env": dict_return[key] = vars(self.nodes[node_index])[key] else: dict_return["walls"] = self.nodes[node_index].env.walls dict_return["special_cells"] = self.nodes[ node_index].env.special return dict_return
data = [] if __name__ == "__main__": rounds = 20 mp.set_start_method = "spawn" num_processes = 10 processes = [] manager = Manager() agents = manager.list() for i in range(rounds * num_processes): agents.append(0) # keeper categories = [] num_mods = 2 orig_agent = QAgent(env) orig_agent.qlearn(600, render=False) for iter in range(rounds): print(colored("Data addition round {} begins!".format(iter), "red")) for i in range(num_processes): results = simulate_env(env, num_mods) modified = results[0] categories.append(results[1]) # agent = QAgent(modified) p = mp.Process(target=connected_qlearn_as_func, args=(orig_agent, modified, i, agents, i + iter * num_processes)) p.start() processes.append(p) for process in processes:
exist = True break if not exist: h.insert(seq, val) if i % 100 == 0: print(i) opt_seq = None opt_val = -1 for element in range(len(h.array)): seq = h.mod_seq(element) modified = make_env(env, seq) agent = QAgent(modified) agent.qlearn(600, render=False) rews = utility(agent) print(colored(rews, "red")) if rews > opt_val: opt_val = rews opt_seq = seq r_dir = os.path.abspath(os.pardir) data_dir = os.path.join(r_dir, "data") file_dir = os.path.join(data_dir, "sl_result_{}.txt".format(num_mods)) with open(file_dir, "w") as file: file.write("Modifications: ") file.write(str(opt_seq)) file.write("\n") file.write("Utility: ")
rounds = 100 mp.set_start_method = "spawn" num_processes = 10 processes = [] manager = Manager() agents = manager.list() for i in range(rounds * num_processes): agents.append(0) # keeper categories = [] num_mods = 3 map_to_numpy = np.asarray(map, dtype="c") env = TaxiEnv(map_to_numpy) # reference environment orig_agent = QAgent(env) orig_agent.qlearn(600, show=False) cell_dict = cell_frequency(orig_agent) wall_dict = wall_interference(orig_agent) modifications = [] for element in wall_dict: modifications.append((0, element[0])) for element in cell_dict[0:14]: row, col = element[0] modifications.append((1, (row, col))) for iter in range(rounds): print(colored("Data addition round {} begins!".format(iter), "red")) for i in range(num_processes): results = simulate_env(env, num_mods)
processes = [] manager = Manager() agents = manager.list() for i in range(rounds * num_processes): agents.append(0) # keeper categories = [] num_mods = 4 for iter in range(rounds): print(colored("Data addition round {} begins!".format(iter), "red")) for i in range(num_processes): results = simulate_env(env, num_mods) modified = results[0] categories.append(results[1]) agent = QAgent(modified) p = mp.Process(target=qlearn_as_func, args=(agent, modified, i, agents, i + iter * num_processes)) p.start() processes.append(p) for process in processes: process.join() for process in processes: process.terminate() for i in range(len(agents)): ut = utility(agents[i]) data.append((categories[i], ut))
index = random.randint(0, N - k) else: index = random.randint(index + 1, N - k + i) res.append(list[index]) return res if num_mods == 6: num_trials = int(2e+6) else: num_trials = int(1e+6) orig_agent = QAgent(env) orig_agent.qlearn(600) cell_dict = cell_frequency(orig_agent) wall_dict = wall_interference(orig_agent) modifications = [] for element in wall_dict: modifications.append((0, element[0][0], element[0][1])) for element in cell_dict[0:14]: row, col = element[0] modifications.append((1, row, col)) # Initialize and build heap sz = min(12 * num_mods, len(x_test)) h = Heap(model, x_test[0:sz], sz) h.build_heap()
def batch_greedy(env, num_mods, num_mods_per_run, ls_num_iters): # Parameters # ========================================================= # env: original environment # num_mods: total number of modifications # num_mods_per_run: total number of modifications considered in combination # ls_num_iters: list of number of iterations run for each number of modifications # ========================================================= # Example: [50, 200] means that if num_mods == 1, run 50 iterations, and if num_mods == 2, run 200 iterations. ref = copy.deepcopy(env) mods_ret = [] # answer of this algorithm # Initialize an MCTS tree tree = Tree(env, max_layer=num_mods) tree.initialize() # Keep a running count count = num_mods # Keep a running list of modifications ls_mods = copy.deepcopy(tree.modifications) # Initialize baseline baseline = 8 + 0.25 * num_mods_per_run assert (count >= num_mods_per_run) assert (len(ls_num_iters) == num_mods_per_run) while count > 0: print(colored(ls_mods, "red")) n = 0 if count >= num_mods_per_run: n = num_mods_per_run else: n = count tree = Tree(ref, max_layer=n) tree.initialize() tree.threshold = baseline tree.modifications = copy.deepcopy(ls_mods) # Find out number of iterations num_iter = ls_num_iters[n - 1] # Perform an MCTS search tree.ucb_search(iterations=num_iter) a = tree.best_observed_choice() for elem in a[0]: mods_ret.append(elem) # Transform the environment for elem in a[0]: if elem[0] == 0: # wall ref = ref.transition([(elem[1], elem[2])]) elif elem[0] == 1: # cell ref.special.append((elem[1], elem[2])) ls_mods.remove(elem) count -= n print(colored(mods_ret, "red")) # Increase baseline baseline += 0.5 * n # Find utility agent = QAgent(ref) agent.qlearn(600) rews = utility(agent) return (mods_ret, rews)