def add(self, action): transitions = [] for relation in self.machine.graph.transitions: if isinstance(relation.right, Stop) and (relation.label == 0 or isinstance(relation.left, Start)): a = Action(action=action) if relation.label == 0: transitions.append( MachineRelation(left=relation.left, right=a, label=0)) else: transitions.append( MachineRelation(left=relation.left, right=a)) transitions.append( MachineRelation(left=a, right=self.machine.graph.get_stop(), label=0)) else: transitions.append(relation) res = MachineGraph(transitions=transitions) for vertex in res.get_special_vertices(Action): # print("::", res.graph.action_vertex_label_mapping[vertex]) if not res.vertex_mapping[ vertex] and not res.vertex_reverse_mapping[vertex]: continue if 1 not in res.action_vertex_label_mapping[vertex]: res.transitions.append( MachineRelation(left=vertex, right=res.get_stop(), label=1)) self.machine = RandomMachine(graph=MachineGraph( transitions=transitions))
def get_machine(self): transitions = [] for left_ind in range(len(self.vertex_types)): for right_ind in range(len(self.vertex_types)): left = self.vertex_types[left_ind] right = self.vertex_types[right_ind] if (2 ** (left_ind * len(self.vertex_types) + right_ind)) & self.binary_matrix_representation: if isinstance(left, Action): transitions.append(MachineRelation(left=left, right=right, label=0)) else: transitions.append(MachineRelation(left=left, right=right)) start, stop = None, None for vertex in self.vertex_types: if isinstance(vertex, Start): start = vertex elif isinstance(vertex, Stop): stop = vertex assert start is not None assert stop is not None for vertex in [_ for _ in self.vertex_types if isinstance(_, Action)]: transitions.append(MachineRelation(left=vertex, right=stop, label=1)) return AbstractMachine(MachineGraph(transitions=transitions, vertices=self.vertex_types))
def __init__(self): action = Action(action=0) transition = ( MachineRelation(left=Start(), right=action), MachineRelation(left=action, right=action, label=0), MachineRelation(left=action, right=Stop(), label=1), ) super().__init__(graph=MachineGraph(transitions=transition))
def add_action_transitions(transitions): res = MachineGraph(transitions=transitions) stop = res.get_stop() for vertex in res.get_special_vertices(Action): if not res.vertex_mapping[vertex] and not res.vertex_reverse_mapping[ vertex]: continue if 1 not in res.action_vertex_label_mapping[vertex]: res.transitions.append( MachineRelation(left=vertex, right=stop, label=1)) return res.transitions
def __init__(self, env: MazeWorldEpisodeLength): stop = Stop() up1 = Action(env.ACTIONS.UP) up2 = Action(env.ACTIONS.UP) transitions = [ MachineRelation(left=Start(), right=up1), MachineRelation(left=up1, right=up2, label=0), MachineRelation(left=up2, right=stop, label=0), ] transitions = add_action_transitions(transitions) super().__init__(graph=MachineGraph(transitions=transitions))
def __init__(self, env: MazeWorldEpisodeLength): left4 = Action(action=env.ACTIONS.LEFT) left5 = Action(action=env.ACTIONS.LEFT) up4 = Action(action=env.ACTIONS.UP) up5 = Action(action=env.ACTIONS.UP) choice1 = Choice() choice2 = Choice() left = Action(action=env.ACTIONS.LEFT) right = Action(action=env.ACTIONS.RIGHT) up = Action(action=env.ACTIONS.UP) down = Action(action=env.ACTIONS.DOWN) stop = Stop() transitions = ( MachineRelation(left=Start(), right=choice1), MachineRelation(left=choice1, right=left4), MachineRelation(left=left4, right=left5, label=0), MachineRelation(left=left5, right=choice2, label=0), MachineRelation(left=left4, right=stop, label=1), MachineRelation(left=left5, right=stop, label=1), MachineRelation(left=choice1, right=up4), MachineRelation(left=up4, right=up5, label=0), MachineRelation(left=up5, right=choice2, label=0), MachineRelation(left=up4, right=stop, label=1), MachineRelation(left=up5, right=stop, label=1), MachineRelation(left=choice2, right=left), MachineRelation(left=choice2, right=right), MachineRelation(left=choice2, right=up), MachineRelation(left=choice2, right=down), MachineRelation( left=left, right=stop, label=1, ), MachineRelation(left=right, right=stop, label=1), MachineRelation(left=up, right=stop, label=1), MachineRelation(left=down, right=stop, label=1), MachineRelation( left=left, right=stop, label=0, ), MachineRelation(left=right, right=stop, label=0), MachineRelation(left=up, right=stop, label=0), MachineRelation(left=down, right=stop, label=0), ) super().__init__(graph=MachineGraph(transitions=transitions))
def _reset(self): self.machine = RandomMachine() self.state = tuple() self.last_reward = 0 self.action_space = spaces.Discrete(len(self.ACTIONS)) # TODO implement done self._done = False self.vertex_added = 0 self.edges_added = 0 self.machine = RandomMachine(graph=MachineGraph( transitions=[MachineRelation(left=Start(), right=Stop())]))
def go(transitions, brute_force, index_): machine = AbstractMachine(MachineGraph(transitions=transitions)) am = RootMachine(LoopInvokerMachine(machine)) # if randrange(1000) == 0: # draw_graph("{brute_force}".format(**locals()), am.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict())) # exit(0) if is_it_machine_runnable(machine): sum_rew = 0 try: params = HAMParamsCommon(environments[0]) ham_runner(ham=am, num_episodes=2, env=environments[0], params=params) sum_rew = sum(params.logs["ep_rewards"]) except ChildProcessError: # print(brute_force) pass # if randrange(1500) == 0: # draw_graph("bf{brute_force}".format(**locals()), am.get_graph_to_draw()) if sum_rew > 0: # TODO # with open("out.txt", "a") as f: # f.write(str(brute_force) + "\n") # return # print("\n\n EPISODE REWARD: ", sum_rew) # draw_graph("{sum_rew}__{brute_force}".format(**locals()), am.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict())) rew = None print("\n\n\n") for e in environments: params = HAMParamsCommon(e) ham_runner(ham=am, num_episodes=600, env=e, params=params) if rew is None: rew = 0 rew += sum(params.logs["ep_rewards"]) print("to_add:", sum(params.logs["ep_rewards"])) # except ChildProcessError: # draw_graph("{rew}__{brute_force}".format(**locals()), am.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict())) # exit(0) # pass if rew is not None: draw_graph( "{rew}__{brute_force}_{index_}".format(**locals()), am.get_graph_to_draw( action_to_name_mapping=env.get_actions_as_dict()))
def __init__(self, env: MazeWorldEpisodeLength): stop = Stop() left = Action(env.ACTIONS.LEFT) up = Action(env.ACTIONS.UP) choice = Choice() transitions = [ MachineRelation(left=Start(), right=left), MachineRelation(left=left, right=choice, label=0), MachineRelation(left=choice, right=left), MachineRelation(left=choice, right=up), MachineRelation(left=up, right=stop, label=0), ] transitions = add_action_transitions(transitions) super().__init__(graph=MachineGraph(transitions=transitions))
def super_runner(call_me_maybe, env): start = Start() choice_one = Choice() actions = [Action(action=_) for _ in env.get_actions_as_dict().values()] stop = Stop() call = Call(call_me_maybe) transitions = [ MachineRelation(left=start, right=choice_one), ] for action in actions: transitions.append(MachineRelation(left=choice_one, right=action)) transitions.append(MachineRelation(left=action, right=stop, label=0)) transitions.append(MachineRelation(left=action, right=stop, label=1)) transitions.append(MachineRelation(left=choice_one, right=call)) transitions.append(MachineRelation(left=call, right=stop)) return AbstractMachine(graph=MachineGraph(transitions=transitions))
def __init__(self, env): start = Start() choice_one = Choice() actions = [ Action(action=_) for _ in env.get_actions_as_dict().values() ] stop = Stop() transitions = [ MachineRelation(left=start, right=choice_one), ] for action in actions: transitions.append(MachineRelation(left=choice_one, right=action)) transitions.append( MachineRelation(left=action, right=stop, label=0)) transitions.append( MachineRelation(left=action, right=stop, label=1)) super().__init__(graph=MachineGraph(transitions=transitions))
def __init__(self, env: ArmEnvToggleTopOnly): d1 = Action(action=env.ACTIONS.UP) d2 = Action(action=env.ACTIONS.UP) d3 = Action(action=env.ACTIONS.UP) # d4 = Action(action=env.ACTIONS.UP) stop = Stop() transitions = ( MachineRelation(left=Start(), right=d1), MachineRelation(left=d1, right=d2, label=0), MachineRelation(left=d2, right=d3, label=0), MachineRelation(left=d3, right=stop, label=0), # MachineRelation(left=d4, right=stop, label=0), MachineRelation(left=d1, right=stop, label=1), MachineRelation(left=d2, right=stop, label=1), MachineRelation(left=d3, right=stop, label=1), # MachineRelation(left=d4, right=stop, label=1), ) super().__init__(graph=MachineGraph(transitions=transitions))
def main(): class UpMachine4(AbstractMachine): def __init__(self, env: ArmEnvToggleTopOnly): d1 = Action(action=env.ACTIONS.UP) d2 = Action(action=env.ACTIONS.UP) d3 = Action(action=env.ACTIONS.UP) d4 = Action(action=env.ACTIONS.UP) stop = Stop() transitions = ( MachineRelation(left=Start(), right=d1), MachineRelation(left=d1, right=d2, label=0), MachineRelation(left=d2, right=d3, label=0), MachineRelation(left=d3, right=d4, label=0), MachineRelation(left=d4, right=stop, label=0), MachineRelation(left=d1, right=stop, label=1), MachineRelation(left=d2, right=stop, label=1), MachineRelation(left=d3, right=stop, label=1), MachineRelation(left=d4, right=stop, label=1), ) super().__init__(graph=MachineGraph(transitions=transitions)) class UpMachine3(AbstractMachine): def __init__(self, env: ArmEnvToggleTopOnly): d1 = Action(action=env.ACTIONS.UP) d2 = Action(action=env.ACTIONS.UP) d3 = Action(action=env.ACTIONS.UP) # d4 = Action(action=env.ACTIONS.UP) stop = Stop() transitions = ( MachineRelation(left=Start(), right=d1), MachineRelation(left=d1, right=d2, label=0), MachineRelation(left=d2, right=d3, label=0), MachineRelation(left=d3, right=stop, label=0), # MachineRelation(left=d4, right=stop, label=0), MachineRelation(left=d1, right=stop, label=1), MachineRelation(left=d2, right=stop, label=1), MachineRelation(left=d3, right=stop, label=1), # MachineRelation(left=d4, right=stop, label=1), ) super().__init__(graph=MachineGraph(transitions=transitions)) a = [ Choice(), Action(ArmEnvToggleTopOnly.ACTIONS.RIGHT), Action(ArmEnvToggleTopOnly.ACTIONS.LEFT), Action(ArmEnvToggleTopOnly.ACTIONS.DOWN), # Action(ArmEnvToggleTopOnly.ACTIONS.UP), Call(machine_to_call=UpMachine4(environments[1])), ] transitions = [] for i in a: for j in a: if randrange(2): if isinstance(i, Action): transitions.append( MachineRelation(left=i, right=j, label=0)) else: transitions.append(MachineRelation(left=i, right=j)) # len_ = len(goodhams) # print(len_) # len_4 = len_ // 4 + 1 # l1, r1 = 0, len_4 # l2, r2 = len_4, 2 * len_4 # l3, r3 = 2 * len_4, 3 * len_4 # l4, r4 = 3 * len_4, 4 * len_4 # print(l1, r1 ) # print(l2, r2 ) # print(l3, r3 ) # print(l4, r4 ) # exit(0) # for brute_force in goodhams: # for index, brute_force in enumerate(goodhams[l1: r1]): # for index, brute_force in enumerate(goodhams[l2: r2]): # for index, brute_force in enumerate(goodhams[l3: r3]): brute_force = 1180698 # if bin(brute_force).count("1") > 12 or bin(brute_force).count("1") < 4: # continue # continue go_continue = False transitions = [] ss = set() for ii in range(len(a)): for jj in range(len(a)): i = a[ii] j = a[jj] if (2**(ii * len(a) + jj)) & brute_force: if isinstance(i, Action): transitions.append( MachineRelation(left=i, right=j, label=0)) else: transitions.append(MachineRelation(left=i, right=j)) if ii in ss and isinstance(a[ii], (Action, Call)): go_continue = True break ss.add(ii) stop = Stop() for ii in range(len(a)): if ii not in ss: i = a[ii] if isinstance(i, Action): transitions.append(MachineRelation(left=i, right=stop, label=0)) else: transitions.append(MachineRelation(left=i, right=stop)) for i in a: if isinstance(i, Action): transitions.append(MachineRelation(left=i, right=stop, label=1)) transitions.append(MachineRelation(left=Start(), right=a[0])) machine = AbstractMachine(MachineGraph(transitions=transitions)) am = RootMachine(LoopInvokerMachine(machine)) env = environments[0] draw_graph( "{brute_force}".format(**locals()), am.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict())) name = "02_auto" def run(global_env): full_name = name params = HAMParamsCommon(environments[0]) ham_runner(ham=am, num_episodes=global_episodes, env=env, params=params) rewards = params.logs["ep_rewards"] # with open(full_name + " cumulative_reward.txt", "w") as w: # for out in get_cumulative_rewards(rewards=rewards): # w.write(str(out) + '\n', ) with open(full_name + " reward.txt", "w") as w: for out in rewards: w.write(str(out) + '\n', ) def main(): # for global_env in EnvironmentsArticle().environments: run(EnvironmentsArticle().environments[0]) if __name__ == '__main__': main()
def __init__(self, env): pull_up_start = Start() pull_up_on = Action(action=env.get_actions_as_dict()["ON"]) pull_up_down_01 = Action(action=env.get_actions_as_dict()["DOWN"]) pull_up_down_02 = Action(action=env.get_actions_as_dict()["DOWN"]) pull_up_down_03 = Action(action=env.get_actions_as_dict()["DOWN"]) pull_up_down_04 = Action(action=env.get_actions_as_dict()["DOWN"]) pull_up_up_01 = Action(action=env.get_actions_as_dict()["UP"]) pull_up_up_02 = Action(action=env.get_actions_as_dict()["UP"]) pull_up_up_03 = Action(action=env.get_actions_as_dict()["UP"]) pull_up_up_04 = Action(action=env.get_actions_as_dict()["UP"]) pull_up_stop = Stop() pull_up_transitions = ( MachineRelation(left=pull_up_start, right=pull_up_on), MachineRelation(left=pull_up_on, right=pull_up_down_01, label=0), MachineRelation(left=pull_up_down_01, right=pull_up_down_02, label=0), MachineRelation(left=pull_up_down_02, right=pull_up_down_03, label=0), MachineRelation(left=pull_up_down_03, right=pull_up_down_04, label=0), MachineRelation(left=pull_up_down_04, right=pull_up_up_01, label=0), MachineRelation(left=pull_up_up_01, right=pull_up_up_02, label=0), MachineRelation(left=pull_up_up_02, right=pull_up_up_03, label=0), MachineRelation(left=pull_up_up_03, right=pull_up_up_04, label=0), MachineRelation(left=pull_up_up_04, right=pull_up_stop, label=0), MachineRelation(left=pull_up_on, right=pull_up_stop, label=1), MachineRelation(left=pull_up_down_01, right=pull_up_stop, label=1), MachineRelation(left=pull_up_down_02, right=pull_up_stop, label=1), MachineRelation(left=pull_up_down_03, right=pull_up_stop, label=1), MachineRelation(left=pull_up_down_04, right=pull_up_stop, label=1), MachineRelation(left=pull_up_up_01, right=pull_up_stop, label=1), MachineRelation(left=pull_up_up_02, right=pull_up_stop, label=1), MachineRelation(left=pull_up_up_03, right=pull_up_stop, label=1), MachineRelation(left=pull_up_up_04, right=pull_up_stop, label=1), ) pull_up = AbstractMachine( MachineGraph(transitions=pull_up_transitions)) start = Start() choice_one = Choice() left = Action(action=env.get_actions_as_dict()["LEFT"]) right = Action(action=env.get_actions_as_dict()["RIGHT"]) off = Action(action=env.get_actions_as_dict()["OFF"]) call = Call(machine_to_call=pull_up) stop = Stop() transitions = ( MachineRelation(left=start, right=choice_one), MachineRelation(left=choice_one, right=left), MachineRelation(left=choice_one, right=right), MachineRelation(left=choice_one, right=off), MachineRelation(left=choice_one, right=call), MachineRelation(left=call, right=stop), MachineRelation(left=left, right=stop, label=0), MachineRelation(left=right, right=stop, label=0), MachineRelation(left=off, right=stop, label=0), MachineRelation(left=left, right=stop, label=1), MachineRelation(left=right, right=stop, label=1), MachineRelation(left=off, right=stop, label=1), ) super().__init__(graph=MachineGraph(transitions=transitions))
MachineRelation(left=pull_up_down_04, right=pull_up_up_01, label=0), MachineRelation(left=pull_up_up_01, right=pull_up_up_02, label=0), MachineRelation(left=pull_up_up_02, right=pull_up_up_03, label=0), MachineRelation(left=pull_up_up_03, right=pull_up_up_04, label=0), MachineRelation(left=pull_up_up_04, right=pull_up_stop, label=0), MachineRelation(left=pull_up_on, right=pull_up_stop, label=1), MachineRelation(left=pull_up_down_01, right=pull_up_stop, label=1), MachineRelation(left=pull_up_down_02, right=pull_up_stop, label=1), MachineRelation(left=pull_up_down_03, right=pull_up_stop, label=1), MachineRelation(left=pull_up_down_04, right=pull_up_stop, label=1), MachineRelation(left=pull_up_up_01, right=pull_up_stop, label=1), MachineRelation(left=pull_up_up_02, right=pull_up_stop, label=1), MachineRelation(left=pull_up_up_03, right=pull_up_stop, label=1), MachineRelation(left=pull_up_up_04, right=pull_up_stop, label=1), ) pull_up = AbstractMachine(MachineGraph(transitions=pull_up_transitions)) start = Start() choice_one = Choice() left = Action(action=env.get_actions_as_dict()["LEFT"]) right = Action(action=env.get_actions_as_dict()["RIGHT"]) off = Action(action=env.get_actions_as_dict()["OFF"]) call = Call(machine_to_call=pull_up) stop = Stop() transitions = ( MachineRelation(left=start, right=choice_one), MachineRelation(left=choice_one, right=left), MachineRelation(left=choice_one, right=right),