def draw_ru(self, filename):
action_to_name_mapping = self.env.get_actions_as_dict()
ru_mapping = {"LEFT": "ВЛЕВО",
"RIGHT": "ВПРАВО",
"DOWN": "ВНИЗ",
"TOGGLE": "ПЕРЕКЛ.",
"UP": "ВВЕРХ"
}
action_to_name_mapping_ru = {
}
for key in action_to_name_mapping.keys():
assert key in ru_mapping, "don't worry you just should add translation of key <<{key}>> to ru_mapping dict placed above".format(**locals())
action_to_name_mapping_ru[ru_mapping[key]] = action_to_name_mapping[key]
draw_graph(filename, self.get_machine().get_graph_to_draw(action_to_name_mapping=action_to_name_mapping_ru, no_edges_with_exit_f=True))
s = None
with open("{filename}.svg".format(**locals()), "r") as f:
s = f.readlines()
s = [re.sub(r"Action\d+", r"Действие", _) for _ in s]
s = [re.sub(r"Choice\d+", r"Выбор", _) for _ in s]
s = [re.sub(r"Call\d+", r"Вызов", _) for _ in s]
s = [re.sub(r"Stop\d+", r"Стоп", _) for _ in s]
s = [re.sub(r"Start\d+", r"Старт", _) for _ in s]
with open("{filename}.svg".format(**locals()), "w") as f:
f.writelines(s)
def draw(self, filename):
draw_graph(filename, self.get_machine().get_graph_to_draw(action_to_name_mapping=self.env.get_actions_as_dict(), no_edges_with_exit_f=True))
s = None
with open("{filename}.svg".format(**locals()), "r") as f:
s = f.readlines()
s = [re.sub(r"Action\d+", r"Action", _) for _ in s]
s = [re.sub(r"Choice\d+", r"Choice", _) for _ in s]
s = [re.sub(r"Call\d+", r"Call", _) for _ in s]
s = [re.sub(r"Stop\d+", r"Stop", _) for _ in s]
s = [re.sub(r"Start\d+", r"Start", _) for _ in s]
with open("{filename}.svg".format(**locals()), "w") as f:
f.writelines(s)
def go(transitions, brute_force, index_):
machine = AbstractMachine(MachineGraph(transitions=transitions))
am = RootMachine(LoopInvokerMachine(machine))
# if randrange(1000) == 0:
# draw_graph("{brute_force}".format(**locals()), am.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict()))
# exit(0)
if is_it_machine_runnable(machine):
sum_rew = 0
try:
params = HAMParamsCommon(environments[0])
ham_runner(ham=am,
num_episodes=2,
env=environments[0],
params=params)
sum_rew = sum(params.logs["ep_rewards"])
except ChildProcessError:
# print(brute_force)
pass
# if randrange(1500) == 0:
# draw_graph("bf{brute_force}".format(**locals()), am.get_graph_to_draw())
if sum_rew > 0:
# TODO
# with open("out.txt", "a") as f:
# f.write(str(brute_force) + "\n")
# return
# print("\n\n EPISODE REWARD: ", sum_rew)
# draw_graph("{sum_rew}__{brute_force}".format(**locals()), am.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict()))
rew = None
print("\n\n\n")
for e in environments:
params = HAMParamsCommon(e)
ham_runner(ham=am, num_episodes=600, env=e, params=params)
if rew is None:
rew = 0
rew += sum(params.logs["ep_rewards"])
print("to_add:", sum(params.logs["ep_rewards"]))
# except ChildProcessError:
# draw_graph("{rew}__{brute_force}".format(**locals()), am.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict()))
# exit(0)
# pass
if rew is not None:
draw_graph(
"{rew}__{brute_force}_{index_}".format(**locals()),
am.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
def _step(self, action):
self.state = self.state + tuple([action])
self.ham = RootMachine(
LoopInvokerMachine(
machine_to_invoke=super_runner(self.machine, self.env)))
reward = None
if action is None:
raise KeyError
elif action == self.ACTIONS.ACTION_01:
self.add(Action(action=action))
elif action == self.ACTIONS.ACTION_02:
self.add(Action(action=action))
elif action == self.ACTIONS.ACTION_03:
self.add(Action(action=action))
elif action == self.ACTIONS.ACTION_04:
self.add(Action(action=action))
elif action == self.ACTIONS.ACTION_05:
self.add(Action(action=action))
elif action == self.ACTIONS.ACTION_06:
self.add(Action(action=action))
if is_it_machine_runnable(self.machine):
if self.state in self.dp:
reward = self.dp[self.state]
else:
params = HAMParamsCommon(self.env)
ham_runner(ham=self.ham,
num_episodes=self.num_of_episodes,
env=self.env,
params=params,
no_output=True)
reward = sum(params.logs["ep_rewards"])
self.dp[self.state] = reward
draw_graph(
"pics/" + str(reward).rjust(10, "0") + str(self.state) + " ",
self.machine.get_graph_to_draw(
action_to_name_mapping=self.env.get_actions_as_dict()))
observation = self.state
if len(self.state) >= self.max_size:
self._done = True
return observation, reward, self._done, None
def run(global_env):
rewards = None
if isinstance(global_env, ArmEnvArticle):
env = global_env.env
internal_machine = M1(env=env)
machine = RootMachine(
LoopInvokerMachine(
super_runner(call_me_maybe=internal_machine, env=env)))
draw_graph(file_name="maze_env_special",
graph=internal_machine.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
params = HAMParamsCommon(env)
ham_runner(ham=machine,
num_episodes=global_env.episodes_count,
env=env,
params=params)
rewards = params.logs["ep_rewards"]
elif isinstance(global_env, MazeEnvArticle):
env = global_env.env
internal_machine = M2(env=env)
machine = RootMachine(
LoopInvokerMachine(
super_runner(call_me_maybe=internal_machine, env=env)))
draw_graph(file_name="maze_env_special",
graph=internal_machine.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
params = HAMParamsCommon(env)
ham_runner(ham=machine,
num_episodes=global_env.episodes_count,
env=env,
params=params)
rewards = params.logs["ep_rewards"]
elif isinstance(global_env, MazeEnvArticleSpecial):
env = global_env.env
internal_machine = M3(env=env)
machine = RootMachine(
LoopInvokerMachine(
super_runner(call_me_maybe=internal_machine, env=env)))
draw_graph(file_name="maze_env_special",
graph=internal_machine.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
params = HAMParamsCommon(env)
ham_runner(ham=machine,
num_episodes=global_env.episodes_count,
env=env,
params=params)
rewards = params.logs["ep_rewards"]
else:
raise KeyError
if rewards is not None:
full_name = "_" + global_env.__class__.__name__
with open(full_name + " cumulative_reward.txt", "w") as w:
for out in get_cumulative_rewards(rewards=rewards):
w.write(str(out) + '\n', )
with open(full_name + " reward.txt", "w") as w:
for out in rewards:
w.write(str(out) + '\n', )
def main(begin_seed=0):
for seed in range(begin_seed, begin_seed + 5000):
# maze = maze_world_input_special()
# maze = generate_maze_please(size_x=2, size_y=2)
# env = MazeWorldEpisodeLength(maze=maze)
# global_env, save_folder = MazeEnvArticleSpecial(), "laby_spec/"
global_env, save_folder = MazeEnvArticle(), "laby/"
# global_env, save_folder = ArmEnvArticle(), "arm/"
env, num_episodes = global_env.env, global_env.episodes_count
new_machine = create_random_machine(maximal_number_of_vertex=6, maximal_number_of_edges=6, random_seed=seed,
env=env)
if is_it_machine_runnable(new_machine):
params = HAMParamsCommon(env)
try:
ham_runner(
ham=RootMachine(LoopInvokerMachine(machine_to_invoke=super_runner(new_machine, env))),
num_episodes=num_episodes,
env=env, params=params,
no_output=True
)
ham_runner(ham=RootMachine(machine_to_invoke=LoopInvokerMachine(new_machine)),
num_episodes=num_episodes,
env=env, params=params, no_output=True)
# to_plot.append(PlotParams(curve_to_draw=params.logs["ep_rewards"], label="Random" + str(seed + 1)))
reward = sum(params.logs["ep_rewards"])
draw_graph(save_folder + str(reward) + ":::" + str(seed),
new_machine.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict()))
# draw_graph("pics/" + str(reward).rjust(10, "0"),
# new_machine.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict()))
except KeyError:
print("keyError", end="")
except AssertionError:
print("assertion", end="")
plot_multi(to_plot)
def run(global_env):
rewards = None
if isinstance(global_env, ArmEnvArticle):
pass
elif isinstance(global_env, MazeEnvArticle):
pass
elif isinstance(global_env, MazeEnvArticleSpecial):
env = global_env.env
seed = 573846788
internal_machine = create_random_machine(maximal_number_of_vertex=6,
maximal_number_of_edges=6,
random_seed=seed,
env=env)
machine = RootMachine(machine_to_invoke=LoopInvokerMachine(
machine_to_invoke=internal_machine))
draw_graph(file_name="maze_env_special",
graph=internal_machine.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
params = HAMParamsCommon(env)
ham_runner(ham=machine,
num_episodes=global_env.episodes_count,
env=env,
params=params)
rewards = params.logs["ep_rewards"]
else:
raise KeyError
if rewards is not None:
full_name = name + "_" + global_env.__class__.__name__
# with open(full_name + " cumulative_reward.txt", "w") as w:
# for out in get_cumulative_rewards(rewards=rewards):
# w.write(str(out) + '\n', )
with open(full_name + " reward.txt", "w") as w:
for out in rewards:
w.write(str(out) + '\n', )
def main(global_env, begin_seed=0):
for seed in range(begin_seed, begin_seed + 5000):
env = global_env.env
num_episodes = global_env.episodes_count
new_machine = create_random_machine(maximal_number_of_vertex=6,
maximal_number_of_edges=6,
random_seed=seed,
env=env)
if is_it_machine_runnable(new_machine):
params = HAMParamsCommon(env)
try:
ham_runner(ham=RootMachine(
machine_to_invoke=LoopInvokerMachine(new_machine)),
num_episodes=num_episodes,
env=env,
params=params)
if sum(params.logs["ep_rewards"][-100:]) > 0:
print("{test}done_it".format(**locals()),
sum(params.logs["ep_rewards"]))
to_plot.append(
PlotParams(curve_to_draw=params.logs["ep_rewards"],
label="Random" + str(seed + 1)))
draw_graph(
"pics/" + str(seed),
new_machine.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
except KeyError:
print("keyError", end="")
except AssertionError:
print("assertion", end="")
plot_multi(to_plot)
def run(global_env):
if isinstance(global_env, ArmEnvArticle):
env = global_env.env
internal_machine = PullUpMachine(env=env)
machine = RootMachine(machine_to_invoke=LoopInvokerMachine(
machine_to_invoke=internal_machine))
params = HAMParamsCommon(env)
draw_graph(file_name="arm_env",
graph=internal_machine.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
ham_runner(ham=machine,
num_episodes=global_env.episodes_count,
env=env,
params=params)
rewards = params.logs["ep_rewards"]
elif isinstance(global_env, MazeEnvArticle):
env = global_env.env
internal_machine = InterestingMachine(env=env)
machine = RootMachine(machine_to_invoke=LoopInvokerMachine(
machine_to_invoke=internal_machine))
draw_graph(file_name="maze_env",
graph=internal_machine.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
params = HAMParamsCommon(env)
ham_runner(ham=machine,
num_episodes=global_env.episodes_count,
env=env,
params=params)
rewards = params.logs["ep_rewards"]
elif isinstance(global_env, MazeEnvArticleSpecial):
env = global_env.env
internal_machine = InterestingMachineLeftUpInteresting(env=env)
machine = RootMachine(machine_to_invoke=LoopInvokerMachine(
machine_to_invoke=internal_machine))
draw_graph(file_name="maze_env_special",
graph=internal_machine.get_graph_to_draw(
action_to_name_mapping=env.get_actions_as_dict()))
params = HAMParamsCommon(env)
ham_runner(ham=machine,
num_episodes=global_env.episodes_count,
env=env,
params=params)
rewards = params.logs["ep_rewards"]
else:
raise KeyError
full_name = name + "_" + global_env.__class__.__name__
# with open(full_name + " cumulative_reward.txt", "w") as w:
# for out in get_cumulative_rewards(rewards=rewards):
# w.write(str(out) + '\n', )
with open(full_name + " reward.txt", "w") as w:
for out in rewards:
w.write(str(out) + '\n', )
def main():
class UpMachine4(AbstractMachine):
def __init__(self, env: ArmEnvToggleTopOnly):
d1 = Action(action=env.ACTIONS.UP)
d2 = Action(action=env.ACTIONS.UP)
d3 = Action(action=env.ACTIONS.UP)
d4 = Action(action=env.ACTIONS.UP)
stop = Stop()
transitions = (
MachineRelation(left=Start(), right=d1),
MachineRelation(left=d1, right=d2, label=0),
MachineRelation(left=d2, right=d3, label=0),
MachineRelation(left=d3, right=d4, label=0),
MachineRelation(left=d4, right=stop, label=0),
MachineRelation(left=d1, right=stop, label=1),
MachineRelation(left=d2, right=stop, label=1),
MachineRelation(left=d3, right=stop, label=1),
MachineRelation(left=d4, right=stop, label=1),
)
super().__init__(graph=MachineGraph(transitions=transitions))
class UpMachine3(AbstractMachine):
def __init__(self, env: ArmEnvToggleTopOnly):
d1 = Action(action=env.ACTIONS.UP)
d2 = Action(action=env.ACTIONS.UP)
d3 = Action(action=env.ACTIONS.UP)
# d4 = Action(action=env.ACTIONS.UP)
stop = Stop()
transitions = (
MachineRelation(left=Start(), right=d1),
MachineRelation(left=d1, right=d2, label=0),
MachineRelation(left=d2, right=d3, label=0),
MachineRelation(left=d3, right=stop, label=0),
# MachineRelation(left=d4, right=stop, label=0),
MachineRelation(left=d1, right=stop, label=1),
MachineRelation(left=d2, right=stop, label=1),
MachineRelation(left=d3, right=stop, label=1),
# MachineRelation(left=d4, right=stop, label=1),
)
super().__init__(graph=MachineGraph(transitions=transitions))
a = [
Choice(),
Action(ArmEnvToggleTopOnly.ACTIONS.RIGHT),
Action(ArmEnvToggleTopOnly.ACTIONS.LEFT),
Action(ArmEnvToggleTopOnly.ACTIONS.DOWN),
# Action(ArmEnvToggleTopOnly.ACTIONS.UP),
Call(machine_to_call=UpMachine4(environments[1])),
]
transitions = []
for i in a:
for j in a:
if randrange(2):
if isinstance(i, Action):
transitions.append(
MachineRelation(left=i, right=j, label=0))
else:
transitions.append(MachineRelation(left=i, right=j))
# len_ = len(goodhams)
# print(len_)
# len_4 = len_ // 4 + 1
# l1, r1 = 0, len_4
# l2, r2 = len_4, 2 * len_4
# l3, r3 = 2 * len_4, 3 * len_4
# l4, r4 = 3 * len_4, 4 * len_4
# print(l1, r1 )
# print(l2, r2 )
# print(l3, r3 )
# print(l4, r4 )
# exit(0)
# for brute_force in goodhams:
# for index, brute_force in enumerate(goodhams[l1: r1]):
# for index, brute_force in enumerate(goodhams[l2: r2]):
# for index, brute_force in enumerate(goodhams[l3: r3]):
brute_force = 1180698
# if bin(brute_force).count("1") > 12 or bin(brute_force).count("1") < 4:
# continue
# continue
go_continue = False
transitions = []
ss = set()
for ii in range(len(a)):
for jj in range(len(a)):
i = a[ii]
j = a[jj]
if (2**(ii * len(a) + jj)) & brute_force:
if isinstance(i, Action):
transitions.append(
MachineRelation(left=i, right=j, label=0))
else:
transitions.append(MachineRelation(left=i, right=j))
if ii in ss and isinstance(a[ii], (Action, Call)):
go_continue = True
break
ss.add(ii)
stop = Stop()
for ii in range(len(a)):
if ii not in ss:
i = a[ii]
if isinstance(i, Action):
transitions.append(MachineRelation(left=i, right=stop,
label=0))
else:
transitions.append(MachineRelation(left=i, right=stop))
for i in a:
if isinstance(i, Action):
transitions.append(MachineRelation(left=i, right=stop, label=1))
transitions.append(MachineRelation(left=Start(), right=a[0]))
machine = AbstractMachine(MachineGraph(transitions=transitions))
am = RootMachine(LoopInvokerMachine(machine))
env = environments[0]
draw_graph(
"{brute_force}".format(**locals()),
am.get_graph_to_draw(action_to_name_mapping=env.get_actions_as_dict()))
name = "02_auto"
def run(global_env):
full_name = name
params = HAMParamsCommon(environments[0])
ham_runner(ham=am,
num_episodes=global_episodes,
env=env,
params=params)
rewards = params.logs["ep_rewards"]
# with open(full_name + " cumulative_reward.txt", "w") as w:
# for out in get_cumulative_rewards(rewards=rewards):
# w.write(str(out) + '\n', )
with open(full_name + " reward.txt", "w") as w:
for out in rewards:
w.write(str(out) + '\n', )
def main():
# for global_env in EnvironmentsArticle().environments:
run(EnvironmentsArticle().environments[0])
if __name__ == '__main__':
main()
def draw_system_machines():
s1 = RandomMachine()
s2 = LoopInvokerMachine(machine_to_invoke=s1)
s3 = RootMachine(machine_to_invoke=s2)
draw_graph("full_hie", s3.get_graph_to_draw())