def part_two(env):
with open("machines_part_one.json") as json_file:
machines = [MachineStored.ms_from_machine(AutoMachineSimple(env), env)]
machines_to_save = []
for ms_dict in json.load(json_file):
machines.append(
MachineStored.from_dict(graph_dict=ms_dict, env=env))
m_id = 0
params = HAMParamsCommon(env)
am = AutoMachineSimple(env)
runner(
ham=am,
num_episodes=2000,
env=env,
params=params,
on_model_mapping={},
)
qv = params.q_value
for on_model_part in list(reversed(env.get_all_on_model())):
for ms in machines:
machine = ms.get_machine()
params = HAMParamsCommon(env)
params.q_value = qv
runner(
ham=am,
num_episodes=1,
env=env,
params=params,
on_model_mapping={on_model_part: machine},
)
to_plot = list()
to_plot.append(
PlotParams(curve_to_draw=params.logs["ep_rewards"],
label="HAM_with_pull_up"))
total_reward = sum(params.logs["ep_rewards"])
print("rewards sum:", total_reward)
# plot_multi(to_plot, filename="pics/" + str(m_id) + ":::" + str(on_model_part) + ":::" + str(ms.binary_matrix_representation) + ":::" + str(sum(params.logs["ep_rewards"])))
# ms.draw("pics/" + str(m_id) + ":" + str(ms.binary_matrix_representation) + ":" + str(total_reward))
m_id += 1
if total_reward > 10:
machines_to_save.append(ms)
with open("machines_part_two.json", "w") as out_f:
t = compress_graphs_dicts([_.to_dict() for _ in machines_to_save])
json.dump(obj=t, fp=out_f, sort_keys=True, indent=4)
def generate_machines_by_ids(env, vertexes, ids):
machines = []
for max_vertex_count in range(7):
vc = vertex_combination(vertex_types=vertexes,
max_vertex_count=max_vertex_count)
for index, vertex_types in enumerate(vc):
for graph_id in ids:
ms = MachineStored(vertex_types=vertex_types,
binary_matrix_representation=graph_id,
env=env)
if is_ham_ok(ms.get_machine_without_on_model()):
if check_for_one_component_graph(
ms.get_machine_without_on_model()):
if is_it_machine_runnable(
ms.get_machine_without_on_model()):
machines.append(ms)
return machines
def part_seven(env):
with open("machines_part_six.json") as json_file:
cluster_best_machine_mapper_str_key = json.load(json_file)
cluster_best_machine_mapper = {}
for key in cluster_best_machine_mapper_str_key:
tuple_key = key
tuple_key = tuple_key.replace("(", "")
tuple_key = tuple_key.replace(")", "")
tuple_key = tuple(map(eval, tuple_key.split(",")))
cluster_best_machine_mapper[tuple_key] = MachineStored.from_dict(
cluster_best_machine_mapper_str_key[key],
env=env).get_machine()
MachineStored.from_dict(cluster_best_machine_mapper_str_key[key],
env=env).draw("ololo" + str(key))
params = HAMParamsCommon(env)
runner(
ham=AutoMachineSimple(env),
num_episodes=2000,
env=env,
params=params,
on_model_mapping=cluster_best_machine_mapper,
# no_output=True,
)
to_plot = list()
to_plot.append(
PlotParams(curve_to_draw=params.logs["ep_rewards"],
label="clustering"))
save_to_gif("olololo", params.logs["gif"][-1])
params = HAMParamsCommon(env)
runner(
ham=AutoMachineSimple(env),
num_episodes=2000,
env=env,
params=params,
on_model_mapping={},
# no_output=True,
)
to_plot.append(
PlotParams(curve_to_draw=params.logs["ep_rewards"],
label="q-learning"))
plot_multi(to_plot, filename="ololo_result")
def main():
env = ArmEnvToggleTopOnly(size_x=5,
size_y=5,
cubes_cnt=4,
episode_max_length=600,
finish_reward=100,
action_minus_reward=-0.001,
tower_target_size=4)
vertexes = sorted([
Stop(),
Start(),
Action(env.ACTIONS.LEFT),
Action(env.ACTIONS.RIGHT),
Action(env.ACTIONS.UP),
Action(env.ACTIONS.DOWN),
# Action(env.ACTIONS.TOGGLE),
Choice(),
# Action(env.ACTIONS.LEFT),
# Action(env.ACTIONS.RIGHT),
# Action(env.ACTIONS.UP),
# Action(env.ACTIONS.DOWN),
# Action(env.ACTIONS.TOGGLE),
# Choice(),
])
# clearing directory
pathlib.Path('pics/').mkdir(parents=True, exist_ok=True)
shutil.rmtree('pics/')
pathlib.Path('pics/').mkdir(parents=True, exist_ok=True)
# brute force
for max_vertex_count in range(7):
vc = vertex_combination(vertex_types=vertexes,
max_vertex_count=max_vertex_count)
for index, vertex_types in enumerate(vc):
for graph_id in sorted(
get_graph_id_fast(
MachineStored(vertex_types=vertex_types,
binary_matrix_representation=412,
env=env))):
ms = MachineStored(vertex_types=vertex_types,
binary_matrix_representation=graph_id,
env=env)
if is_ham_ok(ms.get_machine_without_on_model()):
if check_for_one_component_graph(
ms.get_machine_without_on_model()):
if is_it_machine_runnable(
ms.get_machine_without_on_model()):
ms.draw("pics/" + str(max_vertex_count) + ":" +
str(index) + ":" + str(graph_id))
print("added")
def part_four(env):
with open("machines_part_three.json") as json_file:
cluster_best_machine_mapper_str_key = json.load(json_file)
cluster_best_machine_mapper = {}
for key in cluster_best_machine_mapper_str_key:
tuple_key = key
# tuple_key = key
tuple_key = tuple_key.replace("(", "")
tuple_key = tuple_key.replace(")", "")
tuple_key = tuple(map(eval, tuple_key.split(",")))
cluster_best_machine_mapper[tuple_key] = MachineStored.from_dict(
cluster_best_machine_mapper_str_key[key]["graph_dict"],
env=env)
cluster_best_machine_mapper_machine = {}
for i in cluster_best_machine_mapper:
cluster_best_machine_mapper_machine[
i] = cluster_best_machine_mapper[i].get_machine()
params = HAMParamsCommon(env)
runner(
ham=AutoMachineSimple(env),
num_episodes=300,
env=env,
params=params,
on_model_mapping=cluster_best_machine_mapper_machine,
no_output=True,
)
for cluster in cluster_best_machine_mapper:
ms = cluster_best_machine_mapper[cluster]
ms.draw(filename=str(cluster))
to_plot = list()
to_plot.append(
PlotParams(curve_to_draw=params.logs["ep_rewards"],
label="clustering, same env"))
plot_multi(to_plot, filename="a")
def generate_good_graphs(env, vertexes, vertex_count):
good_graphs = []
vertex_count += 1
for max_vertex_count in range(vertex_count):
vc = vertex_combination(vertex_types=vertexes,
max_vertex_count=max_vertex_count)
for index, vertex_types in enumerate(vc):
for graph_id in sorted(
get_graph_id_fast(
MachineStored(vertex_types=vertex_types,
binary_matrix_representation=412,
env=env))):
ms = MachineStored(vertex_types=vertex_types,
binary_matrix_representation=graph_id,
env=env)
if is_ham_ok(ms.get_machine_without_on_model()):
if check_for_one_component_graph(
ms.get_machine_without_on_model()):
if is_it_machine_runnable(
ms.get_machine_without_on_model()):
good_graphs.append(ms)
return good_graphs
def part_three(env):
with open("machines_part_two.json") as json_file:
machines = []
for ms_dict in json.load(json_file):
machines.append(
MachineStored.from_dict(graph_dict=ms_dict, env=env))
cluster_best_result_mapper = {}
cluster_best_machine_mapper = {}
clusters_to_save = {}
for on_model_part in list(reversed(env.get_all_on_model())):
for index, ms in enumerate(machines):
machine = ms.get_machine()
total_reward = 0
for tests in range(5):
params = HAMParamsCommon(env)
runner(
ham=AutoMachineSimple(env),
num_episodes=30,
env=env,
params=params,
on_model_mapping={on_model_part: machine},
no_output=True,
)
to_plot = list()
to_plot.append(
PlotParams(curve_to_draw=params.logs["ep_rewards"],
label="HAM_with_pull_up"))
total_reward += sum(params.logs["ep_rewards"])
# print(total_reward)
on_model_part_str = str(on_model_part)
if on_model_part_str in cluster_best_result_mapper:
if cluster_best_result_mapper[
on_model_part_str] < total_reward:
cluster_best_result_mapper[
on_model_part_str], cluster_best_machine_mapper[
on_model_part_str] = total_reward, ms.to_dict(
)
clusters_to_save[on_model_part_str] = {
"total_reward": total_reward,
"graph_dict": ms.to_dict()
}
else:
cluster_best_result_mapper[
on_model_part_str], cluster_best_machine_mapper[
on_model_part_str] = total_reward, ms.to_dict()
clusters_to_save[on_model_part_str] = {
"total_reward": total_reward,
"graph_dict": ms.to_dict()
}
# print('\n')
print("****")
ms_len = len(machines)
print("machine {index} of {ms_len}".format(**locals()))
print()
for i in ms.vertex_types:
print(i)
print(on_model_part_str, total_reward)
# print(clusters_to_save)
# exit(0)
with open("machines_part_three.json", "w") as out_f:
json.dump(obj=clusters_to_save, fp=out_f, sort_keys=True, indent=4)
def part_six(env):
#
with open("machines_part_three.json") as json_file:
cluster_best_machine_mapper_str_key = json.load(json_file)
ololo_mapping = {}
ololo_to_sort = []
for key in cluster_best_machine_mapper_str_key:
tuple_key = key
tuple_key = tuple_key.replace("(", "")
tuple_key = tuple_key.replace(")", "")
tuple_key = tuple(map(eval, tuple_key.split(",")))
ololo_mapping[tuple_key] = MachineStored.from_dict(
cluster_best_machine_mapper_str_key[key]["graph_dict"],
env=env)
ololo_to_sort.append([
cluster_best_machine_mapper_str_key[key]["total_reward"],
tuple_key
])
best_clusters = {}
for i in sorted(ololo_to_sort, reverse=True):
key = i[1]
print(key, type(key), key[0])
# print(ololo_mapping[key])
total_reward_a = 0
for i in range(10):
params = HAMParamsCommon(env)
to_run = {}
ss = {**best_clusters, key: ololo_mapping[key]}
for i in ss:
to_run[i] = ss[i].get_machine()
runner(
ham=AutoMachineSimple(env),
num_episodes=800,
env=env,
params=params,
on_model_mapping=to_run,
)
total_reward_a += sum(params.logs["ep_rewards"])
total_reward_b = 0
for i in range(10):
to_run = {}
ss = {**best_clusters}
for i in ss:
to_run[i] = ss[i].get_machine()
to_run = {}
params = HAMParamsCommon(env)
runner(
ham=AutoMachineSimple(env),
num_episodes=800,
env=env,
params=params,
on_model_mapping=to_run,
)
total_reward_b += sum(params.logs["ep_rewards"])
if total_reward_a > total_reward_b:
best_clusters[key] = ololo_mapping[key]
print()
print(total_reward_a, " ::: ", total_reward_b)
clusters_to_save = {}
for i in best_clusters:
on_model_part_str = str(i)
clusters_to_save[on_model_part_str] = best_clusters[i].to_dict()
with open("machines_part_six.json", "w") as out_f:
json.dump(obj=clusters_to_save, fp=out_f, sort_keys=True, indent=4)