def plot_2d_stats(stats, name):
for ck in stats:
if ck != "ID":
for ak in stats[ck]:
context_values_keys = sorted(stats[ck][ak].keys())
context_values = dict(zip(context_values_keys, range(len(context_values_keys))))
action_values_keys = sorted(set(flatten(list(map(lambda x: x.keys(), stats[ck][ak].values())))))
action_values = dict(zip(action_values_keys, range(len(action_values_keys))))
ck_ak_stats = np.zeros((len(context_values), len(action_values)))
maximum = 0
for cv in sorted(stats[ck][ak]):
for av in sorted(stats[ck][ak][cv]):
ck_ak_stats[context_values[cv], action_values[av]] = stats[ck][ak][cv][av]["rate"]
maximum = max(maximum, stats[ck][ak][cv][av]["rate"])
plt.imshow(ck_ak_stats.T, interpolation="none")
plt.clim([0, maximum])
if ck_ak_stats.shape[0] > ck_ak_stats.shape[1]:
plt.colorbar(orientation="horizontal")
else:
plt.colorbar(orientation="vertical")
plt.xticks(list(range(len(context_values))), list(context_values_keys), rotation='vertical')
plt.yticks(list(range(len(action_values))), list(action_values_keys))
plt.xlabel(ck)
plt.ylabel(ak)
plt.title("Revenue / show")
dir = "stats/" + name + "/rate_interaction/"
create_directory(dir)
plt.savefig(dir + ck + "-" + ak)
plt.close()
def plot_1d_stats(stats, means, name):
for attr in stats.keys():
if attr != "ID":
attr_values, attr_stat_list = stat_dict_to_list(stats[attr])
_, rate_axis = plt.subplots()
count_axis = rate_axis.twinx()
if isinstance(attr_values[0], str) or isinstance(attr_values[0], int):
ind = np.arange(0, len(attr_values), 1)
rate_axis.bar(ind, attr_stat_list[STATS[2]], width=.8)
count_axis.plot(ind + .4, attr_stat_list[STATS[0]], 'k-')
plt.xticks(ind + .4, tuple(attr_values))
elif isinstance(attr_values[0], float):
rate_axis.scatter(attr_values, attr_stat_list[STATS[2]])
count_axis.plot(attr_values, attr_stat_list[STATS[0]], 'k-')
else:
raise TypeError("Unexpected datatype. Don't know how to plot")
rate_axis.axhline(y=means["rate"])
plt.title("Runid's = " + str(list(data.keys())))
rate_axis.set_ylim(0, 20)
rate_axis.set_ylabel("Avg revenue")
rate_axis.set_xlabel(attr)
count_axis.set_ylim(0, max(attr_stat_list[STATS[0]]) * 1.1)
count_axis.set_ylabel("Occurrences")
dir = "stats/" + name + "/rate/"
create_directory(dir)
plt.savefig(dir + attr)
plt.close()
def save(self):
create_directory("log")
create_directory("agents")
time_str = self.time_str + "_"
np.save('log/' + time_str + self.name, self.data)
np.save('agents/' + time_str + self.agent.name,
self.agent.to_saveable())
def plot_1d_stats(stats, means, name):
for attr in stats.keys():
if attr != "ID":
attr_values, attr_stat_list = stat_dict_to_list(stats[attr])
_, rate_axis = plt.subplots()
count_axis = rate_axis.twinx()
if isinstance(attr_values[0], str) or isinstance(
attr_values[0], int):
ind = np.arange(0, len(attr_values), 1)
rate_axis.bar(ind, attr_stat_list[STATS[2]], width=.8)
count_axis.plot(ind + .4, attr_stat_list[STATS[0]], 'k-')
plt.xticks(ind + .4, tuple(attr_values))
elif isinstance(attr_values[0], float):
rate_axis.scatter(attr_values, attr_stat_list[STATS[2]])
count_axis.plot(attr_values, attr_stat_list[STATS[0]], 'k-')
else:
raise TypeError("Unexpected datatype. Don't know how to plot")
rate_axis.axhline(y=means["rate"])
plt.title("Runid's = " + str(list(data.keys())))
rate_axis.set_ylim(0, 20)
rate_axis.set_ylabel("Avg revenue")
rate_axis.set_xlabel(attr)
count_axis.set_ylim(0, max(attr_stat_list[STATS[0]]) * 1.1)
count_axis.set_ylabel("Occurrences")
dir = "stats/" + name + "/rate/"
create_directory(dir)
plt.savefig(dir + attr)
plt.close()
def info_vs_fitness(dir, filter):
names = {'indi': 'Individuals', 'cxpb': 'Crossover probability', 'mutpb':'Mutation probability', 'gen':'Generations', 'eatype':'EA type', 'set': 'Dataset'}
for info in ['set']: #['eatype', 'indi', 'cxpb', 'mutpb', 'gen']:
xs, ys_hard_best, ys_hard_mean, ys_soft_best, ys_soft_mean = data_info_vs_fitness(dir, filter, info)
create_directory('analyze')
plot_fitness(xs, ys_hard_best, ys_hard_mean, "Hard constraint fitness vs {}".format(names[info]), names[info], "Avg fitness", continuous=info not in ['eatype', 'set'])
matplotlib.pyplot.savefig('analyze/hard_vs_{}'.format(info))
plot_fitness(xs, ys_soft_best, ys_soft_mean, "Soft constraint fitness vs {}".format(names[info]), names[info], "Avg fitness", continuous=info not in ['eatype', 'set'])
matplotlib.pyplot.savefig('analyze/soft_vs_{}'.format(info))
def plot_2d_stats(stats, name):
for ck in stats:
if ck != "ID":
for ak in stats[ck]:
context_values_keys = sorted(stats[ck][ak].keys())
context_values = dict(
zip(context_values_keys, range(len(context_values_keys))))
action_values_keys = sorted(
set(
flatten(
list(
map(lambda x: x.keys(),
stats[ck][ak].values())))))
action_values = dict(
zip(action_values_keys, range(len(action_values_keys))))
ck_ak_stats = np.zeros(
(len(context_values), len(action_values)))
maximum = 0
for cv in sorted(stats[ck][ak]):
for av in sorted(stats[ck][ak][cv]):
ck_ak_stats[
context_values[cv],
action_values[av]] = stats[ck][ak][cv][av]["rate"]
maximum = max(maximum, stats[ck][ak][cv][av]["rate"])
plt.imshow(ck_ak_stats.T, interpolation="none")
plt.clim([0, maximum])
if ck_ak_stats.shape[0] > ck_ak_stats.shape[1]:
plt.colorbar(orientation="horizontal")
else:
plt.colorbar(orientation="vertical")
plt.xticks(list(range(len(context_values))),
list(context_values_keys),
rotation='vertical')
plt.yticks(list(range(len(action_values))),
list(action_values_keys))
plt.xlabel(ck)
plt.ylabel(ak)
plt.title("Revenue / show")
dir = "stats/" + name + "/rate_interaction/"
create_directory(dir)
plt.savefig(dir + ck + "-" + ak)
plt.close()
def main():
args = parser.parse_args()
subfolders = ['real', 'fake']
args_list = []
for subfolder in subfolders:
file_list = [video_label for video_label in os.listdir(os.path.join(args.input, subfolder))]
for video_id in file_list:
input_path = os.path.join(args.input, subfolder, video_id)
output_path = create_directory(os.path.join(args.output, subfolder))
args_list.append((input_path, os.path.join(output_path, video_id)))
args_list = split_list(args_list, 20)
for arg_tuples in args_list:
with concurrent.futures.ProcessPoolExecutor() as executor:
results = [executor.submit(set_fps, input_path, output_path, args.fps)
for input_path, output_path in arg_tuples]
for f in concurrent.futures.as_completed(results):
print(f.result())
def save(self):
create_directory("log")
create_directory("agents")
time_str = self.time_str + "_"
np.save('log/' + time_str + self.name, self.data)
np.save('agents/' + time_str + self.agent.name, self.agent.to_saveable())
def save_data(ea, pop, logbook):
def temp_file(name):
return os.path.join(log_dir, str(random.randint(0, 10 ** 10)) + name)
def write_to_zip_and_remove(temp, path):
zip_file.write(temp, path)
os.remove(temp)
def pickle_save_zip(data, folder, file):
temp = temp_file(file)
path = os.path.join(folder, file)
f = open(temp, 'wb')
pickle.dump(data, f)
f.close()
write_to_zip_and_remove(temp, path)
def save_str(object, folder, file):
temp = temp_file(file)
path = os.path.join(folder, file)
f = open(temp, 'w')
f.write(str(object))
f.close()
write_to_zip_and_remove(temp, path)
def save_readable_population(pop, folder, file):
path = os.path.join(folder, file)
temp = temp_file(file)
f = open(temp, 'w')
pop = sorted(pop, key=lambda x: x.fitness, reverse=True)
for i, ind in enumerate(pop):
f.write("Fitness = {}\n".format(ind.fitness.wvalues))
f.write(str(ind))
f.write("===\n\n")
f.close()
write_to_zip_and_remove(temp, path)
def plot_progress(logbook, parameters, folder, file):
matplotlib.pyplot.clf()
path = os.path.join(folder, file)
temp = temp_file(file)
parameter_str = ", ".join([str(k) + '$=' + str(parameters[k]) + '$' for k in sorted(parameters)])
plot_ea_progress(logbook, parameter_str)
matplotlib.pyplot.savefig(temp)
write_to_zip_and_remove(temp, path)
# Save
print("Saving {}".format(ea.save_name))
log_dir = "logs"
create_directory(log_dir)
name = str(ea.save_name).replace(" ", "_")
zip_file = zipfile.ZipFile(os.path.join(log_dir, name + ".zip"), 'w', zipfile.ZIP_DEFLATED)
pickle_save_zip(logbook, "logbook", "raw.bin")
pickle_save_zip(pop, "pop", "complete.bin")
pickle_save_zip(ea.hof, "pop", "hof.bin")
save_str(logbook, "logbook", "show.txt")
save_str(str(ea), "logbook", "info.json")
save_readable_population(pop, "pop", "show.txt")
plot_progress(logbook.chapters["hard"], ea.jsonify()["ea"], "pop", "plot_hard.png")
plot_progress(logbook.chapters["soft"], ea.jsonify()["ea"], "pop", "plot_soft.png")
zip_file.close()
print("Done saving {}".format(ea.save_name))