def run(self):
folders = how_many_fatherFolder(self.path)
for experiemnt in folders:
logging.debug("Folder under analysis -> " + str(experiemnt))
second_path = self.path + experiemnt + "/"
res = how_many_folder(second_path)
num_folder = len(res)
logging.debug("Folder to analise -> " + str(num_folder))
for el in res:
path_here = second_path + str(el) + "/"
names = []
for i in os.listdir(path_here):
if os.path.isfile(
os.path.join(path_here, i)
) and 'trajectory-generate-aSs-' in i and ".zip" in i:
names.append(i)
names = sorted_nicely(names)
pops = Populations()
# find the trajectories ID and Points
trajectories = self.read_trajectory_info(path_here +
"trajectory.zip")
for tra in trajectories:
pops.add_population(Population(tra))
number_of_trajectories = pops.get_number_trajectories()
total_distances = []
numb = 0
logging.debug("Analysing Trajectories...")
for i in tqdm.tqdm(range(len(names))):
name = names[i]
# obtain info from the file
individuals = self.read_info(path_here + name)
# store the msd per trajectory
distance_per_trajectories = {}
for j in range(number_of_trajectories):
distances = []
for indiv in individuals:
if indiv.trajectoryID == pops.get_population(
j).tra.trajectoryID:
distances.append(indiv.MSD)
array = np.array(distances)
MSD = (np.sum(array)) / len(array)
distance_per_trajectories.update({j: MSD})
total_distances.append(distance_per_trajectories)
self.print_graph(total_distances, path_here)
path = "/Volumes/TheMaze/TuringLearning/march/singleLSTM/"
folders = how_many_fatherFolder(path)
if "Figure_1.png" in folders:
folders.remove("Figure_1.png")
if "Figure_2.png" in folders:
folders.remove("Figure_2.png")
i = 0
for experiemnt in folders:
logging.debug("Folder under analysis -> " + str(experiemnt))
first_path = path + experiemnt + "/"
res = how_many_folder(first_path)
missing = []
if len(res) != 5:
if 3 not in res:
missing.append(3)
if 4 not in res:
missing.append(4)
if 5 not in res:
missing.append(5)
if 6 not in res:
missing.append(6)
if len(missing) > 0:
logging.debug("...............Folder missing -> " + str(missing))
for el in res:
if __name__ == "__main__":
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.DEBUG)
first_path = "/Users/alessandrozonta/Desktop/t/"
folders = how_many_fatherFolder(first_path)
folders = ["Experiment-cc"]
for experiemnt in folders:
logging.debug("Folder under analysis -> " + str(experiemnt))
second_path = first_path + experiemnt + "/"
res = how_many_folder(second_path)
num_folder = len(res)
logging.debug("Folder to analise -> " + str(num_folder))
for el in res:
path = second_path + str(el) + "/classifier.log"
with open(path) as f:
content = f.readlines()
real_point_line = []
interesting_list = []
for el in content:
if "Output LSTM transformed" in el:
def total_graph_mse(self):
folders = how_many_fatherFolder(self.path)
folders = [s for s in folders if not re.search('txt', s)]
folders = [s for s in folders if not re.search('jpg', s)]
folders = [s for s in folders if not re.search('png', s)]
dfs = DataFrame(columns=['exp', 'neurones', 'MSD', "div"])
index = 0
for experiemnt in folders:
logging.debug("Folder under analysis -> " + str(experiemnt))
name_experiement = str(experiemnt).replace("Experiment-tgcfs-", "")
second_path = self.path + experiemnt + "/"
res = how_many_folder(second_path)
res = [s for s in res if not re.search('txt', s)]
res = [s for s in res if not re.search('jpg', s)]
res = [s for s in res if not re.search('png', s)]
res = sorted_nicely(res)
num_folder = len(res)
logging.debug("Folder to analise -> " + str(num_folder))
for el in res:
logging.debug("Folder under analysis -> " + str(el))
path_here = second_path + str(el) + "/MSD.txt"
tratt = str(el).replace("neurones", "")
with open(path_here) as f:
content = f.readlines()
div = -1
if "ETH" in name_experiement:
div = 0
elif "G" in name_experiement:
div = 1
elif "I" in name_experiement:
div = 2
d = {
"exp": name_experiement,
"neurones": tratt,
"MSD": float(content[0]),
"div": div
}
df = DataFrame(data=d, index=[index])
dfs = dfs.append(df)
index += 1
sns.factorplot(x="exp",
y="MSD",
hue="neurones",
data=dfs,
kind="bar",
palette="muted")
a = dfs.loc[dfs['div'] == 0]
sns.factorplot(x="exp",
y="MSD",
hue="neurones",
data=a,
kind="bar",
palette="muted")
a = dfs.loc[dfs['div'] == 1]
sns.factorplot(x="exp",
y="MSD",
hue="neurones",
data=a,
kind="bar",
palette="muted")
a = dfs.loc[dfs['div'] == 2]
sns.factorplot(x="exp",
y="MSD",
hue="neurones",
data=a,
kind="bar",
palette="muted")
plt.show()
def run(self):
folders = how_many_fatherFolder(self.path)
folders = [s for s in folders if not re.search('txt', s)]
folders = [s for s in folders if not re.search('jpg', s)]
folders = [s for s in folders if not re.search('png', s)]
for experiemnt in folders:
logging.debug("Folder under analysis -> " + str(experiemnt))
second_path = self.path + experiemnt + "/"
res = how_many_folder(second_path)
folders = [s for s in folders if not re.search('txt', s)]
folders = [s for s in folders if not re.search('jpg', s)]
folders = [s for s in folders if not re.search('png', s)]
num_folder = len(res)
logging.debug("Folder to analise -> " + str(num_folder))
for el in res:
logging.debug("Folder under analysis -> " + str(el))
path_here = second_path + str(el) + "/"
names = []
for i in os.listdir(path_here):
if os.path.isfile(
os.path.join(path_here, i)
) and 'trajectory-generate-aSs-' in i and ".zip" in i:
names.append(i)
names = sorted_nicely(names)
pops = Populations()
# find the trajectories ID and Points
trajectories = self.read_trajectory_info(path_here +
"trajectory.zip")
for tra in trajectories:
pops.add_population(Population(tra))
# analysing the fitness
logging.debug("Analysing the fitness...")
max_agent, max_classifier = self.find_max_values_fitness(
path_here)
agent_generations_info, classifier_generations_info = self.read_fitness(
path_here, max_agent, max_classifier)
x = np.arange(len(agent_generations_info))
y_agent = []
std_agent = []
for element in agent_generations_info:
y_agent.append(element.mean)
std_agent.append(element.std)
y_classifier = []
std_classifier = []
for element in classifier_generations_info:
y_classifier.append(element.mean)
std_classifier.append(element.std)
# print fitnes
self.print_fitnes(x, y_agent, std_agent, y_classifier,
std_classifier, path_here)
total_distances = []
total_distances_msd = []
std_distances = []
last_generations_values = []
logging.debug("Analysing Trajectories...")
for i in tqdm.tqdm(range(len(names))):
name = names[i]
# obtain info from the file
individuals = self.read_info(path_here + name)
if i == len(names) - 1:
for ind in individuals:
for el in ind.array:
last_generations_values.append(el)
msds = []
for ind in individuals:
msds.append(ind.MSD)
total_distances.append(np.mean(np.array(msds)))
std_distances.append(np.std(np.array(msds)))
# store the msd per trajectory
distance_per_trajectories = {}
for j in range(number_of_trajectories):
distances = []
for indiv in individuals:
if indiv.trajectoryID == pops.get_population(
j).tra.trajectoryID:
distances.append(indiv.MSD)
array = np.array(distances)
MSD = (np.sum(array)) / len(array)
distance_per_trajectories.update({j: MSD})
total_distances_msd.append(distance_per_trajectories)
# print graph msd per trajectory
self.print_graph_msd_per_trajectory(total_distances_msd,
path_here)
# print graph total msd
self.print_graph_msd_total(total_distance, std_distances,
path_here)
# save the last value
array = np.array(last_generations_values)
MSD = (np.sum(array)) / len(array)
with open(path_here + "/MSD.txt", "w") as text_file:
text_file.write(str(MSD))
def run(self, one_tra_per_graph, f, n):
folders = how_many_fatherFolder(self.path)
if len(f) > 0:
folders = f
for experiemnt in folders:
logging.debug("Folder under analysis -> " + str(experiemnt))
second_path = first_path + experiemnt + "/"
res = how_many_folder(second_path)
if len(n) > 0:
res = n
num_folder = len(res)
logging.debug("Folder to analise -> " + str(num_folder))
for el in res:
path = second_path + str(el) + "/"
logging.debug("Folder under analysis -> " + str(el))
# check max fitness achiavable
max_fitness = find_max_fitnes(path)
if max_fitness is None:
max_fitness = 500
pops = Populations()
# find the trajectories ID and Points
trajectories = self.read_trajectory_info(path + "trajectory.zip")
for tra in trajectories:
pops.add_population(Population(tra))
pictures = []
for i in os.listdir(path):
if os.path.isfile(os.path.join(path, i)) and "_tmp0" in i and ".png" in i:
pictures.append(i)
# if (len(pictures)) > 700:
# logging.debug("Pictures already available, skipping the folder")
# break
# count how many generation I have and find name files storing it
names = []
for i in os.listdir(path):
if os.path.isfile(os.path.join(path, i)) and self.name_to_check in i and ".zip" in i:
names.append(i)
names = sorted_nicely(names)
if len(names) == 0:
logging.debug("No files to generate")
break
# keep only last pic
if self.printingOnlyLastPic:
names = names[-self.howManyLastPic:]
# load all the info to make the boundaries
logging.debug("Loading Information...")
for i in tqdm.tqdm(range(len(names))):
name = names[i]
individuals = self.read_info(path + name)
for ind in individuals:
pop = pops.get_pop_id(ind.trajectoryID)
pop.add_individual(ind)
logging.debug("Generating Graphs...")
# now I have all the individuals in the populations
if one_tra_per_graph:
name = names[0]
numb = 0
indivi = self.read_info(path + name)
self.print_trajectories_each_in_graph(indivi, pops, numb, max_fitness, path)
else:
for i in tqdm.tqdm(range(len(names))):
name = names[i]
numb = int(name.replace(name_to_check, "").replace(".zip", "").split("-")[0])
indivi = self.read_info(path + name)
self.print_trajectory(indivi, pops, numb, max_fitness, path)
def meanAllAgents(old_path):
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p',
level=logging.DEBUG)
folders = how_many_fatherFolder(old_path)
dff = DataFrame(columns=['exp', 'trajectories', 'MSD'])
iii = 0
for experiemnt in folders:
logging.debug("Folder under analysis -> " + str(experiemnt))
name_experiement = str(experiemnt).replace("Experiment-tgcfs-",
"").replace("-ETH", "")
second_path = old_path + experiemnt + "/"
res = how_many_folder(second_path)
num_folder = len(res)
logging.debug("Folder to analise -> " + str(num_folder))
for el in res:
path = second_path + str(el) + "/"
tratt = str(el)
names = []
for i in os.listdir(path):
if os.path.isfile(
os.path.join(path, i)
) and 'trajectory-generate-aSs-' in i and ".zip" in i:
names.append(i)
names = sorted_nicely(names)
total_distances = []
numb = 0
logging.debug("Analysing Trajectories...")
for i in tqdm.tqdm(range(len(names))):
name = names[i]
numb += 1
# name = "trajectory-generatedPoints-" + str(numb) + "-" + str(numb) + ".zip"
trajectories_label, json_file, id_label = reanInfo(path + name)
# number = 0
# while number < len(id_label):
# real points
lat_real = []
lng_real = []
# generated points
lat_generated = []
lng_generated = []
label_real = []
label_generated = []
for labels in trajectories_label:
for el in json_file[labels]["real"]:
if el[0] not in lat_real:
lat_real.append(el[0])
lng_real.append(el[1])
label_real.append(json_file[labels]["id"])
for el in json_file[labels]["generated"]:
if el[0] not in lat_generated:
lat_generated.append(el[0])
lng_generated.append(el[1])
label_generated.append(json_file[labels]["id"])
distance_per_trajectories = {}
# now for every trajectory compute the distance of the generated distance
for i in range(len(label_real)):
index = [
j for j, x in enumerate(label_generated)
if x == label_real[i]
]
distances = []
for ind in index:
a = np.array((lat_real[i], lng_real[i]))
b = np.array((lat_generated[ind], lng_generated[ind]))
value = np.linalg.norm(a - b) * 100000
value = pow(value, 2)
distances.append(value)
array = np.array(distances)
MSD = (np.sum(array)) / len(array)
distance_per_trajectories.update({i: MSD})
total_distances.append(distance_per_trajectories)
#
# df = DataFrame(columns=['gen', 'tra', 'MSD'])
#
# x = []
# x = np.arange(0, len(total_distances))
# i = 0
# for el in total_distances:
# for k in el.keys():
# d = {"gen": i, "tra": k, "MSD": el[k]}
# dfs = DataFrame(data=d, index=[i])
# df = df.append(dfs)
# i += 1
# sns.set_style("darkgrid")
# df = df[df.columns].astype(float)
# g = sns.lmplot(x="gen", y="MSD", hue="tra", data=df, scatter_kws={"s": 1}, fit_reg=False)
last_line = total_distances[len(total_distances) - 1]
arr = []
for k in last_line.keys():
arr.append(last_line[k])
array = np.array(arr)
MSD = (np.sum(array)) / len(array)
logging.debug(MSD)
dd = {"exp": name_experiement, "trajectories": tratt, "MSD": MSD}
dfss = DataFrame(data=dd, index=[iii])
iii += 1
dff = dff.append(dfss)
# df.plot(x='gen', y='MSD')
# sns.lmplot(x="gen", y="MSD", hue="tra", data=df)
# a = df.loc[df['tra'] == 0]
# ax = a.plot(x='gen', y='MSD', kind='scatter', label="0")
# for i in range(1, 5):
# a = df.loc[df['tra'] == i]
# a.plot(x='gen', y='MSD', ax=ax, kind='scatter',label=i)
# g.set(ylim=(0, 300))
# for j in range(5):
# a = df.loc[df['tra'] == j]
# a.plot(x='gen', y='MSD', ylim=(0,0.00000007))
# plt.figure(0)
# sns.set_style("darkgrid")
# plt.errorbar(x, mean, std)
# plt.errorbar(x, min)
# plt.errorbar(x, max_value)
# # plt.plot(median)
# plt.legend(("mean Difference", "min Difference", "max Difference"))
# plt.xlabel("Generation")
# plt.ylabel("Distance (metres) point generated with real point")
# plt.legend(("Max Distance", "Min Distance", "Median Distance"))
# save_name = path + 'msd.png'
# plt.savefig(save_name, dpi=500, facecolor='w', edgecolor='w', orientation='portrait', papertype=None,
# format=None, transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)
# plt.close()
# logging.debug("Graph saved!")
# os.system("rm movie.mp4")
# os.system("ffmpeg -f image2 -r 2 -i _tmp%05d.png -vcodec mpeg4 -y movie.mp4")
# os.system("rm _tmp*.png")
# logging.debug("End Program")
sns.factorplot(x="exp",
y="MSD",
hue="trajectories",
data=dff,
kind="bar",
palette="muted")
plt.show()