for data_path in paths:
json_data = util.open_json_result(os.path.join(path, folder, sub['name'], data_path))
if type(json_data) is list:
d = json_data[0]
else:
d = json_data
data[folder][sub['name']].append(d)
#Average and find breakeven points, for final series.
breakeven_points = {}
for folder in folders:
breakeven_points[folder] = []
for sub in sub_folders:
pr_avg = util.average(data[folder][sub['name']], pr_key_y, pr_key_x)
breakeven = util.find_breakeven(pr_avg, samples=4)
breakeven_points[folder].append({"x": sub['value'], "y": breakeven[1]})
print breakeven_points
#series.append({"name": folder, "data": pr_avg})
#
#Summary figure for MSE loss
loss_points = {}
for folder in folders:
loss_points[folder] = []
for sub in sub_folders:
loss_avg = util.average(data[folder][sub['name']], 'events', lc_key_x)
last_loss = loss_avg[-1][lc_key_y]
loss_points[folder].append({"x": sub['value'], "y": last_loss})
for folder in folders:
paths = os.listdir(os.path.join(path, folder, sub_folder))
nr_tests += len(paths)
print("Folder", folder, "length", len(paths))
all_tests.append(paths)
data[folder] = []
for t in range(len(all_tests)):
for data_path in all_tests[t]:
json_data = util.open_json_result(os.path.join(path, folders[t], sub_folder, data_path))
if type(json_data) is list:
d = json_data[0]
else:
d = json_data
data[folders[t]].append(d)
series = []
for folder in folders:
pr_avg = util.average(data[folder], pr_key_y, pr_key_x)
breakeven_points = util.find_breakeven(pr_avg, samples=30)
print(folder, breakeven_points)
series.append({"name": folder, "data": pr_avg, "breakeven": breakeven_points})
util.display_precision_recall_plot(series)
series = []
for folder in folders:
loss_avg = util.average(data[folder], 'events', lc_key_x)
print(folder, loss_avg[-1]['test_loss'])
series.append({"name": folder, "data": loss_avg, "y_key": lc_key_y})
util.display_loss_curve_plot(series)
data[folders[t]].append(d)
series = []
manual_breakeven = [
[0.513, 0.595, 0.687, 0.712], [0.337, 0.411, 0.462, 0.524]
] #Very uneven curves is hard to approximate by polyfit. (finds breakeven automatically)
compare_series = []
for j, pr in enumerate(folders[1:]):
pr_per_epoch = []
for i, curve in enumerate(data[pr]):
samples = 10
if i < len(manual_breakeven[j]):
breakeven_points = [0, manual_breakeven[j][i]]
else:
breakeven_points = util.find_breakeven(curve[pr_key_y],
samples=samples)
print(folder, breakeven_points)
name = (pr_epoch * i) + 5
series.append({
"name": "Epoch " + str(name),
"data": curve[pr_key_y],
"breakeven": breakeven_points
})
pr_per_epoch.append({'epoch': name, 'breakeven': breakeven_points[-1]})
compare_series.append({
'name': pr,
"data": pr_per_epoch,
'y_key': 'breakeven'
})
util.display_precision_recall_plot(series)
else:
d = json_data
data[folders[t]].append(d)
series = []
manual_breakeven = [
[0.513, 0.595, 0.687, 0.712],
[0.337, 0.411, 0.462, 0.524],
] # Very uneven curves is hard to approximate by polyfit. (finds breakeven automatically)
compare_series = []
for j, pr in enumerate(folders[1:]):
pr_per_epoch = []
for i, curve in enumerate(data[pr]):
samples = 10
if i < len(manual_breakeven[j]):
breakeven_points = [0, manual_breakeven[j][i]]
else:
breakeven_points = util.find_breakeven(curve[pr_key_y], samples=samples)
print(folder, breakeven_points)
name = (pr_epoch * i) + 5
series.append({"name": "Epoch " + str(name), "data": curve[pr_key_y], "breakeven": breakeven_points})
pr_per_epoch.append({"epoch": name, "breakeven": breakeven_points[-1]})
compare_series.append({"name": pr, "data": pr_per_epoch, "y_key": "breakeven"})
util.display_precision_recall_plot(series)
series = []
loss_avg = util.average(data[folders[0]], "events", lc_key_x)
series.append({"name": folders[0], "data": loss_avg, "y_key": lc_key_y})
util.display_two_axis_plot(series, compare_series)
data[folder] = []
for t in range(len(all_tests)):
for data_path in all_tests[t]:
json_data = util.open_json_result(
os.path.join(path, folders[t], sub_folder, data_path))
if type(json_data) is list:
d = json_data[0]
else:
d = json_data
data[folders[t]].append(d)
series = []
for folder in folders:
pr_avg = util.average(data[folder], pr_key_y, pr_key_x)
breakeven_points = util.find_breakeven(pr_avg, samples=30)
print(folder, breakeven_points)
series.append({
"name": folder,
"data": pr_avg,
"breakeven": breakeven_points
})
util.display_precision_recall_plot(series)
series = []
for folder in folders:
loss_avg = util.average(data[folder], 'events', lc_key_x)
print(folder, loss_avg[-1]['test_loss'])
series.append({"name": folder, "data": loss_avg, "y_key": lc_key_y})
util.display_loss_curve_plot(series)
