def write_out_ankit_stat(self, submission_csv, ankit_csv, stat_path):
"""Write out stats using ankit's evaluation.
"""
create_folder(os.path.dirname(stat_path))
f = open(stat_path, 'w')
f.write("# --------- Ankitshah009's Evaluation ---------\n")
f.close()
gtDS = FileFormat(ankit_csv)
pdDS = FileFormat(submission_csv)
gtDS.computeMetrics(pdDS, stat_path) # write to stat csv
with open(stat_path, 'rb') as f:
reader = csv.reader(f, delimiter='\t')
lis = list(reader)
def at_visualize(at_prob_mat_path, weak_gt_csv, lbs, out_path):
lb_to_idx = {lb: index for index, lb in enumerate(lbs)}
create_folder(os.path.dirname(out_path))
my_open = get_my_open(weak_gt_csv)
with my_open(weak_gt_csv, 'rt') as f:
reader = csv.reader(f, delimiter=',')
lis = list(reader)
# Get gt_dict
gt_dict = {}
for li in lis:
na = li[0]
value = li[3]
if na not in gt_dict.keys():
gt_dict[na] = [value]
else:
gt_dict[na].append(value)
with gzip.open(at_prob_mat_path, 'rt') as f:
reader = csv.reader(f, delimiter='\t')
lis = list(reader)
# Create debug.csv
f_write = open(out_path, 'w')
for li in lis:
na = li[0]
values = li[1:]
lbs = gt_dict[na]
for lb in lbs:
if lb in lb_to_idx.keys():
index = lb_to_idx[lb]
if not values[index][0] == '*':
values[index] = '*' + values[index]
else:
print(lb, "not key (Please ignore)!")
f_write.write(na)
for i1 in range(len(values)):
f_write.write('\t' + values[i1])
f_write.write('\r\n')
f_write.close()
print("Write out at_visualize.csv successfully!\n")
def write_stat_to_csv(self, stat, stat_path):
"""Write out stat to stat csv.
Args:
stat_path: string, path of stat csv to write out.
"""
create_folder(os.path.dirname(stat_path))
f = open(stat_path, 'w')
f.write('\n')
f.write("\n\n# --------- Event wise evaluation ---------\n")
f.write("\t")
for lb in self.lbs:
f.write("\t" + lb[0:6])
f.write("{0:12}".format("\nthres:"))
for i1 in range(len(self.lbs)):
f.write("\t" + "%s" % stat['thres_ary'][i1])
f.write("{0:12}".format("\ntp:"))
for lb in self.lbs:
f.write("\t" + "%d" % stat['event_wise'][lb]['tp'])
f.write("{0:12}".format("\nfn:"))
for lb in self.lbs:
f.write("\t" + "%d" % stat['event_wise'][lb]['fn'])
f.write("{0:12}".format("\nfp:"))
for lb in self.lbs:
f.write("\t" + "%d" % stat['event_wise'][lb]['fp'])
f.write("{0:12}".format("\ntn:"))
for lb in self.lbs:
f.write("\t" + "%d" % stat['event_wise'][lb]['tn'])
f.write("{0:12}".format("\nprecision:"))
for lb in self.lbs:
f.write("\t" + "%.3f" % stat['event_wise'][lb]['precision'])
f.write("{0:12}".format("\nrecall:"))
for lb in self.lbs:
f.write("\t" + "%.3f" % stat['event_wise'][lb]['recall'])
f.write("{0:12}".format("\nf_value:"))
for lb in self.lbs:
f.write("\t" + "%.3f" % stat['event_wise'][lb]['f_value'])
f.write(
"\n\n# --------- Total evaluation (Mix all events together) ---------"
)
f.write("{0:16}".format("\ntotal tp:") +
"\t%d" % stat['overall']['tp'])
f.write("{0:16}".format("\ntotal fn:") +
"\t%d" % stat['overall']['fn'])
f.write("{0:16}".format("\ntotal fp:") +
"\t%d" % stat['overall']['fp'])
f.write("{0:16}".format("\ntotal tn:") +
"\t%d" % stat['overall']['tn'])
f.write("\n")
f.write("{0:18}".format("\ntotal precision:") +
"\t%.3f" % stat['overall']['precision'])
f.write("{0:18}".format("\ntotal recall:") +
"\t%.3f" % stat['overall']['recall'])
f.write("{0:18}".format("\ntotal f_value:") +
"\t%.3f" % stat['overall']['f_value'])
f.write("{0:18}".format("\ntotal error_rate:") +
"\t%.3f" % stat['overall']['error_rate'])
f.write("\n")
f.close()
print("Write out stat to", stat_path, "successfully!")
def write_stat_to_csv(self, stat, stat_path, mode='w'):
"""Write out stats.
"""
create_folder(os.path.dirname(stat_path))
f = open(stat_path, mode)
f.write('\n')
f.write("\n\n# --------- Event wise evaluation ---------\n")
f.write("\t")
for lb in self.lbs:
f.write("\t" + lb[0:6])
f.write("{0:12}".format("\nthres:"))
for i1 in range(len(self.lbs)):
f.write("\t" + "%s" % stat['thres_ary'][i1])
f.write("{0:12}".format("\ntp:"))
for lb in self.lbs:
f.write("\t" + "%d" % stat['event_wise'][lb]['tp'])
f.write("{0:12}".format("\nfn:"))
for lb in self.lbs:
f.write("\t" + "%d" % stat['event_wise'][lb]['fn'])
f.write("{0:12}".format("\nfp:"))
for lb in self.lbs:
f.write("\t" + "%d" % stat['event_wise'][lb]['fp'])
f.write("{0:12}".format("\ntn:"))
for lb in self.lbs:
f.write("\t" + "%d" % stat['event_wise'][lb]['tn'])
f.write("{0:12}".format("\nprecision:"))
for lb in self.lbs:
f.write("\t" + "%.3f" % stat['event_wise'][lb]['precision'])
f.write("{0:12}".format("\nrecall:"))
for lb in self.lbs:
f.write("\t" + "%.3f" % stat['event_wise'][lb]['recall'])
f.write("{0:12}".format("\nf_value:"))
for lb in self.lbs:
f.write("\t" + "%.3f" % stat['event_wise'][lb]['f_value'])
f.write("{0:12}".format("\neer:"))
for lb in self.lbs:
f.write("\t" + "%.3f" % stat['event_wise'][lb]['eer'])
f.write("{0:12}".format("\nauc:"))
for lb in self.lbs:
f.write("\t" + "%.3f" % stat['event_wise'][lb]['auc'])
f.write("\n")
f.write("{0:12}".format("\nmean precision:") +
"\t%.3f" % self._get_mean_stat(stat, 'precision'))
f.write("{0:12}".format("\nmean recall:") +
"\t%.3f" % self._get_mean_stat(stat, 'recall'))
f.write("{0:12}".format("\nmean f_value:") +
"\t%.3f" % self._get_mean_stat(stat, 'f_value'))
f.write("{0:12}".format("\nmean eer:") +
"\t%.3f" % self._get_mean_stat(stat, 'eer'))
f.write("{0:12}".format("\nmean auc:") +
"\t%.3f" % self._get_mean_stat(stat, 'auc'))
f.write(
"\n\n# --------- Total evaluation (Mix all events together) ---------"
)
f.write("{0:16}".format("\ntotal tp:") +
"\t%d" % stat['overall']['tp'])
f.write("{0:16}".format("\ntotal fn:") +
"\t%d" % stat['overall']['fn'])
f.write("{0:16}".format("\ntotal fp:") +
"\t%d" % stat['overall']['fp'])
f.write("{0:16}".format("\ntotal tn:") +
"\t%d" % stat['overall']['tn'])
f.write("\n")
f.write("{0:18}".format("\ntotal precision:") +
"\t%.3f" % stat['overall']['precision'])
f.write("{0:18}".format("\ntotal recall:") +
"\t%.3f" % stat['overall']['recall'])
f.write("{0:18}".format("\ntotal f_value:") +
"\t%.3f" % stat['overall']['f_value'])
f.write("\n")
f.close()
print("Write out stat to", stat_path, "successfully!")