def convert_to_html(xml_f):
xpath = os.path.join('xml', xml_f)
l = xml2list(xpath)
list2html(l, f'{xml_f[:-4]}.png', img_d, 'html')
def run_evaluate(predict_dir, target_dir, img_dir=None, simi=False, thres=0):
fp_list = []
classification_p_list = []
total_intersection = 0
total_prediction = 0
total_gt = 0
for predict_f in os.listdir(predict_dir):
predict_path = os.path.join(predict_dir, predict_f)
target_path = os.path.join(target_dir, predict_f)
predict_list = xml2list(predict_path)
target_list = xml2list(target_path)
if img_dir is not None:
img_p = os.path.join(img_dir, predict_f[:-4] + '.png')
img = Image.open(img_p)
for predict in predict_list:
p_cls, p_bb = predict
p_bb = [x - 5 for x in p_bb]
d = ImageDraw.Draw(img)
d.rectangle(p_bb, outline=color_classes[p_cls])
img.save(f'outputs/{predict_f[:-4] + ".png"}')
list_map = match_lists(predict_list, target_list)
tbb_map = {}
for predict in predict_list:
p_cls, p_bb = predict
p_score = 0.1
p_bb = tuple(p_bb)
matched_target = list_map[(p_cls, p_bb, p_score)]
if matched_target is None:
fp_list.append((predict, 'background'))
continue
t, iou = matched_target
t_cls, t_bb = t
t_cls = ICDAR_convert[t_cls]
if t_bb in tbb_map:
tbb_map[t_bb].append(p_bb)
else:
tbb_map[t_bb] = [p_bb]
if p_cls == t_cls:
if iou < thres:
fp_list.append((predict, 'localization'))
continue
fp_list.append((predict, 'correct'))
classification_p_list.append((p_cls, t_cls))
continue
else:
if iou >= thres:
classification_p_list.append((p_cls, t_cls))
sim = False
for s in similar_class_sets:
if p_cls in s and t_cls in s:
sim = True
break
if sim:
if simi:
fp_list.append((predict, 'correct'))
else:
fp_list.append((predict, 'similar'))
else:
fp_list.append((predict, 'other'))
page_intersection = 0
page_prediction = 0
page_gt = 0
for t_bb in tbb_map:
for prediction in tbb_map[t_bb]:
x_left = max(t_bb[0], prediction[0])
y_top = max(t_bb[1], prediction[1])
x_right = min(t_bb[2], prediction[2])
y_bottom = min(t_bb[3], prediction[3])
intersection_area = (x_right - x_left) * (y_bottom - y_top)
page_intersection += intersection_area
page_prediction += (prediction[2] - prediction[0]) * (
prediction[3] - prediction[1])
page_gt += (t_bb[2] - t_bb[0]) * (t_bb[3] - t_bb[1])
total_intersection += page_intersection
total_prediction += page_prediction
total_gt += page_gt
print('Bounding box Precision')
bb_precision = total_intersection / total_prediction
print(bb_precision)
print('--------')
print('Bounding box Recall')
bb_recall = total_intersection / total_gt
print(bb_recall)
print('--------')
print('Bounding box F1')
bb_f1 = 2 * bb_precision * bb_recall / (bb_precision + bb_recall)
print(bb_f1)
print('---------')
class_counts = {}
for p in classification_p_list:
p_cls, t_cls = p
if p_cls not in class_counts:
class_counts[p_cls] = {}
if t_cls in class_counts[p_cls]:
class_counts[p_cls][t_cls] += 1
else:
class_counts[p_cls][t_cls] = 1
class_precisions = {}
all_tp = 0
all_denom = 0
for p_cls in class_counts:
tp = 0
fp = 0
for t_cls in class_counts[p_cls]:
if p_cls == t_cls:
tp = class_counts[p_cls][t_cls]
else:
fp += class_counts[p_cls][t_cls]
denom = tp + fp
all_tp += tp
all_denom += denom
class_precisions[
p_cls] = tp / denom if denom != 0 else 'No false positives or true positives found'
print('All class precision')
all_precision = all_tp / all_denom
print(all_precision)
print('-----------------')
all_tp = 0
all_denom = 0
class_recalls = {}
for p_cls in class_counts:
tp = class_counts[p_cls][p_cls] if p_cls in class_counts[p_cls] else 0
fn = 0
for p2_cls in class_counts:
if p2_cls == p_cls:
continue
if p_cls in class_counts[p2_cls]:
fn += class_counts[p2_cls][p_cls]
denom = tp + fn
all_tp += tp
all_denom += denom
class_recalls[
p_cls] = tp / denom if denom != 0 else 'No false negatives or true positives found'
print('All class recall')
all_recall = all_tp / all_denom
print(all_recall)
print('--------------')
print('All class F1')
all_f1 = 2 * all_precision * all_recall / (all_precision + all_recall)
print(all_f1)
print('--------------')
print('Class recalls')
print(class_recalls)
print('------------')
print('Class precisions')
print(class_precisions)
print('------------')
class_f1 = {}
for cl in class_recalls:
rec = class_recalls[cl]
prec = class_precisions[cl]
if rec + prec == 0:
class_f1[cl] = 0
continue
class_f1[cl] = 2 * rec * prec / (rec + prec)
print('Class F1s')
print(class_f1)
print('-------------')
print('Class counts')
print(class_counts)
df = pd.DataFrame(class_counts)
df = df.fillna(value=0)
df['Total'] = df.sum(axis=1)
print(df[sorted(df.columns)])
print('------------')
tp_num = 0
fp_num = 0
current_class = None
roc_tp = [0]
roc_fp = [0]
p_r_curve = []
for p in fp_list:
predict, category = p
is_tp = category == 'correct'
if is_tp:
if current_class is None:
current_class = True
continue
if not current_class:
roc_tp.append(tp_num)
roc_fp.append(fp_num)
tp_num += 1
else:
if current_class is None:
current_class = False
continue
if current_class:
roc_tp.append(tp_num)
roc_fp.append(fp_num)
fp_num += 1
precision = tp_num / (tp_num + fp_num)
p_r_curve.append((precision, tp_num))
roc_tp.append(tp_num)
roc_fp.append(fp_num)
p_r_curve = [(x, y / tp_num) for x, y in p_r_curve]
max_ps = []
for i in range(11):
chk_num = i / 10
m_p = 0
for x, y in p_r_curve:
if y <= chk_num:
continue
if x > m_p:
m_p = x
max_ps.append(m_p)
mAP = sum(max_ps) / len(max_ps)
uz = list(zip(*p_r_curve))
make_p_r_curve(uz[0], uz[1])
normalized_tp = [x / tp_num for x in roc_tp]
normalized_fp = [x / fp_num for x in roc_fp]
make_roc_chart(normalized_tp, normalized_fp)
filtered_fp_list = [fp for fp in fp_list if fp[1] != 'correct']
print(f'True Positives: {tp_num}')
print(f'False Positives: {fp_num}')
return filtered_fp_list