def get_confusion_matrix(test_im_dir,detection_csv_file, names_file, outname = 'test_temp/confusion_matrix.csv'):
file_bbox = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
bbox = load_bbox(file_bbox,test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, images[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
detection_csv = detection_csv_file
gt_df = pd.DataFrame(ground_truth, columns=('root_image','obj_class', 'x', 'y', 'w', 'h'))
gt_df['obj_class'] = gt_df.obj_class.astype(str)
dt_df = pd.read_csv(detection_csv)
######################### convert classes according .names file ###################
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
##########################
dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())]
dt_df = dt_df.sort_values(['obj_class'])
dt_df['correct'] = 0
gt_df = gt_df.sort_values(['obj_class'])
gt_df['used'] = 0
########################### generating template dict for per classes metrics ###############
# get all classes (detected and GT)
all_classes = []
for dift_class in dt_df.obj_class.unique():
all_classes.append(dift_class)
for dift_class in gt_df.obj_class.unique():
if dift_class not in all_classes:
all_classes.append(dift_class)
all_classes = sorted(all_classes)
template_dic = {}
for dift_class_x in all_classes :
template_dic[dift_class_x] = {}
for dift_class_y in all_classes :
template_dic[dift_class_x][dift_class_y] = 0
####################### comparison of detected objects to GT #################################
# overlap threshold for acceptance
overlap_threshold = 0.5
for image in dt_df.root_image.unique():
dt_df_img = dt_df[dt_df['root_image'] == image]
gt_df_img = gt_df[gt_df['root_image'] == image]
for rowA in gt_df_img.iterrows():
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
classA = rowA[1].obj_class
for rowB in dt_df_img.iterrows(): # ... with every detected object of this class
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
classB = rowB[1].obj_class
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > overlap_threshold : # i.e. if detection and GT overlap
template_dic[classA][classB] += 1
#template_dic[classB][classA] += 1
matrix_df = pd.DataFrame(template_dic)
#print(matrix_df)
matrix_df.to_csv(outname)
def get_metrics(test_im_dir,detection_csv_file, names_file, examples_file_path, print_res=True, edit_res=False):
################### function to get average precision ################################
def voc_ap(rec, prec):
# from https://github.com/Cartucho/mAP/blob/master/main.py
"""
--- Official matlab code VOC2012---
mrec=[0 ; rec ; 1];
mpre=[0 ; prec ; 0];
for i=numel(mpre)-1:-1:1
mpre(i)=max(mpre(i),mpre(i+1));
end
i=find(mrec(2:end)~=mrec(1:end-1))+1;
ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
"""
rec.insert(0, 0.0) # insert 0.0 at begining of list
rec.append(1.0) # insert 1.0 at end of list
mrec = rec[:]
prec.insert(0, 0.0) # insert 0.0 at begining of list
prec.append(0.0) # insert 0.0 at end of list
mpre = prec[:]
"""
This part makes the precision monotonically decreasing
(goes from the end to the beginning)
matlab: for i=numel(mpre)-1:-1:1
mpre(i)=max(mpre(i),mpre(i+1));
"""
# matlab indexes start in 1 but python in 0, so I have to do:
# range(start=(len(mpre) - 2), end=0, step=-1)
# also the python function range excludes the end, resulting in:
# range(start=(len(mpre) - 2), end=-1, step=-1)
for i in range(len(mpre) - 2, -1, -1):
mpre[i] = max(mpre[i], mpre[i + 1])
"""
This part creates a list of indexes where the recall changes
matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1;
"""
i_list = []
for i in range(1, len(mrec)):
if mrec[i] != mrec[i - 1]:
i_list.append(i) # if it was matlab would be i + 1
"""
The Average Precision (AP) is the area under the curve
(numerical integration)
matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
"""
ap = 0.0
for i in i_list:
ap += ((mrec[i] - mrec[i - 1]) * mpre[i])
return ap, mrec, mpre
###########################################################################################################
######################## Formating both detection and GT data into same template ##########################
file_bbox = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
# sort only images with labels and labels attached to images
image_labelled = []
for i in range(len(file_bbox)):
root_name_bbx = file_bbox[i].split('.')[0]
for image in images:
if image.split('.')[0] == root_name_bbx:
image_labelled.append(image)
image_labelled = sorted(image_labelled)
labels_with_images = []
for i in range(len(image_labelled)):
root_name_image = image_labelled[i].split('.')[0]
for label in file_bbox:
if label.split('.')[0] == root_name_image:
labels_with_images.append(label)
labels_with_images = sorted(labels_with_images)
bbox = load_bbox(labels_with_images, test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, images[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
detection_csv = detection_csv_file
# gt_df stores GT information
# dt_df stores detection information
gt_df = pd.DataFrame(ground_truth, columns=('root_image','obj_class', 'x', 'y', 'w', 'h'))
dt_df = pd.read_csv(detection_csv)
######################### convert classes according .names file ###################
gt_df['obj_class'] = gt_df.obj_class.astype(str)
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
################## compare only images that have associated GT ####################
dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())]
################## get the list of all classes (detected of not) ####################
class_list = []
max_name_class_length = 0
for dift_class in gt_df.obj_class.unique():
class_list.append(dift_class)
# included by Dom
max_name_class_length = max(max_name_class_length, len(dift_class))
# end of included by Dom
for dift_class in dt_df.obj_class.unique():
if dift_class not in class_list:
class_list.append(dift_class)
########################### generating template dict for per classes metrics ###############
mAP_class_dict = {}
prec_class_dict = {}
for dift_class in class_list :
mAP_class_dict[dift_class] = 0
prec_class_dict[dift_class] = 0
####################### initialize variables #############################
dt_df = dt_df.sort_values(['obj_class'])
dt_df['correct'] = 0
dt_df['precision'] = 0.0
dt_df['recall'] = 0.0
gt_df = gt_df.sort_values(['obj_class'])
gt_df['used'] = 0
# overlap threshold for acceptance
overlap_threshold = 0.5
####################### comparison of detected objects to GT #################################
for image in dt_df.root_image.unique():
print(image)
dt_df_img = dt_df[dt_df['root_image'] == image]
gt_df_img = gt_df[gt_df['root_image'] == image]
# list different classes present in GT
gt_classes = gt_df_img.obj_class.unique()
# list out wrong predicted classes
dt_df_correct_classes = dt_df_img[dt_df_img['obj_class'].isin(gt_classes)]
# for each correct class
for dift_class in gt_classes:
gt_df_class = gt_df_img[gt_df_img.obj_class == dift_class]
dt_df_class = dt_df_correct_classes[dt_df_correct_classes.obj_class == dift_class]
for rowA in gt_df_class.iterrows(): # compare each GT object of this class ...
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
used = rowA[1].used
for rowB in dt_df_class.iterrows(): # ... with every detected object of this class
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > overlap_threshold : # i.e. if detection and GT overlap
if used == 0: # gt not found yet
indexB, dataB = rowB
dt_df.at[indexB, 'correct'] = 1
indexA, dataA = rowA
gt_df.at[indexA, 'used'] = 1
df_TP = dt_df[dt_df.correct == 1]
df_FP = dt_df[dt_df.correct == 0]
df_FN = gt_df[gt_df.used == 0]
TP = len(df_TP.correct)
FP = len(df_FP.correct)
FN = len(df_FN.used)
precision_general = float(TP / (TP + FP))
recall_general = float(TP / (TP + FN))
if float((precision_general + recall_general)) != 0:
F1_general = 2 * float((precision_general * recall_general) / (precision_general + recall_general))
def predation_statistics(detections_csv_file):
df = pd.read_csv(detections_csv_file)
# generating template for result df
with open('cfg/insects_detailed_predation.names') as f:
labels = [line.rstrip('\n') for line in f]
all_preys = []
all_predators = []
for label in labels:
if label.split(' ')[1] == 'prey':
all_preys.append(label.split(' ')[0])
if label.split(' ')[1] == 'predator':
all_predators.append(label.split(' ')[0])
preys = []
predators = []
for dift_class in df.obj_class.unique():
if dift_class in all_preys:
preys.append(dift_class)
if dift_class in all_predators:
predators.append(dift_class)
dic_of_dic = {}
for prey in preys:
class_dic = {}
for dift_class in predators:
class_dic[dift_class] = []
dic_of_dic[prey] = class_dic
#
for image in df.root_image.unique():
df_image = df[df.root_image == image]
preys_img = []
pred_img = []
for classe in df_image.obj_class.unique():
if classe in preys:
preys_img.append(classe)
for classe in df_image.obj_class.unique():
if classe in predators:
pred_img.append(classe)
for classe_prey in preys_img:
df_prey = df_image[df_image.obj_class == classe_prey]
for rowA in df_prey.iterrows():
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
for classe_pred in pred_img:
used = False
count = 0
df_pred = df_image[df_image.obj_class == classe_pred]
for rowB in df_pred.iterrows():
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > 0:
used = True
count += 1
if used:
dic_of_dic[classe_prey][classe_pred].append(count)
# get number of interactions between each prey and pred
int_num_dic = {}
for prey in preys:
class_dic = {}
for dift_class in predators:
class_dic[dift_class] = 0
int_num_dic[prey] = class_dic
for prey in preys:
for pred in predators:
int_num_dic[prey][pred] = len(dic_of_dic[prey][pred])
number_df = pd.DataFrame(int_num_dic)
#print(number_df)
number_df.to_csv('results/predation_event_numbers.csv')
# get mean predator number per predation
int_num_dic = {}
for prey in preys:
class_dic = {}
for dift_class in predators:
class_dic[dift_class] = 0
int_num_dic[prey] = class_dic
for prey in preys:
for pred in predators:
if len(dic_of_dic[prey][pred]) != 0:
mean = float(
float(sum(dic_of_dic[prey][pred])) /
float(len(dic_of_dic[prey][pred])))
int_num_dic[prey][pred] = mean
else:
int_num_dic[prey][pred] = 'Na'
mean_df = pd.DataFrame(int_num_dic)
#print(mean_df)
mean_df.to_csv('results/mean_pred_per_prey.csv')
# get max predator number per predation
int_num_dic = {}
for prey in preys:
class_dic = {}
for dift_class in predators:
class_dic[dift_class] = 0
int_num_dic[prey] = class_dic
for prey in preys:
for pred in predators:
if len(dic_of_dic[prey][pred]) != 0:
max_pred = max(dic_of_dic[prey][pred])
int_num_dic[prey][pred] = max_pred
else:
int_num_dic[prey][pred] = 'Na'
max_df = pd.DataFrame(int_num_dic)
#print(max_df)
max_df.to_csv('results/max_pred_per_prey.csv')
# get mode predator number per predation
int_num_dic = {}
for prey in preys:
class_dic = {}
for dift_class in predators:
class_dic[dift_class] = 0
int_num_dic[prey] = class_dic
for prey in preys:
for pred in predators:
if len(dic_of_dic[prey][pred]) != 0:
mode_pred = max(set(dic_of_dic[prey][pred]),
key=dic_of_dic[prey][pred].count)
int_num_dic[prey][pred] = mode_pred
else:
int_num_dic[prey][pred] = 'Na'
mode_df = pd.DataFrame(int_num_dic)
#print(mode_df)
mode_df.to_csv('results/mode_pred_per_prey.csv')
def count_interactions_detection(detections_csv_file,
print_res=False,
edit_res=False,
outname='results/object_inter.csv'):
df = pd.read_csv(detections_csv_file)
# generating template for result df
list_of_list = []
for image in df.root_image.unique():
df_image = df[df.root_image == image]
date = df_image['date'].iloc[0]
for combo in combinations(df_image.obj_class.unique(), 2):
#print(combo)
list = [image, date, combo[0], combo[1], 0]
list_of_list.append(list)
df_inter = pd.DataFrame(list_of_list,
columns=('root_image', 'date', 'classA', 'classB',
'count'))
#
for image in df.root_image.unique():
df_image = df[df.root_image == image]
df_inter_image = df_inter[df_inter.root_image == image]
for classe in df_image.obj_class.unique():
df_class = df_image[df_image.obj_class == classe]
df_not_class = df_image[df_image.obj_class != classe]
#print(classe)
for rowA in df_class.iterrows():
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
classA = rowA[1].obj_class
for rowB in df_not_class.iterrows():
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
classB = rowB[1].obj_class
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > 0:
df_inter_bis = df_inter_image[df_inter_image.classA ==
classA]
df_inter_ter = df_inter_bis[df_inter_bis.classB ==
classB]
if len(df_inter_ter.classA) == 0:
df_inter_bis = df_inter_image[df_inter_image.classA
== classB]
df_inter_ter = df_inter_bis[df_inter_bis.classB ==
classA]
index_to_change = df_inter_ter.index[0]
df_inter.at[index_to_change, 'count'] += 1
df_inter.loc[:, 'count'] /= 2
#pd.to_numeric(df_inter.loc[:,'count'], downcast='integer')
df_inter = df_inter.sort_values(['root_image'])
if print_res:
print(df_inter)
if edit_res:
df_inter.to_csv(outname, index=False)
def count_intra_gt(test_im_dir,
names_file,
print_res=False,
edit_res=False,
outname='results/object_intra_gt.csv'):
# clean data preparation : remove images without labels, txt files that are no label file ...
file_bbox = sorted(
[f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
image_labelled = []
for i in range(len(file_bbox)):
root_name_bbx = file_bbox[i].split('.')[0]
for image in images:
if image.split('.')[0] == root_name_bbx:
image_labelled.append(image)
image_labelled = sorted(image_labelled)
labels_with_images = []
for i in range(len(image_labelled)):
root_name_image = image_labelled[i].split('.')[0]
for label in file_bbox:
if label.split('.')[0] == root_name_image:
labels_with_images.append(label)
labels_with_images = sorted(labels_with_images)
bbox = load_bbox(labels_with_images, test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, image_labelled[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
gt_df = pd.DataFrame(ground_truth,
columns=('root_image', 'obj_class', 'x', 'y', 'w',
'h'))
gt_df['root_image_path'] = 'test_images/' + gt_df['root_image'] + '.JPG'
gt_df['obj_class'] = gt_df.obj_class.astype(str)
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
df = gt_df # I have only copied the previous function, now that df are of the same format
# generating template for result df
list_of_list = []
for image in df.root_image_path.unique():
date = get_date_taken(image)
df_image = df[df.root_image_path == image]
for obj_class in df_image.obj_class.unique():
list = [image, date, obj_class, 0]
list_of_list.append(list)
df_inter = pd.DataFrame(list_of_list,
columns=('root_image_path', 'date', 'obj_class',
'count'))
#
for image in df.root_image_path.unique():
date = get_date_taken(image)
df_image = df[df.root_image_path == image]
df_inter_image = df_inter[df_inter.root_image_path == image]
for classe in df_image.obj_class.unique():
df_class = df_image[df_image.obj_class == classe]
for rowA in df_class.iterrows():
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
classA = rowA[1].obj_class
df_comp = df_class.drop([rowA[0]])
for rowB in df_comp.iterrows():
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
classB = rowB[1].obj_class
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > 0:
df_inter_bis = df_inter_image[df_inter_image.obj_class
== classA]
index_to_change = df_inter_bis.index[0]
df_inter.at[index_to_change, 'count'] += 1
df_inter.loc[:, 'count'] /= 2
#pd.to_numeric(df_inter.loc[:,'count'], downcast='integer')
df_inter = df_inter.sort_values(['root_image_path'])
if print_res:
print(df_inter)
if edit_res:
df_inter.to_csv(outname, index=False)
def get_metrics(test_im_dir,
detection_csv_file,
names_file,
examples_file_path,
print_res=True,
edit_res=False):
################### function to get average precision ################################
def voc_ap(rec, prec):
# from https://github.com/Cartucho/mAP/blob/master/main.py
"""
--- Official matlab code VOC2012---
mrec=[0 ; rec ; 1];
mpre=[0 ; prec ; 0];
for i=numel(mpre)-1:-1:1
mpre(i)=max(mpre(i),mpre(i+1));
end
i=find(mrec(2:end)~=mrec(1:end-1))+1;
ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
"""
rec.insert(0, 0.0) # insert 0.0 at begining of list
rec.append(1.0) # insert 1.0 at end of list
mrec = rec[:]
prec.insert(0, 0.0) # insert 0.0 at begining of list
prec.append(0.0) # insert 0.0 at end of list
mpre = prec[:]
"""
This part makes the precision monotonically decreasing
(goes from the end to the beginning)
matlab: for i=numel(mpre)-1:-1:1
mpre(i)=max(mpre(i),mpre(i+1));
"""
# matlab indexes start in 1 but python in 0, so I have to do:
# range(start=(len(mpre) - 2), end=0, step=-1)
# also the python function range excludes the end, resulting in:
# range(start=(len(mpre) - 2), end=-1, step=-1)
for i in range(len(mpre) - 2, -1, -1):
mpre[i] = max(mpre[i], mpre[i + 1])
"""
This part creates a list of indexes where the recall changes
matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1;
"""
i_list = []
for i in range(1, len(mrec)):
if mrec[i] != mrec[i - 1]:
i_list.append(i) # if it was matlab would be i + 1
"""
The Average Precision (AP) is the area under the curve
(numerical integration)
matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
"""
ap = 0.0
for i in i_list:
ap += ((mrec[i] - mrec[i - 1]) * mpre[i])
return ap, mrec, mpre
###########################################################################################################
######################## Formating both detection and GT data into same template ##########################
file_bbox = sorted(
[f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
# sort only images with labels and labels attached to images
image_labelled = []
for i in range(len(file_bbox)):
root_name_bbx = file_bbox[i].split('.')[0]
for image in images:
if image.split('.')[0] == root_name_bbx:
image_labelled.append(image)
image_labelled = sorted(image_labelled)
labels_with_images = []
for i in range(len(image_labelled)):
root_name_image = image_labelled[i].split('.')[0]
for label in file_bbox:
if label.split('.')[0] == root_name_image:
labels_with_images.append(label)
labels_with_images = sorted(labels_with_images)
bbox = load_bbox(labels_with_images, test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, images[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
detection_csv = detection_csv_file
# gt_df stores GT information
# dt_df stores detection information
gt_df = pd.DataFrame(ground_truth,
columns=('root_image', 'obj_class', 'x', 'y', 'w',
'h'))
dt_df = pd.read_csv(detection_csv)
######################### convert classes according .names file ###################
gt_df['obj_class'] = gt_df.obj_class.astype(str)
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
################## compare only images that have associated GT ####################
dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())]
################## get the list of all classes (detected of not) ####################
class_list = []
max_name_class_length = 0
for dift_class in gt_df.obj_class.unique():
class_list.append(dift_class)
# included by Dom
max_name_class_length = max(max_name_class_length, len(dift_class))
# end of included by Dom
for dift_class in dt_df.obj_class.unique():
if dift_class not in class_list:
class_list.append(dift_class)
########################### generating template dict for per classes metrics ###############
mAP_class_dict = {}
prec_class_dict = {}
for dift_class in class_list:
mAP_class_dict[dift_class] = 0
prec_class_dict[dift_class] = 0
####################### initialize variables #############################
dt_df = dt_df.sort_values(['obj_class'])
dt_df['correct'] = 0
dt_df['precision'] = 0.0
dt_df['recall'] = 0.0
gt_df = gt_df.sort_values(['obj_class'])
gt_df['used'] = 0
# overlap threshold for acceptance
overlap_threshold = 0.5
####################### comparison of detected objects to GT #################################
for image in dt_df.root_image.unique():
print(image)
dt_df_img = dt_df[dt_df['root_image'] == image]
gt_df_img = gt_df[gt_df['root_image'] == image]
# list different classes present in GT
gt_classes = gt_df_img.obj_class.unique()
# list out wrong predicted classes
dt_df_correct_classes = dt_df_img[dt_df_img['obj_class'].isin(
gt_classes)]
# for each correct class
for dift_class in gt_classes:
gt_df_class = gt_df_img[gt_df_img.obj_class == dift_class]
dt_df_class = dt_df_correct_classes[dt_df_correct_classes.obj_class
== dift_class]
for rowA in gt_df_class.iterrows(
): # compare each GT object of this class ...
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
used = rowA[1].used
for rowB in dt_df_class.iterrows(
): # ... with every detected object of this class
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > overlap_threshold: # i.e. if detection and GT overlap
if used == 0: # gt not found yet
indexB, dataB = rowB
dt_df.at[indexB, 'correct'] = 1
indexA, dataA = rowA
gt_df.at[indexA, 'used'] = 1
df_TP = dt_df[dt_df.correct == 1]
df_FP = dt_df[dt_df.correct == 0]
df_FN = gt_df[gt_df.used == 0]
TP = len(df_TP.correct)
FP = len(df_FP.correct)
FN = len(df_FN.used)
precision_general = float(TP / (TP + FP))
recall_general = float(TP / (TP + FN))
F1_general = 2 * float((precision_general * recall_general) /
(precision_general + recall_general))
########################### mAP #############################
# once we have for each detection its status (TP or FP), we can compute average precision for each class
# for this we will need to compute recall and precision for each class
# included by Dom
gt_classes = gt_df.obj_class.unique()
nb_classes = len(gt_classes)
df_metrics_classes = pd.DataFrame(pd.np.empty((nb_classes, 10)),
columns=[
'classes', 'precision', 'recall',
'F1', 'mAP', 'GT', 'TP', 'FP', 'FN',
'training_examples'
])
df_metrics_classes['classes'] = gt_classes
df_metrics_classes['precision'] = 0
df_metrics_classes['recall'] = 0
df_metrics_classes['F1'] = 0
df_metrics_classes['mAP'] = 0
df_metrics_classes['GT'] = 0
df_metrics_classes['TP'] = 0
df_metrics_classes['FP'] = 0
df_metrics_classes['FN'] = 0
# end of included by Dom
df_examples = pd.read_csv(examples_file_path, index_col='name')
print('*********')
print(df_examples)
for dift_class in gt_df.obj_class.unique():
gt_df_class = gt_df[gt_df.obj_class == dift_class]
items_to_detect = len(gt_df_class)
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'GT'] = items_to_detect
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'training_examples'] = df_examples.at[
dift_class, 'count']
FN_count = 0
for row in gt_df_class.iterrows():
index, data = row
if (data['used'] == 0):
FN_count += 1
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'FN'] = FN_count
for dift_class in dt_df.obj_class.unique():
gt_df_class = gt_df[gt_df.obj_class == dift_class]
dt_df_class = dt_df[dt_df.obj_class == dift_class]
# GT number of objects to find (potential positives = PP)
PP = len(gt_df_class)
if PP == 0: # if false positive
mAP_class_dict[dift_class] = 0
if PP != 0:
TP_count = 0
FP_count = 0
row_count = 0 # (for recall)
for row in dt_df_class.iterrows():
row_count += 1
index, data = row
if (data['correct'] == 1):
TP_count += 1
else:
FP_count += 1
# if (data['used'] == 0):
# FN_count +=1
recall = float(TP_count / PP)
#print('recall = ', recall)
precision = float(TP_count / row_count)
#print('precision = ', precision)
dt_df_class.at[index, 'recall'] = recall
dt_df_class.at[index, 'precision'] = precision
prec = dt_df_class['precision'].tolist()
rec = dt_df_class['recall'].tolist()
# included by Dom
avg_prec_class = mean(prec)
if avg_prec_class != 0 and recall != 0:
F1_score_class = 2 * float(
(avg_prec_class * recall) / (avg_prec_class + recall))
else:
F1_score_class = 0
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'precision'] = avg_prec_class
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'recall'] = recall
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'F1'] = F1_score_class
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'TP'] = TP_count
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'FP'] = FP_count
# end of included by Dom
ap, mrec, mprec = voc_ap(
rec, prec) # according previously defined function
mAP_class_dict[dift_class] += ap
# this is just a fix for some issues pandas can encounter with .from_dict() function
# but sometimes, it does work perfectly fine.
new_dict = {k: [v] for k, v in mAP_class_dict.items()}
temp_df = pd.DataFrame(new_dict, index=['mAP'])
mAP_class_df = temp_df.T
df_metrics_classes.training_examples = df_metrics_classes.training_examples.astype(
int)
for dift_class in dt_df.obj_class.unique():
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'mAP'] = mAP_class_df.at[dift_class, 'mAP']
# mAP_class_df = pd.DataFrame.from_dict(mAP_class_dict, orient = 'index', columns = ['mAP'])
mAP = float(float(sum(mAP_class_dict.values())) / float(len(class_list)))
if print_res:
print('\n metrics per class_______________________\n')
print(df_metrics_classes)
print('\n general metrics _________________\n')
print('mAP: \t\t%.4f' % mAP)
print('precision: \t%.4f' % precision_general)
print('recall: \t%.4f' % recall_general)
print('F1: \t\t%.4f' % F1_general)
df_metrics_classes.to_csv('test_temp/metrics_per_class.ods',
index=False)
df_metrics_classes.to_csv('test_temp/metrics_per_class.csv',
index=False)
df_general_metrics = pd.DataFrame(
pd.np.empty((1, 4)), columns=['mAP', 'precision', 'recall', 'F1'])
df_general_metrics['mAP'] = mAP
df_general_metrics['precision'] = precision_general
df_general_metrics['recall'] = recall_general
df_general_metrics['F1'] = F1_general
df_general_metrics.to_csv('test_temp/general_metrics.csv', index=False)
df_general_metrics.to_csv('test_temp/general_metrics.ods', index=False)
#################### edit results to csv file
#modified by Dom; variables were not defined; maybe to delete?
if edit_res:
res_list = [['mAP', mAP], ['precision', precision_general],
['recall', recall_general], ['F1', F1_general]]
res_df = pd.DataFrame(res_list, columns=['metric', 'value'])
# print(res_df.T)
# res_df.to_csv('test_temp/general_metrics.csv', index=False)
# res_df.to_csv('test_temp/general_metrics.ods', index=False)
return TP, FP, FN, precision_general, recall_general, F1_general, mAP, mAP_class_dict