def get_metrics(test_im_dir,detection_csv_file, names_file, examples_file_path, print_res=True, edit_res=False):
################### function to get average precision ################################
def voc_ap(rec, prec):
# from https://github.com/Cartucho/mAP/blob/master/main.py
"""
--- Official matlab code VOC2012---
mrec=[0 ; rec ; 1];
mpre=[0 ; prec ; 0];
for i=numel(mpre)-1:-1:1
mpre(i)=max(mpre(i),mpre(i+1));
end
i=find(mrec(2:end)~=mrec(1:end-1))+1;
ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
"""
rec.insert(0, 0.0) # insert 0.0 at begining of list
rec.append(1.0) # insert 1.0 at end of list
mrec = rec[:]
prec.insert(0, 0.0) # insert 0.0 at begining of list
prec.append(0.0) # insert 0.0 at end of list
mpre = prec[:]
"""
This part makes the precision monotonically decreasing
(goes from the end to the beginning)
matlab: for i=numel(mpre)-1:-1:1
mpre(i)=max(mpre(i),mpre(i+1));
"""
# matlab indexes start in 1 but python in 0, so I have to do:
# range(start=(len(mpre) - 2), end=0, step=-1)
# also the python function range excludes the end, resulting in:
# range(start=(len(mpre) - 2), end=-1, step=-1)
for i in range(len(mpre) - 2, -1, -1):
mpre[i] = max(mpre[i], mpre[i + 1])
"""
This part creates a list of indexes where the recall changes
matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1;
"""
i_list = []
for i in range(1, len(mrec)):
if mrec[i] != mrec[i - 1]:
i_list.append(i) # if it was matlab would be i + 1
"""
The Average Precision (AP) is the area under the curve
(numerical integration)
matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
"""
ap = 0.0
for i in i_list:
ap += ((mrec[i] - mrec[i - 1]) * mpre[i])
return ap, mrec, mpre
###########################################################################################################
######################## Formating both detection and GT data into same template ##########################
file_bbox = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
# sort only images with labels and labels attached to images
image_labelled = []
for i in range(len(file_bbox)):
root_name_bbx = file_bbox[i].split('.')[0]
for image in images:
if image.split('.')[0] == root_name_bbx:
image_labelled.append(image)
image_labelled = sorted(image_labelled)
labels_with_images = []
for i in range(len(image_labelled)):
root_name_image = image_labelled[i].split('.')[0]
for label in file_bbox:
if label.split('.')[0] == root_name_image:
labels_with_images.append(label)
labels_with_images = sorted(labels_with_images)
bbox = load_bbox(labels_with_images, test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, images[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
detection_csv = detection_csv_file
# gt_df stores GT information
# dt_df stores detection information
gt_df = pd.DataFrame(ground_truth, columns=('root_image','obj_class', 'x', 'y', 'w', 'h'))
dt_df = pd.read_csv(detection_csv)
######################### convert classes according .names file ###################
gt_df['obj_class'] = gt_df.obj_class.astype(str)
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
################## compare only images that have associated GT ####################
dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())]
################## get the list of all classes (detected of not) ####################
class_list = []
max_name_class_length = 0
for dift_class in gt_df.obj_class.unique():
class_list.append(dift_class)
# included by Dom
max_name_class_length = max(max_name_class_length, len(dift_class))
# end of included by Dom
for dift_class in dt_df.obj_class.unique():
if dift_class not in class_list:
class_list.append(dift_class)
########################### generating template dict for per classes metrics ###############
mAP_class_dict = {}
prec_class_dict = {}
for dift_class in class_list :
mAP_class_dict[dift_class] = 0
prec_class_dict[dift_class] = 0
####################### initialize variables #############################
dt_df = dt_df.sort_values(['obj_class'])
dt_df['correct'] = 0
dt_df['precision'] = 0.0
dt_df['recall'] = 0.0
gt_df = gt_df.sort_values(['obj_class'])
gt_df['used'] = 0
# overlap threshold for acceptance
overlap_threshold = 0.5
####################### comparison of detected objects to GT #################################
for image in dt_df.root_image.unique():
print(image)
dt_df_img = dt_df[dt_df['root_image'] == image]
gt_df_img = gt_df[gt_df['root_image'] == image]
# list different classes present in GT
gt_classes = gt_df_img.obj_class.unique()
# list out wrong predicted classes
dt_df_correct_classes = dt_df_img[dt_df_img['obj_class'].isin(gt_classes)]
# for each correct class
for dift_class in gt_classes:
gt_df_class = gt_df_img[gt_df_img.obj_class == dift_class]
dt_df_class = dt_df_correct_classes[dt_df_correct_classes.obj_class == dift_class]
for rowA in gt_df_class.iterrows(): # compare each GT object of this class ...
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
used = rowA[1].used
for rowB in dt_df_class.iterrows(): # ... with every detected object of this class
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > overlap_threshold : # i.e. if detection and GT overlap
if used == 0: # gt not found yet
indexB, dataB = rowB
dt_df.at[indexB, 'correct'] = 1
indexA, dataA = rowA
gt_df.at[indexA, 'used'] = 1
df_TP = dt_df[dt_df.correct == 1]
df_FP = dt_df[dt_df.correct == 0]
df_FN = gt_df[gt_df.used == 0]
TP = len(df_TP.correct)
FP = len(df_FP.correct)
FN = len(df_FN.used)
precision_general = float(TP / (TP + FP))
recall_general = float(TP / (TP + FN))
if float((precision_general + recall_general)) != 0:
F1_general = 2 * float((precision_general * recall_general) / (precision_general + recall_general))
def get_confusion_matrix(test_im_dir,detection_csv_file, names_file, outname = 'test_temp/confusion_matrix.csv'):
file_bbox = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
bbox = load_bbox(file_bbox,test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, images[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
detection_csv = detection_csv_file
gt_df = pd.DataFrame(ground_truth, columns=('root_image','obj_class', 'x', 'y', 'w', 'h'))
gt_df['obj_class'] = gt_df.obj_class.astype(str)
dt_df = pd.read_csv(detection_csv)
######################### convert classes according .names file ###################
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
##########################
dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())]
dt_df = dt_df.sort_values(['obj_class'])
dt_df['correct'] = 0
gt_df = gt_df.sort_values(['obj_class'])
gt_df['used'] = 0
########################### generating template dict for per classes metrics ###############
# get all classes (detected and GT)
all_classes = []
for dift_class in dt_df.obj_class.unique():
all_classes.append(dift_class)
for dift_class in gt_df.obj_class.unique():
if dift_class not in all_classes:
all_classes.append(dift_class)
all_classes = sorted(all_classes)
template_dic = {}
for dift_class_x in all_classes :
template_dic[dift_class_x] = {}
for dift_class_y in all_classes :
template_dic[dift_class_x][dift_class_y] = 0
####################### comparison of detected objects to GT #################################
# overlap threshold for acceptance
overlap_threshold = 0.5
for image in dt_df.root_image.unique():
dt_df_img = dt_df[dt_df['root_image'] == image]
gt_df_img = gt_df[gt_df['root_image'] == image]
for rowA in gt_df_img.iterrows():
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
classA = rowA[1].obj_class
for rowB in dt_df_img.iterrows(): # ... with every detected object of this class
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
classB = rowB[1].obj_class
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > overlap_threshold : # i.e. if detection and GT overlap
template_dic[classA][classB] += 1
#template_dic[classB][classA] += 1
matrix_df = pd.DataFrame(template_dic)
#print(matrix_df)
matrix_df.to_csv(outname)
def slice_train(orig_dir, slice_dir, sliceHeight=416, sliceWidth=416, zero_frac_thresh=0.2, overlap=0.2, Pobj = 0.4, Pimage = 0.5, slice_sep='|', out_ext='.png', verbose=False) :
def slice_tot(image_path, bbox_path, out_name, outdir):
image_path_tot = os.path.join(orig_dir,image_path)
image0 = cv2.imread(image_path_tot, 1) # color
if len(out_ext) == 0:
ext = '.' + image_path.split('.')[-1]
else:
ext = out_ext
win_h, win_w = image0.shape[:2]
nH = image0.shape[0]
nW = image0.shape[1]
# if slice sizes are large than image, pad the edges
pad = 0
if sliceHeight > win_h:
pad = sliceHeight - win_h
if sliceWidth > win_w:
pad = max(pad, sliceWidth - win_w)
# pad the edge of the image with black pixels
if pad > 0:
border_color = (0,0,0)
image0 = cv2.copyMakeBorder(image0, pad, pad, pad, pad,
cv2.BORDER_CONSTANT, value=border_color)
win_size = sliceHeight*sliceWidth
n_ims = 0
n_ims_nonull = 0
dx = int((1. - overlap) * sliceWidth)
dy = int((1. - overlap) * sliceHeight)
# bbox
m = len(bbox_path)
data_txt_abs = bbox_path
for j in range(m):
data_txt_abs[j][0] = int(bbox_path[j][0]) # classe
data_txt_abs[j][1] = int(bbox_path[j][1] * nW) # x
data_txt_abs[j][2] = int(bbox_path[j][2] * nH) # y
data_txt_abs[j][3] = int(bbox_path[j][3] * nW) # w
data_txt_abs[j][4] = int(bbox_path[j][4] * nH) # h
for y0 in range(0, image0.shape[0], dy):#sliceHeight):
for x0 in range(0, image0.shape[1], dx):#sliceWidth):
n_ims += 1
# if (n_ims % 100) == 0:
# print (n_ims)
# make sure we don't have a tiny image on the edge
if y0+sliceHeight > image0.shape[0]:
y = image0.shape[0] - sliceHeight
else:
y = y0
if x0+sliceWidth > image0.shape[1]:
x = image0.shape[1] - sliceWidth
else:
x = x0
# extract image
window_c = image0[y:y + sliceHeight, x:x + sliceWidth]
# get black and white image
window = cv2.cvtColor(window_c, cv2.COLOR_BGR2GRAY)
# find threshold that's not black
# https://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_imgproc/py_thresholding/py_thresholding.html?highlight=threshold
ret,thresh1 = cv2.threshold(window, 2, 255, cv2.THRESH_BINARY)
non_zero_counts = cv2.countNonZero(thresh1)
zero_counts = win_size - non_zero_counts
zero_frac = float(zero_counts) / win_size
#print "zero_frac", zero_fra
# skip if image is mostly empty
if zero_frac >= zero_frac_thresh:
if verbose:
print ("Zero frac too high at:", zero_frac)
continue
# else save
else:
#outpath = os.path.join(outdir, out_name + \
#'|' + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\
#'_' + str(pad) + ext)
outpath = os.path.join(outdir, out_name + \
slice_sep + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\
'_' + str(pad) + '_' + str(win_w) + '_' + str(win_h) + ext)
#outpath = os.path.join(outdir, 'slice_' + out_name + \
#'_' + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\
#'_' + str(pad) + '.jpg')
if verbose:
print ("outpath:", outpath)
cv2.imwrite(outpath, window_c)
n_ims_nonull += 1
##################### slicing bboxes #############################################
x_min_slice = x
x_max_slice = x + sliceWidth
y_min_slice = y
y_max_slice = y + sliceHeight
x_slice_abs = x + int(0.5 * sliceWidth)
y_slice_abs = y + int(0.5 * sliceHeight)
txt_file_name = ("%s.txt" % (outpath[0:-4]))
text_file = open(txt_file_name, "w")
m = len(data_txt[i])
for j in range(m):
X_obj_abs = data_txt_abs[j][1]
Y_obj_abs = data_txt_abs[j][2]
W_obj_abs = data_txt_abs[j][3]
H_obj_abs = data_txt_abs[j][4]
object_area_abs = W_obj_abs * H_obj_abs
X_min_obj_abs = int(X_obj_abs - 0.5 * W_obj_abs)
X_max_obj_abs = int(X_obj_abs + 0.5 * W_obj_abs)
Y_min_obj_abs = int(Y_obj_abs - 0.5 * H_obj_abs)
Y_max_obj_abs = int(Y_obj_abs + 0.5 * H_obj_abs)
Inter = int(Calc_IoU.calc_Inter(X_obj_abs, x_slice_abs, Y_obj_abs, y_slice_abs, W_obj_abs, sliceWidth, H_obj_abs, sliceHeight))
object_covering = float(Inter/object_area_abs)
window_covering = float(Inter/win_size)
centroid_inside_window = ((x_min_slice <= X_obj_abs <= x_max_slice) and (y_min_slice <= Y_obj_abs <= y_max_slice))
if (object_covering > Pobj or window_covering > Pimage or centroid_inside_window):
x_obj_slice_temp = float((X_obj_abs - x_min_slice) / sliceWidth)
y_obj_slice_temp = float((Y_obj_abs - y_min_slice) / sliceHeight)
w_obj_slice_temp = float(W_obj_abs / sliceWidth)
h_obj_slice_temp = float(H_obj_abs / sliceHeight)
# print(w_obj_slice_temp, h_obj_slice_temp)
x_min_obj_slice = float(x_obj_slice_temp - 0.5 * w_obj_slice_temp)
x_max_obj_slice = float(x_obj_slice_temp + 0.5 * w_obj_slice_temp)
y_min_obj_slice = float(y_obj_slice_temp - 0.5 * h_obj_slice_temp)
y_max_obj_slice = float(y_obj_slice_temp + 0.5 * h_obj_slice_temp)
# print("xmin xmax ymin ymax")
# print(x_min_obj_slice, x_max_obj_slice, y_min_obj_slice, y_max_obj_slice)
if (x_min_obj_slice < 0):
x_min_obj_slice = 0
if (y_min_obj_slice < 0):
y_min_obj_slice = 0
if (x_max_obj_slice > 1):
x_max_obj_slice = 1
if (y_max_obj_slice > 1):
y_max_obj_slice = 1
w_obj_slice = float(x_max_obj_slice - x_min_obj_slice)
h_obj_slice = float(y_max_obj_slice - y_min_obj_slice)
x_obj_slice = float(x_min_obj_slice + 0.5 * w_obj_slice)
y_obj_slice = float(y_min_obj_slice + 0.5 * h_obj_slice)
#print(x_obj_slice, y_obj_slice, w_obj_slice, h_obj_slice)
text_file.write("%d %4f %4f %4f %4f\n" % (
data_txt_abs[j][0], x_obj_slice, y_obj_slice, w_obj_slice,
h_obj_slice))
text_file.close()
# remove file if empty
# print(os.path.getsize(txt_file_name))
if os.stat(txt_file_name).st_size == 0:
os.remove(txt_file_name)
return
data_image = sorted([f for f in os.listdir(orig_dir) if (f.endswith('.JPG') or f.endswith('.jpg'))])
n = len(data_image)
# print(origine_slice)
files_bbox = sorted([f for f in os.listdir(orig_dir) if f.endswith('.txt')])
# print(files_bbox)
data_txt = load_bbox(files_bbox, orig_dir)
for i in range(n):
slice_tot(data_image[i],data_txt[i] ,data_image[i][0:-4], slice_dir)
def slice_test(orig_dir, slice_dir, sliceHeight=256, sliceWidth=256, zero_frac_thresh=0.2, overlap=0.2, slice_sep='|', out_ext='.png', verbose=False) :
def slice_im(image_path, out_name, outdir):
image_path_tot = os.path.join(orig_dir,image_path)
image0 = cv2.imread(image_path_tot, 1) # color
if len(out_ext) == 0:
ext = '.' + image_path.split('.')[-1]
else:
ext = out_ext
win_h, win_w = image0.shape[:2]
nH = image0.shape[0]
nW = image0.shape[1]
# if slice sizes are large than image, pad the edges
pad = 0
if sliceHeight > win_h:
pad = sliceHeight - win_h
if sliceWidth > win_w:
pad = max(pad, sliceWidth - win_w)
# pad the edge of the image with black pixels
if pad > 0:
border_color = (0,0,0)
image0 = cv2.copyMakeBorder(image0, pad, pad, pad, pad,
cv2.BORDER_CONSTANT, value=border_color)
win_size = sliceHeight*sliceWidth
n_ims = 0
n_ims_nonull = 0
dx = int((1. - overlap) * sliceWidth)
dy = int((1. - overlap) * sliceHeight)
for y0 in range(0, image0.shape[0], dy):#sliceHeight):
for x0 in range(0, image0.shape[1], dx):#sliceWidth):
n_ims += 1
# if (n_ims % 100) == 0:
# print (n_ims)
# make sure we don't have a tiny image on the edge
if y0+sliceHeight > image0.shape[0]:
y = image0.shape[0] - sliceHeight
else:
y = y0
if x0+sliceWidth > image0.shape[1]:
x = image0.shape[1] - sliceWidth
else:
x = x0
# extract image
window_c = image0[y:y + sliceHeight, x:x + sliceWidth]
# get black and white image
window = cv2.cvtColor(window_c, cv2.COLOR_BGR2GRAY)
# find threshold that's not black
# https://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_imgproc/py_thresholding/py_thresholding.html?highlight=threshold
ret,thresh1 = cv2.threshold(window, 2, 255, cv2.THRESH_BINARY)
non_zero_counts = cv2.countNonZero(thresh1)
zero_counts = win_size - non_zero_counts
zero_frac = float(zero_counts) / win_size
#print "zero_frac", zero_fra
# skip if image is mostly empty
if zero_frac >= zero_frac_thresh:
if verbose:
print ("Zero frac too high at:", zero_frac)
continue
# else save
else:
#outpath = os.path.join(outdir, out_name + \
#'|' + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\
#'_' + str(pad) + ext)
outpath = os.path.join(outdir, out_name + \
slice_sep + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\
'_' + str(pad) + '_' + str(win_w) + '_' + str(win_h) + ext)
#outpath = os.path.join(outdir, 'slice_' + out_name + \
#'_' + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\
#'_' + str(pad) + '.jpg')
if verbose:
print ("outpath:", outpath)
cv2.imwrite(outpath, window_c)
n_ims_nonull += 1
return
data_image = sorted([f for f in os.listdir(orig_dir) if (f.endswith('.JPG') or f.endswith('.jpg'))])
n = len(data_image)
# print(origine_slice)
files_bbox = sorted([f for f in os.listdir(orig_dir) if f.endswith('.txt')])
# print(files_bbox)
data_txt = load_bbox(files_bbox, orig_dir)
for i in range(n):
slice_im(data_image[i], data_image[i][0:-4], slice_dir)
def slice_test(orig_dir,
slice_dir,
sliceHeight=256,
sliceWidth=256,
zero_frac_thresh=0.2,
overlap=0.2,
slice_sep='|',
out_ext='.jpg',
verbose=False):
def slice_im(image_path, out_name, outdir):
image_path_tot = os.path.join(orig_dir, image_path)
image0 = Image.open(
image_path_tot) # cv2.imread(image_path_tot, 1) # color
if len(out_ext) == 0:
ext = '.' + image_path.split('.')[-1]
else:
ext = out_ext
#
width, height = image0.size
print(width, height)
dx = int((1. - overlap) * sliceWidth)
dy = int((1. - overlap) * sliceHeight)
for y0 in range(0, height, dy): # sliceHeight):
for x0 in range(0, width, dx): # sliceWidth):
# make sure we don't have a tiny image on the edge
if y0 + sliceHeight > height:
y = height - sliceHeight
else:
y = y0
if x0 + sliceWidth > width:
x = width - sliceWidth
else:
x = x0
# extract image
window_c = image0.crop((x, y, x + sliceWidth, y + sliceHeight))
outpath = os.path.join(outdir, out_name + \
slice_sep + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(
sliceWidth) + ext)
window_c.save(outpath)
return
data_image = sorted([
f for f in os.listdir(orig_dir)
if (f.endswith('.JPG') or f.endswith('.jpg'))
])
n = len(data_image)
# print(origine_slice)
files_bbox = sorted(
[f for f in os.listdir(orig_dir) if f.endswith('.txt')])
# print(files_bbox)
data_txt = load_bbox(files_bbox, orig_dir)
for i in range(n):
slice_im(data_image[i], data_image[i][0:-4], slice_dir)
def slice_train(orig_dir,
slice_dir,
sliceHeight=256,
sliceWidth=256,
zero_frac_thresh=0.2,
overlap=0.2,
Pobj=0.4,
Pimage=0.5,
slice_sep='|',
out_ext='.jpg',
verbose=False):
def slice_tot(image_path, bbox_path, out_name, outdir):
image_path_tot = os.path.join(orig_dir, image_path)
image0 = Image.open(
image_path_tot) # cv2.imread(image_path_tot, 1) # color
if len(out_ext) == 0:
ext = '.' + image_path.split('.')[-1]
else:
ext = out_ext
#
width, height = image0.size
print(width, height)
dx = int((1. - overlap) * sliceWidth)
dy = int((1. - overlap) * sliceHeight)
# bbox
m = len(bbox_path)
data_txt_abs = bbox_path
for j in range(m):
data_txt_abs[j][0] = int(bbox_path[j][0]) # classe
data_txt_abs[j][1] = int(bbox_path[j][1] * nW) # x
data_txt_abs[j][2] = int(bbox_path[j][2] * nH) # y
data_txt_abs[j][3] = int(bbox_path[j][3] * nW) # w
data_txt_abs[j][4] = int(bbox_path[j][4] * nH) # h
for y0 in range(0, height, dy): #sliceHeight):
for x0 in range(0, width, dx): #sliceWidth):
# make sure we don't have a tiny image on the edge
if y0 + sliceHeight > height:
y = height - sliceHeight
else:
y = y0
if x0 + sliceWidth > width:
x = width - sliceWidth
else:
x = x0
# extract image
window_c = image0.crop((x, y, x + sliceWidth, y + sliceHeight))
outpath = os.path.join(outdir, out_name + \
slice_sep + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(
sliceWidth) + ext)
window_c.save(outpath)
##################### slicing bboxes #############################################
x_min_slice = x
x_max_slice = x + sliceWidth
y_min_slice = y
y_max_slice = y + sliceHeight
x_slice_abs = x + int(0.5 * sliceWidth)
y_slice_abs = y + int(0.5 * sliceHeight)
txt_file_name = ("%s.txt" % (outpath[0:-4]))
text_file = open(txt_file_name, "w")
m = len(data_txt[i])
for j in range(m):
X_obj_abs = data_txt_abs[j][1]
Y_obj_abs = data_txt_abs[j][2]
W_obj_abs = data_txt_abs[j][3]
H_obj_abs = data_txt_abs[j][4]
object_area_abs = W_obj_abs * H_obj_abs
X_min_obj_abs = int(X_obj_abs - 0.5 * W_obj_abs)
X_max_obj_abs = int(X_obj_abs + 0.5 * W_obj_abs)
Y_min_obj_abs = int(Y_obj_abs - 0.5 * H_obj_abs)
Y_max_obj_abs = int(Y_obj_abs + 0.5 * H_obj_abs)
Inter = int(
Calc_IoU.calc_Inter(X_obj_abs, x_slice_abs, Y_obj_abs,
y_slice_abs, W_obj_abs, sliceWidth,
H_obj_abs, sliceHeight))
object_covering = float(Inter / object_area_abs)
window_covering = float(Inter / win_size)
centroid_inside_window = (
(x_min_slice <= X_obj_abs <= x_max_slice)
and (y_min_slice <= Y_obj_abs <= y_max_slice))
if (object_covering > Pobj or window_covering > Pimage
or centroid_inside_window):
x_obj_slice_temp = float(
(X_obj_abs - x_min_slice) / sliceWidth)
y_obj_slice_temp = float(
(Y_obj_abs - y_min_slice) / sliceHeight)
w_obj_slice_temp = float(W_obj_abs / sliceWidth)
h_obj_slice_temp = float(H_obj_abs / sliceHeight)
# print(w_obj_slice_temp, h_obj_slice_temp)
x_min_obj_slice = float(x_obj_slice_temp -
0.5 * w_obj_slice_temp)
x_max_obj_slice = float(x_obj_slice_temp +
0.5 * w_obj_slice_temp)
y_min_obj_slice = float(y_obj_slice_temp -
0.5 * h_obj_slice_temp)
y_max_obj_slice = float(y_obj_slice_temp +
0.5 * h_obj_slice_temp)
# print("xmin xmax ymin ymax")
# print(x_min_obj_slice, x_max_obj_slice, y_min_obj_slice, y_max_obj_slice)
if (x_min_obj_slice < 0):
x_min_obj_slice = 0
if (y_min_obj_slice < 0):
y_min_obj_slice = 0
if (x_max_obj_slice > 1):
x_max_obj_slice = 1
if (y_max_obj_slice > 1):
y_max_obj_slice = 1
w_obj_slice = float(x_max_obj_slice - x_min_obj_slice)
h_obj_slice = float(y_max_obj_slice - y_min_obj_slice)
x_obj_slice = float(x_min_obj_slice +
0.5 * w_obj_slice)
y_obj_slice = float(y_min_obj_slice +
0.5 * h_obj_slice)
#print(x_obj_slice, y_obj_slice, w_obj_slice, h_obj_slice)
text_file.write(
"%d %4f %4f %4f %4f\n" %
(data_txt_abs[j][0], x_obj_slice, y_obj_slice,
w_obj_slice, h_obj_slice))
text_file.close()
# remove file if empty
# print(os.path.getsize(txt_file_name))
if os.stat(txt_file_name).st_size == 0:
os.remove(txt_file_name)
return
data_image = sorted([
f for f in os.listdir(orig_dir)
if (f.endswith('.JPG') or f.endswith('.jpg'))
])
n = len(data_image)
# print(origine_slice)
files_bbox = sorted(
[f for f in os.listdir(orig_dir) if f.endswith('.txt')])
# print(files_bbox)
data_txt = load_bbox(files_bbox, orig_dir)
for i in range(n):
slice_tot(data_image[i], data_txt[i], data_image[i][0:-4], slice_dir)
def count_intra_gt(test_im_dir,
names_file,
print_res=False,
edit_res=False,
outname='results/object_intra_gt.csv'):
# clean data preparation : remove images without labels, txt files that are no label file ...
file_bbox = sorted(
[f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
image_labelled = []
for i in range(len(file_bbox)):
root_name_bbx = file_bbox[i].split('.')[0]
for image in images:
if image.split('.')[0] == root_name_bbx:
image_labelled.append(image)
image_labelled = sorted(image_labelled)
labels_with_images = []
for i in range(len(image_labelled)):
root_name_image = image_labelled[i].split('.')[0]
for label in file_bbox:
if label.split('.')[0] == root_name_image:
labels_with_images.append(label)
labels_with_images = sorted(labels_with_images)
bbox = load_bbox(labels_with_images, test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, image_labelled[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
gt_df = pd.DataFrame(ground_truth,
columns=('root_image', 'obj_class', 'x', 'y', 'w',
'h'))
gt_df['root_image_path'] = 'test_images/' + gt_df['root_image'] + '.JPG'
gt_df['obj_class'] = gt_df.obj_class.astype(str)
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
df = gt_df # I have only copied the previous function, now that df are of the same format
# generating template for result df
list_of_list = []
for image in df.root_image_path.unique():
date = get_date_taken(image)
df_image = df[df.root_image_path == image]
for obj_class in df_image.obj_class.unique():
list = [image, date, obj_class, 0]
list_of_list.append(list)
df_inter = pd.DataFrame(list_of_list,
columns=('root_image_path', 'date', 'obj_class',
'count'))
#
for image in df.root_image_path.unique():
date = get_date_taken(image)
df_image = df[df.root_image_path == image]
df_inter_image = df_inter[df_inter.root_image_path == image]
for classe in df_image.obj_class.unique():
df_class = df_image[df_image.obj_class == classe]
for rowA in df_class.iterrows():
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
classA = rowA[1].obj_class
df_comp = df_class.drop([rowA[0]])
for rowB in df_comp.iterrows():
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
classB = rowB[1].obj_class
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > 0:
df_inter_bis = df_inter_image[df_inter_image.obj_class
== classA]
index_to_change = df_inter_bis.index[0]
df_inter.at[index_to_change, 'count'] += 1
df_inter.loc[:, 'count'] /= 2
#pd.to_numeric(df_inter.loc[:,'count'], downcast='integer')
df_inter = df_inter.sort_values(['root_image_path'])
if print_res:
print(df_inter)
if edit_res:
df_inter.to_csv(outname, index=False)
def count_species_gt(test_im_dir,
names_file,
print_res=False,
edit_res=False,
outname='results/object_count_gt.csv'):
# clean data preparation : remove images without labels, txt files that are no label file ...
file_bbox = sorted(
[f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
image_labelled = []
for i in range(len(file_bbox)):
root_name_bbx = file_bbox[i].split('.')[0]
for image in images:
if image.split('.')[0] == root_name_bbx:
image_labelled.append(image)
image_labelled = sorted(image_labelled)
labels_with_images = []
for i in range(len(image_labelled)):
root_name_image = image_labelled[i].split('.')[0]
for label in file_bbox:
if label.split('.')[0] == root_name_image:
labels_with_images.append(label)
labels_with_images = sorted(labels_with_images)
bbox = load_bbox(labels_with_images, test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, image_labelled[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
gt_df = pd.DataFrame(ground_truth,
columns=('root_image', 'obj_class', 'x', 'y', 'w',
'h'))
gt_df['root_image_path'] = 'test_images/' + gt_df['root_image'] + '.JPG'
gt_df['obj_class'] = gt_df.obj_class.astype(str)
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
list_of_list = []
for image in gt_df.root_image_path.unique():
date = get_date_taken(image)
gt_df_image = gt_df[gt_df.root_image_path == image]
for classe in gt_df_image.obj_class.unique():
count = 0
gt_df_class = gt_df_image[gt_df_image.obj_class == classe]
for row in gt_df_class.iterrows():
count += 1
list = [image, date, classe, count]
list_of_list.append(list)
gt_df_count = pd.DataFrame(list_of_list,
columns=('root_image_path', 'date', 'obj_class',
'count'))
if print_res:
print(gt_df_count)
if edit_res:
gt_df_count.to_csv(outname, index=False)
def get_metrics(test_im_dir,
detection_csv_file,
names_file,
examples_file_path,
print_res=True,
edit_res=False):
################### function to get average precision ################################
def voc_ap(rec, prec):
# from https://github.com/Cartucho/mAP/blob/master/main.py
"""
--- Official matlab code VOC2012---
mrec=[0 ; rec ; 1];
mpre=[0 ; prec ; 0];
for i=numel(mpre)-1:-1:1
mpre(i)=max(mpre(i),mpre(i+1));
end
i=find(mrec(2:end)~=mrec(1:end-1))+1;
ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
"""
rec.insert(0, 0.0) # insert 0.0 at begining of list
rec.append(1.0) # insert 1.0 at end of list
mrec = rec[:]
prec.insert(0, 0.0) # insert 0.0 at begining of list
prec.append(0.0) # insert 0.0 at end of list
mpre = prec[:]
"""
This part makes the precision monotonically decreasing
(goes from the end to the beginning)
matlab: for i=numel(mpre)-1:-1:1
mpre(i)=max(mpre(i),mpre(i+1));
"""
# matlab indexes start in 1 but python in 0, so I have to do:
# range(start=(len(mpre) - 2), end=0, step=-1)
# also the python function range excludes the end, resulting in:
# range(start=(len(mpre) - 2), end=-1, step=-1)
for i in range(len(mpre) - 2, -1, -1):
mpre[i] = max(mpre[i], mpre[i + 1])
"""
This part creates a list of indexes where the recall changes
matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1;
"""
i_list = []
for i in range(1, len(mrec)):
if mrec[i] != mrec[i - 1]:
i_list.append(i) # if it was matlab would be i + 1
"""
The Average Precision (AP) is the area under the curve
(numerical integration)
matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i));
"""
ap = 0.0
for i in i_list:
ap += ((mrec[i] - mrec[i - 1]) * mpre[i])
return ap, mrec, mpre
###########################################################################################################
######################## Formating both detection and GT data into same template ##########################
file_bbox = sorted(
[f for f in os.listdir(test_im_dir) if f.endswith('.txt')])
images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')])
# sort only images with labels and labels attached to images
image_labelled = []
for i in range(len(file_bbox)):
root_name_bbx = file_bbox[i].split('.')[0]
for image in images:
if image.split('.')[0] == root_name_bbx:
image_labelled.append(image)
image_labelled = sorted(image_labelled)
labels_with_images = []
for i in range(len(image_labelled)):
root_name_image = image_labelled[i].split('.')[0]
for label in file_bbox:
if label.split('.')[0] == root_name_image:
labels_with_images.append(label)
labels_with_images = sorted(labels_with_images)
bbox = load_bbox(labels_with_images, test_im_dir)
merged_bbox_list = []
for i in range(len(bbox)):
for j in range(len(bbox[i])):
bbox[i][j].insert(0, images[i][0:-4])
merged_bbox_list = merged_bbox_list + bbox[i]
ground_truth = merged_bbox_list
detection_csv = detection_csv_file
# gt_df stores GT information
# dt_df stores detection information
gt_df = pd.DataFrame(ground_truth,
columns=('root_image', 'obj_class', 'x', 'y', 'w',
'h'))
dt_df = pd.read_csv(detection_csv)
######################### convert classes according .names file ###################
gt_df['obj_class'] = gt_df.obj_class.astype(str)
with open(names_file) as f:
labels = [line.rstrip('\n') for line in f]
for row in gt_df.iterrows():
index, data = row
for i in range(len(labels)):
if (row[1].obj_class == str(float(i))):
gt_df.at[index, 'obj_class'] = labels[i]
################## compare only images that have associated GT ####################
dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())]
################## get the list of all classes (detected of not) ####################
class_list = []
max_name_class_length = 0
for dift_class in gt_df.obj_class.unique():
class_list.append(dift_class)
# included by Dom
max_name_class_length = max(max_name_class_length, len(dift_class))
# end of included by Dom
for dift_class in dt_df.obj_class.unique():
if dift_class not in class_list:
class_list.append(dift_class)
########################### generating template dict for per classes metrics ###############
mAP_class_dict = {}
prec_class_dict = {}
for dift_class in class_list:
mAP_class_dict[dift_class] = 0
prec_class_dict[dift_class] = 0
####################### initialize variables #############################
dt_df = dt_df.sort_values(['obj_class'])
dt_df['correct'] = 0
dt_df['precision'] = 0.0
dt_df['recall'] = 0.0
gt_df = gt_df.sort_values(['obj_class'])
gt_df['used'] = 0
# overlap threshold for acceptance
overlap_threshold = 0.5
####################### comparison of detected objects to GT #################################
for image in dt_df.root_image.unique():
print(image)
dt_df_img = dt_df[dt_df['root_image'] == image]
gt_df_img = gt_df[gt_df['root_image'] == image]
# list different classes present in GT
gt_classes = gt_df_img.obj_class.unique()
# list out wrong predicted classes
dt_df_correct_classes = dt_df_img[dt_df_img['obj_class'].isin(
gt_classes)]
# for each correct class
for dift_class in gt_classes:
gt_df_class = gt_df_img[gt_df_img.obj_class == dift_class]
dt_df_class = dt_df_correct_classes[dt_df_correct_classes.obj_class
== dift_class]
for rowA in gt_df_class.iterrows(
): # compare each GT object of this class ...
xA = rowA[1].x
yA = rowA[1].y
wA = rowA[1].w
hA = rowA[1].h
used = rowA[1].used
for rowB in dt_df_class.iterrows(
): # ... with every detected object of this class
xB = rowB[1].x
yB = rowB[1].y
wB = rowB[1].w
hB = rowB[1].h
IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB)
if IOU > overlap_threshold: # i.e. if detection and GT overlap
if used == 0: # gt not found yet
indexB, dataB = rowB
dt_df.at[indexB, 'correct'] = 1
indexA, dataA = rowA
gt_df.at[indexA, 'used'] = 1
df_TP = dt_df[dt_df.correct == 1]
df_FP = dt_df[dt_df.correct == 0]
df_FN = gt_df[gt_df.used == 0]
TP = len(df_TP.correct)
FP = len(df_FP.correct)
FN = len(df_FN.used)
precision_general = float(TP / (TP + FP))
recall_general = float(TP / (TP + FN))
F1_general = 2 * float((precision_general * recall_general) /
(precision_general + recall_general))
########################### mAP #############################
# once we have for each detection its status (TP or FP), we can compute average precision for each class
# for this we will need to compute recall and precision for each class
# included by Dom
gt_classes = gt_df.obj_class.unique()
nb_classes = len(gt_classes)
df_metrics_classes = pd.DataFrame(pd.np.empty((nb_classes, 10)),
columns=[
'classes', 'precision', 'recall',
'F1', 'mAP', 'GT', 'TP', 'FP', 'FN',
'training_examples'
])
df_metrics_classes['classes'] = gt_classes
df_metrics_classes['precision'] = 0
df_metrics_classes['recall'] = 0
df_metrics_classes['F1'] = 0
df_metrics_classes['mAP'] = 0
df_metrics_classes['GT'] = 0
df_metrics_classes['TP'] = 0
df_metrics_classes['FP'] = 0
df_metrics_classes['FN'] = 0
# end of included by Dom
df_examples = pd.read_csv(examples_file_path, index_col='name')
print('*********')
print(df_examples)
for dift_class in gt_df.obj_class.unique():
gt_df_class = gt_df[gt_df.obj_class == dift_class]
items_to_detect = len(gt_df_class)
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'GT'] = items_to_detect
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'training_examples'] = df_examples.at[
dift_class, 'count']
FN_count = 0
for row in gt_df_class.iterrows():
index, data = row
if (data['used'] == 0):
FN_count += 1
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'FN'] = FN_count
for dift_class in dt_df.obj_class.unique():
gt_df_class = gt_df[gt_df.obj_class == dift_class]
dt_df_class = dt_df[dt_df.obj_class == dift_class]
# GT number of objects to find (potential positives = PP)
PP = len(gt_df_class)
if PP == 0: # if false positive
mAP_class_dict[dift_class] = 0
if PP != 0:
TP_count = 0
FP_count = 0
row_count = 0 # (for recall)
for row in dt_df_class.iterrows():
row_count += 1
index, data = row
if (data['correct'] == 1):
TP_count += 1
else:
FP_count += 1
# if (data['used'] == 0):
# FN_count +=1
recall = float(TP_count / PP)
#print('recall = ', recall)
precision = float(TP_count / row_count)
#print('precision = ', precision)
dt_df_class.at[index, 'recall'] = recall
dt_df_class.at[index, 'precision'] = precision
prec = dt_df_class['precision'].tolist()
rec = dt_df_class['recall'].tolist()
# included by Dom
avg_prec_class = mean(prec)
if avg_prec_class != 0 and recall != 0:
F1_score_class = 2 * float(
(avg_prec_class * recall) / (avg_prec_class + recall))
else:
F1_score_class = 0
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'precision'] = avg_prec_class
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'recall'] = recall
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'F1'] = F1_score_class
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'TP'] = TP_count
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'FP'] = FP_count
# end of included by Dom
ap, mrec, mprec = voc_ap(
rec, prec) # according previously defined function
mAP_class_dict[dift_class] += ap
# this is just a fix for some issues pandas can encounter with .from_dict() function
# but sometimes, it does work perfectly fine.
new_dict = {k: [v] for k, v in mAP_class_dict.items()}
temp_df = pd.DataFrame(new_dict, index=['mAP'])
mAP_class_df = temp_df.T
df_metrics_classes.training_examples = df_metrics_classes.training_examples.astype(
int)
for dift_class in dt_df.obj_class.unique():
df_metrics_classes.loc[df_metrics_classes.classes == dift_class,
'mAP'] = mAP_class_df.at[dift_class, 'mAP']
# mAP_class_df = pd.DataFrame.from_dict(mAP_class_dict, orient = 'index', columns = ['mAP'])
mAP = float(float(sum(mAP_class_dict.values())) / float(len(class_list)))
if print_res:
print('\n metrics per class_______________________\n')
print(df_metrics_classes)
print('\n general metrics _________________\n')
print('mAP: \t\t%.4f' % mAP)
print('precision: \t%.4f' % precision_general)
print('recall: \t%.4f' % recall_general)
print('F1: \t\t%.4f' % F1_general)
df_metrics_classes.to_csv('test_temp/metrics_per_class.ods',
index=False)
df_metrics_classes.to_csv('test_temp/metrics_per_class.csv',
index=False)
df_general_metrics = pd.DataFrame(
pd.np.empty((1, 4)), columns=['mAP', 'precision', 'recall', 'F1'])
df_general_metrics['mAP'] = mAP
df_general_metrics['precision'] = precision_general
df_general_metrics['recall'] = recall_general
df_general_metrics['F1'] = F1_general
df_general_metrics.to_csv('test_temp/general_metrics.csv', index=False)
df_general_metrics.to_csv('test_temp/general_metrics.ods', index=False)
#################### edit results to csv file
#modified by Dom; variables were not defined; maybe to delete?
if edit_res:
res_list = [['mAP', mAP], ['precision', precision_general],
['recall', recall_general], ['F1', F1_general]]
res_df = pd.DataFrame(res_list, columns=['metric', 'value'])
# print(res_df.T)
# res_df.to_csv('test_temp/general_metrics.csv', index=False)
# res_df.to_csv('test_temp/general_metrics.ods', index=False)
return TP, FP, FN, precision_general, recall_general, F1_general, mAP, mAP_class_dict