def get_metrics(test_im_dir,detection_csv_file, names_file, examples_file_path, print_res=True, edit_res=False): ################### function to get average precision ################################ def voc_ap(rec, prec): # from https://github.com/Cartucho/mAP/blob/master/main.py """ --- Official matlab code VOC2012--- mrec=[0 ; rec ; 1]; mpre=[0 ; prec ; 0]; for i=numel(mpre)-1:-1:1 mpre(i)=max(mpre(i),mpre(i+1)); end i=find(mrec(2:end)~=mrec(1:end-1))+1; ap=sum((mrec(i)-mrec(i-1)).*mpre(i)); """ rec.insert(0, 0.0) # insert 0.0 at begining of list rec.append(1.0) # insert 1.0 at end of list mrec = rec[:] prec.insert(0, 0.0) # insert 0.0 at begining of list prec.append(0.0) # insert 0.0 at end of list mpre = prec[:] """ This part makes the precision monotonically decreasing (goes from the end to the beginning) matlab: for i=numel(mpre)-1:-1:1 mpre(i)=max(mpre(i),mpre(i+1)); """ # matlab indexes start in 1 but python in 0, so I have to do: # range(start=(len(mpre) - 2), end=0, step=-1) # also the python function range excludes the end, resulting in: # range(start=(len(mpre) - 2), end=-1, step=-1) for i in range(len(mpre) - 2, -1, -1): mpre[i] = max(mpre[i], mpre[i + 1]) """ This part creates a list of indexes where the recall changes matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1; """ i_list = [] for i in range(1, len(mrec)): if mrec[i] != mrec[i - 1]: i_list.append(i) # if it was matlab would be i + 1 """ The Average Precision (AP) is the area under the curve (numerical integration) matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i)); """ ap = 0.0 for i in i_list: ap += ((mrec[i] - mrec[i - 1]) * mpre[i]) return ap, mrec, mpre ########################################################################################################### ######################## Formating both detection and GT data into same template ########################## file_bbox = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.txt')]) images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')]) # sort only images with labels and labels attached to images image_labelled = [] for i in range(len(file_bbox)): root_name_bbx = file_bbox[i].split('.')[0] for image in images: if image.split('.')[0] == root_name_bbx: image_labelled.append(image) image_labelled = sorted(image_labelled) labels_with_images = [] for i in range(len(image_labelled)): root_name_image = image_labelled[i].split('.')[0] for label in file_bbox: if label.split('.')[0] == root_name_image: labels_with_images.append(label) labels_with_images = sorted(labels_with_images) bbox = load_bbox(labels_with_images, test_im_dir) merged_bbox_list = [] for i in range(len(bbox)): for j in range(len(bbox[i])): bbox[i][j].insert(0, images[i][0:-4]) merged_bbox_list = merged_bbox_list + bbox[i] ground_truth = merged_bbox_list detection_csv = detection_csv_file # gt_df stores GT information # dt_df stores detection information gt_df = pd.DataFrame(ground_truth, columns=('root_image','obj_class', 'x', 'y', 'w', 'h')) dt_df = pd.read_csv(detection_csv) ######################### convert classes according .names file ################### gt_df['obj_class'] = gt_df.obj_class.astype(str) with open(names_file) as f: labels = [line.rstrip('\n') for line in f] for row in gt_df.iterrows(): index, data = row for i in range(len(labels)): if (row[1].obj_class == str(float(i))): gt_df.at[index, 'obj_class'] = labels[i] ################## compare only images that have associated GT #################### dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())] ################## get the list of all classes (detected of not) #################### class_list = [] max_name_class_length = 0 for dift_class in gt_df.obj_class.unique(): class_list.append(dift_class) # included by Dom max_name_class_length = max(max_name_class_length, len(dift_class)) # end of included by Dom for dift_class in dt_df.obj_class.unique(): if dift_class not in class_list: class_list.append(dift_class) ########################### generating template dict for per classes metrics ############### mAP_class_dict = {} prec_class_dict = {} for dift_class in class_list : mAP_class_dict[dift_class] = 0 prec_class_dict[dift_class] = 0 ####################### initialize variables ############################# dt_df = dt_df.sort_values(['obj_class']) dt_df['correct'] = 0 dt_df['precision'] = 0.0 dt_df['recall'] = 0.0 gt_df = gt_df.sort_values(['obj_class']) gt_df['used'] = 0 # overlap threshold for acceptance overlap_threshold = 0.5 ####################### comparison of detected objects to GT ################################# for image in dt_df.root_image.unique(): print(image) dt_df_img = dt_df[dt_df['root_image'] == image] gt_df_img = gt_df[gt_df['root_image'] == image] # list different classes present in GT gt_classes = gt_df_img.obj_class.unique() # list out wrong predicted classes dt_df_correct_classes = dt_df_img[dt_df_img['obj_class'].isin(gt_classes)] # for each correct class for dift_class in gt_classes: gt_df_class = gt_df_img[gt_df_img.obj_class == dift_class] dt_df_class = dt_df_correct_classes[dt_df_correct_classes.obj_class == dift_class] for rowA in gt_df_class.iterrows(): # compare each GT object of this class ... xA = rowA[1].x yA = rowA[1].y wA = rowA[1].w hA = rowA[1].h used = rowA[1].used for rowB in dt_df_class.iterrows(): # ... with every detected object of this class xB = rowB[1].x yB = rowB[1].y wB = rowB[1].w hB = rowB[1].h IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB) if IOU > overlap_threshold : # i.e. if detection and GT overlap if used == 0: # gt not found yet indexB, dataB = rowB dt_df.at[indexB, 'correct'] = 1 indexA, dataA = rowA gt_df.at[indexA, 'used'] = 1 df_TP = dt_df[dt_df.correct == 1] df_FP = dt_df[dt_df.correct == 0] df_FN = gt_df[gt_df.used == 0] TP = len(df_TP.correct) FP = len(df_FP.correct) FN = len(df_FN.used) precision_general = float(TP / (TP + FP)) recall_general = float(TP / (TP + FN)) if float((precision_general + recall_general)) != 0: F1_general = 2 * float((precision_general * recall_general) / (precision_general + recall_general))
def get_confusion_matrix(test_im_dir,detection_csv_file, names_file, outname = 'test_temp/confusion_matrix.csv'): file_bbox = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.txt')]) images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')]) bbox = load_bbox(file_bbox,test_im_dir) merged_bbox_list = [] for i in range(len(bbox)): for j in range(len(bbox[i])): bbox[i][j].insert(0, images[i][0:-4]) merged_bbox_list = merged_bbox_list + bbox[i] ground_truth = merged_bbox_list detection_csv = detection_csv_file gt_df = pd.DataFrame(ground_truth, columns=('root_image','obj_class', 'x', 'y', 'w', 'h')) gt_df['obj_class'] = gt_df.obj_class.astype(str) dt_df = pd.read_csv(detection_csv) ######################### convert classes according .names file ################### with open(names_file) as f: labels = [line.rstrip('\n') for line in f] for row in gt_df.iterrows(): index, data = row for i in range(len(labels)): if (row[1].obj_class == str(float(i))): gt_df.at[index, 'obj_class'] = labels[i] ########################## dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())] dt_df = dt_df.sort_values(['obj_class']) dt_df['correct'] = 0 gt_df = gt_df.sort_values(['obj_class']) gt_df['used'] = 0 ########################### generating template dict for per classes metrics ############### # get all classes (detected and GT) all_classes = [] for dift_class in dt_df.obj_class.unique(): all_classes.append(dift_class) for dift_class in gt_df.obj_class.unique(): if dift_class not in all_classes: all_classes.append(dift_class) all_classes = sorted(all_classes) template_dic = {} for dift_class_x in all_classes : template_dic[dift_class_x] = {} for dift_class_y in all_classes : template_dic[dift_class_x][dift_class_y] = 0 ####################### comparison of detected objects to GT ################################# # overlap threshold for acceptance overlap_threshold = 0.5 for image in dt_df.root_image.unique(): dt_df_img = dt_df[dt_df['root_image'] == image] gt_df_img = gt_df[gt_df['root_image'] == image] for rowA in gt_df_img.iterrows(): xA = rowA[1].x yA = rowA[1].y wA = rowA[1].w hA = rowA[1].h classA = rowA[1].obj_class for rowB in dt_df_img.iterrows(): # ... with every detected object of this class xB = rowB[1].x yB = rowB[1].y wB = rowB[1].w hB = rowB[1].h classB = rowB[1].obj_class IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB) if IOU > overlap_threshold : # i.e. if detection and GT overlap template_dic[classA][classB] += 1 #template_dic[classB][classA] += 1 matrix_df = pd.DataFrame(template_dic) #print(matrix_df) matrix_df.to_csv(outname)
def slice_train(orig_dir, slice_dir, sliceHeight=416, sliceWidth=416, zero_frac_thresh=0.2, overlap=0.2, Pobj = 0.4, Pimage = 0.5, slice_sep='|', out_ext='.png', verbose=False) : def slice_tot(image_path, bbox_path, out_name, outdir): image_path_tot = os.path.join(orig_dir,image_path) image0 = cv2.imread(image_path_tot, 1) # color if len(out_ext) == 0: ext = '.' + image_path.split('.')[-1] else: ext = out_ext win_h, win_w = image0.shape[:2] nH = image0.shape[0] nW = image0.shape[1] # if slice sizes are large than image, pad the edges pad = 0 if sliceHeight > win_h: pad = sliceHeight - win_h if sliceWidth > win_w: pad = max(pad, sliceWidth - win_w) # pad the edge of the image with black pixels if pad > 0: border_color = (0,0,0) image0 = cv2.copyMakeBorder(image0, pad, pad, pad, pad, cv2.BORDER_CONSTANT, value=border_color) win_size = sliceHeight*sliceWidth n_ims = 0 n_ims_nonull = 0 dx = int((1. - overlap) * sliceWidth) dy = int((1. - overlap) * sliceHeight) # bbox m = len(bbox_path) data_txt_abs = bbox_path for j in range(m): data_txt_abs[j][0] = int(bbox_path[j][0]) # classe data_txt_abs[j][1] = int(bbox_path[j][1] * nW) # x data_txt_abs[j][2] = int(bbox_path[j][2] * nH) # y data_txt_abs[j][3] = int(bbox_path[j][3] * nW) # w data_txt_abs[j][4] = int(bbox_path[j][4] * nH) # h for y0 in range(0, image0.shape[0], dy):#sliceHeight): for x0 in range(0, image0.shape[1], dx):#sliceWidth): n_ims += 1 # if (n_ims % 100) == 0: # print (n_ims) # make sure we don't have a tiny image on the edge if y0+sliceHeight > image0.shape[0]: y = image0.shape[0] - sliceHeight else: y = y0 if x0+sliceWidth > image0.shape[1]: x = image0.shape[1] - sliceWidth else: x = x0 # extract image window_c = image0[y:y + sliceHeight, x:x + sliceWidth] # get black and white image window = cv2.cvtColor(window_c, cv2.COLOR_BGR2GRAY) # find threshold that's not black # https://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_imgproc/py_thresholding/py_thresholding.html?highlight=threshold ret,thresh1 = cv2.threshold(window, 2, 255, cv2.THRESH_BINARY) non_zero_counts = cv2.countNonZero(thresh1) zero_counts = win_size - non_zero_counts zero_frac = float(zero_counts) / win_size #print "zero_frac", zero_fra # skip if image is mostly empty if zero_frac >= zero_frac_thresh: if verbose: print ("Zero frac too high at:", zero_frac) continue # else save else: #outpath = os.path.join(outdir, out_name + \ #'|' + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\ #'_' + str(pad) + ext) outpath = os.path.join(outdir, out_name + \ slice_sep + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\ '_' + str(pad) + '_' + str(win_w) + '_' + str(win_h) + ext) #outpath = os.path.join(outdir, 'slice_' + out_name + \ #'_' + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\ #'_' + str(pad) + '.jpg') if verbose: print ("outpath:", outpath) cv2.imwrite(outpath, window_c) n_ims_nonull += 1 ##################### slicing bboxes ############################################# x_min_slice = x x_max_slice = x + sliceWidth y_min_slice = y y_max_slice = y + sliceHeight x_slice_abs = x + int(0.5 * sliceWidth) y_slice_abs = y + int(0.5 * sliceHeight) txt_file_name = ("%s.txt" % (outpath[0:-4])) text_file = open(txt_file_name, "w") m = len(data_txt[i]) for j in range(m): X_obj_abs = data_txt_abs[j][1] Y_obj_abs = data_txt_abs[j][2] W_obj_abs = data_txt_abs[j][3] H_obj_abs = data_txt_abs[j][4] object_area_abs = W_obj_abs * H_obj_abs X_min_obj_abs = int(X_obj_abs - 0.5 * W_obj_abs) X_max_obj_abs = int(X_obj_abs + 0.5 * W_obj_abs) Y_min_obj_abs = int(Y_obj_abs - 0.5 * H_obj_abs) Y_max_obj_abs = int(Y_obj_abs + 0.5 * H_obj_abs) Inter = int(Calc_IoU.calc_Inter(X_obj_abs, x_slice_abs, Y_obj_abs, y_slice_abs, W_obj_abs, sliceWidth, H_obj_abs, sliceHeight)) object_covering = float(Inter/object_area_abs) window_covering = float(Inter/win_size) centroid_inside_window = ((x_min_slice <= X_obj_abs <= x_max_slice) and (y_min_slice <= Y_obj_abs <= y_max_slice)) if (object_covering > Pobj or window_covering > Pimage or centroid_inside_window): x_obj_slice_temp = float((X_obj_abs - x_min_slice) / sliceWidth) y_obj_slice_temp = float((Y_obj_abs - y_min_slice) / sliceHeight) w_obj_slice_temp = float(W_obj_abs / sliceWidth) h_obj_slice_temp = float(H_obj_abs / sliceHeight) # print(w_obj_slice_temp, h_obj_slice_temp) x_min_obj_slice = float(x_obj_slice_temp - 0.5 * w_obj_slice_temp) x_max_obj_slice = float(x_obj_slice_temp + 0.5 * w_obj_slice_temp) y_min_obj_slice = float(y_obj_slice_temp - 0.5 * h_obj_slice_temp) y_max_obj_slice = float(y_obj_slice_temp + 0.5 * h_obj_slice_temp) # print("xmin xmax ymin ymax") # print(x_min_obj_slice, x_max_obj_slice, y_min_obj_slice, y_max_obj_slice) if (x_min_obj_slice < 0): x_min_obj_slice = 0 if (y_min_obj_slice < 0): y_min_obj_slice = 0 if (x_max_obj_slice > 1): x_max_obj_slice = 1 if (y_max_obj_slice > 1): y_max_obj_slice = 1 w_obj_slice = float(x_max_obj_slice - x_min_obj_slice) h_obj_slice = float(y_max_obj_slice - y_min_obj_slice) x_obj_slice = float(x_min_obj_slice + 0.5 * w_obj_slice) y_obj_slice = float(y_min_obj_slice + 0.5 * h_obj_slice) #print(x_obj_slice, y_obj_slice, w_obj_slice, h_obj_slice) text_file.write("%d %4f %4f %4f %4f\n" % ( data_txt_abs[j][0], x_obj_slice, y_obj_slice, w_obj_slice, h_obj_slice)) text_file.close() # remove file if empty # print(os.path.getsize(txt_file_name)) if os.stat(txt_file_name).st_size == 0: os.remove(txt_file_name) return data_image = sorted([f for f in os.listdir(orig_dir) if (f.endswith('.JPG') or f.endswith('.jpg'))]) n = len(data_image) # print(origine_slice) files_bbox = sorted([f for f in os.listdir(orig_dir) if f.endswith('.txt')]) # print(files_bbox) data_txt = load_bbox(files_bbox, orig_dir) for i in range(n): slice_tot(data_image[i],data_txt[i] ,data_image[i][0:-4], slice_dir)
def slice_test(orig_dir, slice_dir, sliceHeight=256, sliceWidth=256, zero_frac_thresh=0.2, overlap=0.2, slice_sep='|', out_ext='.png', verbose=False) : def slice_im(image_path, out_name, outdir): image_path_tot = os.path.join(orig_dir,image_path) image0 = cv2.imread(image_path_tot, 1) # color if len(out_ext) == 0: ext = '.' + image_path.split('.')[-1] else: ext = out_ext win_h, win_w = image0.shape[:2] nH = image0.shape[0] nW = image0.shape[1] # if slice sizes are large than image, pad the edges pad = 0 if sliceHeight > win_h: pad = sliceHeight - win_h if sliceWidth > win_w: pad = max(pad, sliceWidth - win_w) # pad the edge of the image with black pixels if pad > 0: border_color = (0,0,0) image0 = cv2.copyMakeBorder(image0, pad, pad, pad, pad, cv2.BORDER_CONSTANT, value=border_color) win_size = sliceHeight*sliceWidth n_ims = 0 n_ims_nonull = 0 dx = int((1. - overlap) * sliceWidth) dy = int((1. - overlap) * sliceHeight) for y0 in range(0, image0.shape[0], dy):#sliceHeight): for x0 in range(0, image0.shape[1], dx):#sliceWidth): n_ims += 1 # if (n_ims % 100) == 0: # print (n_ims) # make sure we don't have a tiny image on the edge if y0+sliceHeight > image0.shape[0]: y = image0.shape[0] - sliceHeight else: y = y0 if x0+sliceWidth > image0.shape[1]: x = image0.shape[1] - sliceWidth else: x = x0 # extract image window_c = image0[y:y + sliceHeight, x:x + sliceWidth] # get black and white image window = cv2.cvtColor(window_c, cv2.COLOR_BGR2GRAY) # find threshold that's not black # https://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_imgproc/py_thresholding/py_thresholding.html?highlight=threshold ret,thresh1 = cv2.threshold(window, 2, 255, cv2.THRESH_BINARY) non_zero_counts = cv2.countNonZero(thresh1) zero_counts = win_size - non_zero_counts zero_frac = float(zero_counts) / win_size #print "zero_frac", zero_fra # skip if image is mostly empty if zero_frac >= zero_frac_thresh: if verbose: print ("Zero frac too high at:", zero_frac) continue # else save else: #outpath = os.path.join(outdir, out_name + \ #'|' + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\ #'_' + str(pad) + ext) outpath = os.path.join(outdir, out_name + \ slice_sep + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\ '_' + str(pad) + '_' + str(win_w) + '_' + str(win_h) + ext) #outpath = os.path.join(outdir, 'slice_' + out_name + \ #'_' + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str(sliceWidth) +\ #'_' + str(pad) + '.jpg') if verbose: print ("outpath:", outpath) cv2.imwrite(outpath, window_c) n_ims_nonull += 1 return data_image = sorted([f for f in os.listdir(orig_dir) if (f.endswith('.JPG') or f.endswith('.jpg'))]) n = len(data_image) # print(origine_slice) files_bbox = sorted([f for f in os.listdir(orig_dir) if f.endswith('.txt')]) # print(files_bbox) data_txt = load_bbox(files_bbox, orig_dir) for i in range(n): slice_im(data_image[i], data_image[i][0:-4], slice_dir)
def slice_test(orig_dir, slice_dir, sliceHeight=256, sliceWidth=256, zero_frac_thresh=0.2, overlap=0.2, slice_sep='|', out_ext='.jpg', verbose=False): def slice_im(image_path, out_name, outdir): image_path_tot = os.path.join(orig_dir, image_path) image0 = Image.open( image_path_tot) # cv2.imread(image_path_tot, 1) # color if len(out_ext) == 0: ext = '.' + image_path.split('.')[-1] else: ext = out_ext # width, height = image0.size print(width, height) dx = int((1. - overlap) * sliceWidth) dy = int((1. - overlap) * sliceHeight) for y0 in range(0, height, dy): # sliceHeight): for x0 in range(0, width, dx): # sliceWidth): # make sure we don't have a tiny image on the edge if y0 + sliceHeight > height: y = height - sliceHeight else: y = y0 if x0 + sliceWidth > width: x = width - sliceWidth else: x = x0 # extract image window_c = image0.crop((x, y, x + sliceWidth, y + sliceHeight)) outpath = os.path.join(outdir, out_name + \ slice_sep + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str( sliceWidth) + ext) window_c.save(outpath) return data_image = sorted([ f for f in os.listdir(orig_dir) if (f.endswith('.JPG') or f.endswith('.jpg')) ]) n = len(data_image) # print(origine_slice) files_bbox = sorted( [f for f in os.listdir(orig_dir) if f.endswith('.txt')]) # print(files_bbox) data_txt = load_bbox(files_bbox, orig_dir) for i in range(n): slice_im(data_image[i], data_image[i][0:-4], slice_dir)
def slice_train(orig_dir, slice_dir, sliceHeight=256, sliceWidth=256, zero_frac_thresh=0.2, overlap=0.2, Pobj=0.4, Pimage=0.5, slice_sep='|', out_ext='.jpg', verbose=False): def slice_tot(image_path, bbox_path, out_name, outdir): image_path_tot = os.path.join(orig_dir, image_path) image0 = Image.open( image_path_tot) # cv2.imread(image_path_tot, 1) # color if len(out_ext) == 0: ext = '.' + image_path.split('.')[-1] else: ext = out_ext # width, height = image0.size print(width, height) dx = int((1. - overlap) * sliceWidth) dy = int((1. - overlap) * sliceHeight) # bbox m = len(bbox_path) data_txt_abs = bbox_path for j in range(m): data_txt_abs[j][0] = int(bbox_path[j][0]) # classe data_txt_abs[j][1] = int(bbox_path[j][1] * nW) # x data_txt_abs[j][2] = int(bbox_path[j][2] * nH) # y data_txt_abs[j][3] = int(bbox_path[j][3] * nW) # w data_txt_abs[j][4] = int(bbox_path[j][4] * nH) # h for y0 in range(0, height, dy): #sliceHeight): for x0 in range(0, width, dx): #sliceWidth): # make sure we don't have a tiny image on the edge if y0 + sliceHeight > height: y = height - sliceHeight else: y = y0 if x0 + sliceWidth > width: x = width - sliceWidth else: x = x0 # extract image window_c = image0.crop((x, y, x + sliceWidth, y + sliceHeight)) outpath = os.path.join(outdir, out_name + \ slice_sep + str(y) + '_' + str(x) + '_' + str(sliceHeight) + '_' + str( sliceWidth) + ext) window_c.save(outpath) ##################### slicing bboxes ############################################# x_min_slice = x x_max_slice = x + sliceWidth y_min_slice = y y_max_slice = y + sliceHeight x_slice_abs = x + int(0.5 * sliceWidth) y_slice_abs = y + int(0.5 * sliceHeight) txt_file_name = ("%s.txt" % (outpath[0:-4])) text_file = open(txt_file_name, "w") m = len(data_txt[i]) for j in range(m): X_obj_abs = data_txt_abs[j][1] Y_obj_abs = data_txt_abs[j][2] W_obj_abs = data_txt_abs[j][3] H_obj_abs = data_txt_abs[j][4] object_area_abs = W_obj_abs * H_obj_abs X_min_obj_abs = int(X_obj_abs - 0.5 * W_obj_abs) X_max_obj_abs = int(X_obj_abs + 0.5 * W_obj_abs) Y_min_obj_abs = int(Y_obj_abs - 0.5 * H_obj_abs) Y_max_obj_abs = int(Y_obj_abs + 0.5 * H_obj_abs) Inter = int( Calc_IoU.calc_Inter(X_obj_abs, x_slice_abs, Y_obj_abs, y_slice_abs, W_obj_abs, sliceWidth, H_obj_abs, sliceHeight)) object_covering = float(Inter / object_area_abs) window_covering = float(Inter / win_size) centroid_inside_window = ( (x_min_slice <= X_obj_abs <= x_max_slice) and (y_min_slice <= Y_obj_abs <= y_max_slice)) if (object_covering > Pobj or window_covering > Pimage or centroid_inside_window): x_obj_slice_temp = float( (X_obj_abs - x_min_slice) / sliceWidth) y_obj_slice_temp = float( (Y_obj_abs - y_min_slice) / sliceHeight) w_obj_slice_temp = float(W_obj_abs / sliceWidth) h_obj_slice_temp = float(H_obj_abs / sliceHeight) # print(w_obj_slice_temp, h_obj_slice_temp) x_min_obj_slice = float(x_obj_slice_temp - 0.5 * w_obj_slice_temp) x_max_obj_slice = float(x_obj_slice_temp + 0.5 * w_obj_slice_temp) y_min_obj_slice = float(y_obj_slice_temp - 0.5 * h_obj_slice_temp) y_max_obj_slice = float(y_obj_slice_temp + 0.5 * h_obj_slice_temp) # print("xmin xmax ymin ymax") # print(x_min_obj_slice, x_max_obj_slice, y_min_obj_slice, y_max_obj_slice) if (x_min_obj_slice < 0): x_min_obj_slice = 0 if (y_min_obj_slice < 0): y_min_obj_slice = 0 if (x_max_obj_slice > 1): x_max_obj_slice = 1 if (y_max_obj_slice > 1): y_max_obj_slice = 1 w_obj_slice = float(x_max_obj_slice - x_min_obj_slice) h_obj_slice = float(y_max_obj_slice - y_min_obj_slice) x_obj_slice = float(x_min_obj_slice + 0.5 * w_obj_slice) y_obj_slice = float(y_min_obj_slice + 0.5 * h_obj_slice) #print(x_obj_slice, y_obj_slice, w_obj_slice, h_obj_slice) text_file.write( "%d %4f %4f %4f %4f\n" % (data_txt_abs[j][0], x_obj_slice, y_obj_slice, w_obj_slice, h_obj_slice)) text_file.close() # remove file if empty # print(os.path.getsize(txt_file_name)) if os.stat(txt_file_name).st_size == 0: os.remove(txt_file_name) return data_image = sorted([ f for f in os.listdir(orig_dir) if (f.endswith('.JPG') or f.endswith('.jpg')) ]) n = len(data_image) # print(origine_slice) files_bbox = sorted( [f for f in os.listdir(orig_dir) if f.endswith('.txt')]) # print(files_bbox) data_txt = load_bbox(files_bbox, orig_dir) for i in range(n): slice_tot(data_image[i], data_txt[i], data_image[i][0:-4], slice_dir)
def count_intra_gt(test_im_dir, names_file, print_res=False, edit_res=False, outname='results/object_intra_gt.csv'): # clean data preparation : remove images without labels, txt files that are no label file ... file_bbox = sorted( [f for f in os.listdir(test_im_dir) if f.endswith('.txt')]) images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')]) image_labelled = [] for i in range(len(file_bbox)): root_name_bbx = file_bbox[i].split('.')[0] for image in images: if image.split('.')[0] == root_name_bbx: image_labelled.append(image) image_labelled = sorted(image_labelled) labels_with_images = [] for i in range(len(image_labelled)): root_name_image = image_labelled[i].split('.')[0] for label in file_bbox: if label.split('.')[0] == root_name_image: labels_with_images.append(label) labels_with_images = sorted(labels_with_images) bbox = load_bbox(labels_with_images, test_im_dir) merged_bbox_list = [] for i in range(len(bbox)): for j in range(len(bbox[i])): bbox[i][j].insert(0, image_labelled[i][0:-4]) merged_bbox_list = merged_bbox_list + bbox[i] ground_truth = merged_bbox_list gt_df = pd.DataFrame(ground_truth, columns=('root_image', 'obj_class', 'x', 'y', 'w', 'h')) gt_df['root_image_path'] = 'test_images/' + gt_df['root_image'] + '.JPG' gt_df['obj_class'] = gt_df.obj_class.astype(str) with open(names_file) as f: labels = [line.rstrip('\n') for line in f] for row in gt_df.iterrows(): index, data = row for i in range(len(labels)): if (row[1].obj_class == str(float(i))): gt_df.at[index, 'obj_class'] = labels[i] df = gt_df # I have only copied the previous function, now that df are of the same format # generating template for result df list_of_list = [] for image in df.root_image_path.unique(): date = get_date_taken(image) df_image = df[df.root_image_path == image] for obj_class in df_image.obj_class.unique(): list = [image, date, obj_class, 0] list_of_list.append(list) df_inter = pd.DataFrame(list_of_list, columns=('root_image_path', 'date', 'obj_class', 'count')) # for image in df.root_image_path.unique(): date = get_date_taken(image) df_image = df[df.root_image_path == image] df_inter_image = df_inter[df_inter.root_image_path == image] for classe in df_image.obj_class.unique(): df_class = df_image[df_image.obj_class == classe] for rowA in df_class.iterrows(): xA = rowA[1].x yA = rowA[1].y wA = rowA[1].w hA = rowA[1].h classA = rowA[1].obj_class df_comp = df_class.drop([rowA[0]]) for rowB in df_comp.iterrows(): xB = rowB[1].x yB = rowB[1].y wB = rowB[1].w hB = rowB[1].h classB = rowB[1].obj_class IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB) if IOU > 0: df_inter_bis = df_inter_image[df_inter_image.obj_class == classA] index_to_change = df_inter_bis.index[0] df_inter.at[index_to_change, 'count'] += 1 df_inter.loc[:, 'count'] /= 2 #pd.to_numeric(df_inter.loc[:,'count'], downcast='integer') df_inter = df_inter.sort_values(['root_image_path']) if print_res: print(df_inter) if edit_res: df_inter.to_csv(outname, index=False)
def count_species_gt(test_im_dir, names_file, print_res=False, edit_res=False, outname='results/object_count_gt.csv'): # clean data preparation : remove images without labels, txt files that are no label file ... file_bbox = sorted( [f for f in os.listdir(test_im_dir) if f.endswith('.txt')]) images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')]) image_labelled = [] for i in range(len(file_bbox)): root_name_bbx = file_bbox[i].split('.')[0] for image in images: if image.split('.')[0] == root_name_bbx: image_labelled.append(image) image_labelled = sorted(image_labelled) labels_with_images = [] for i in range(len(image_labelled)): root_name_image = image_labelled[i].split('.')[0] for label in file_bbox: if label.split('.')[0] == root_name_image: labels_with_images.append(label) labels_with_images = sorted(labels_with_images) bbox = load_bbox(labels_with_images, test_im_dir) merged_bbox_list = [] for i in range(len(bbox)): for j in range(len(bbox[i])): bbox[i][j].insert(0, image_labelled[i][0:-4]) merged_bbox_list = merged_bbox_list + bbox[i] ground_truth = merged_bbox_list gt_df = pd.DataFrame(ground_truth, columns=('root_image', 'obj_class', 'x', 'y', 'w', 'h')) gt_df['root_image_path'] = 'test_images/' + gt_df['root_image'] + '.JPG' gt_df['obj_class'] = gt_df.obj_class.astype(str) with open(names_file) as f: labels = [line.rstrip('\n') for line in f] for row in gt_df.iterrows(): index, data = row for i in range(len(labels)): if (row[1].obj_class == str(float(i))): gt_df.at[index, 'obj_class'] = labels[i] list_of_list = [] for image in gt_df.root_image_path.unique(): date = get_date_taken(image) gt_df_image = gt_df[gt_df.root_image_path == image] for classe in gt_df_image.obj_class.unique(): count = 0 gt_df_class = gt_df_image[gt_df_image.obj_class == classe] for row in gt_df_class.iterrows(): count += 1 list = [image, date, classe, count] list_of_list.append(list) gt_df_count = pd.DataFrame(list_of_list, columns=('root_image_path', 'date', 'obj_class', 'count')) if print_res: print(gt_df_count) if edit_res: gt_df_count.to_csv(outname, index=False)
def get_metrics(test_im_dir, detection_csv_file, names_file, examples_file_path, print_res=True, edit_res=False): ################### function to get average precision ################################ def voc_ap(rec, prec): # from https://github.com/Cartucho/mAP/blob/master/main.py """ --- Official matlab code VOC2012--- mrec=[0 ; rec ; 1]; mpre=[0 ; prec ; 0]; for i=numel(mpre)-1:-1:1 mpre(i)=max(mpre(i),mpre(i+1)); end i=find(mrec(2:end)~=mrec(1:end-1))+1; ap=sum((mrec(i)-mrec(i-1)).*mpre(i)); """ rec.insert(0, 0.0) # insert 0.0 at begining of list rec.append(1.0) # insert 1.0 at end of list mrec = rec[:] prec.insert(0, 0.0) # insert 0.0 at begining of list prec.append(0.0) # insert 0.0 at end of list mpre = prec[:] """ This part makes the precision monotonically decreasing (goes from the end to the beginning) matlab: for i=numel(mpre)-1:-1:1 mpre(i)=max(mpre(i),mpre(i+1)); """ # matlab indexes start in 1 but python in 0, so I have to do: # range(start=(len(mpre) - 2), end=0, step=-1) # also the python function range excludes the end, resulting in: # range(start=(len(mpre) - 2), end=-1, step=-1) for i in range(len(mpre) - 2, -1, -1): mpre[i] = max(mpre[i], mpre[i + 1]) """ This part creates a list of indexes where the recall changes matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1; """ i_list = [] for i in range(1, len(mrec)): if mrec[i] != mrec[i - 1]: i_list.append(i) # if it was matlab would be i + 1 """ The Average Precision (AP) is the area under the curve (numerical integration) matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i)); """ ap = 0.0 for i in i_list: ap += ((mrec[i] - mrec[i - 1]) * mpre[i]) return ap, mrec, mpre ########################################################################################################### ######################## Formating both detection and GT data into same template ########################## file_bbox = sorted( [f for f in os.listdir(test_im_dir) if f.endswith('.txt')]) images = sorted([f for f in os.listdir(test_im_dir) if f.endswith('.JPG')]) # sort only images with labels and labels attached to images image_labelled = [] for i in range(len(file_bbox)): root_name_bbx = file_bbox[i].split('.')[0] for image in images: if image.split('.')[0] == root_name_bbx: image_labelled.append(image) image_labelled = sorted(image_labelled) labels_with_images = [] for i in range(len(image_labelled)): root_name_image = image_labelled[i].split('.')[0] for label in file_bbox: if label.split('.')[0] == root_name_image: labels_with_images.append(label) labels_with_images = sorted(labels_with_images) bbox = load_bbox(labels_with_images, test_im_dir) merged_bbox_list = [] for i in range(len(bbox)): for j in range(len(bbox[i])): bbox[i][j].insert(0, images[i][0:-4]) merged_bbox_list = merged_bbox_list + bbox[i] ground_truth = merged_bbox_list detection_csv = detection_csv_file # gt_df stores GT information # dt_df stores detection information gt_df = pd.DataFrame(ground_truth, columns=('root_image', 'obj_class', 'x', 'y', 'w', 'h')) dt_df = pd.read_csv(detection_csv) ######################### convert classes according .names file ################### gt_df['obj_class'] = gt_df.obj_class.astype(str) with open(names_file) as f: labels = [line.rstrip('\n') for line in f] for row in gt_df.iterrows(): index, data = row for i in range(len(labels)): if (row[1].obj_class == str(float(i))): gt_df.at[index, 'obj_class'] = labels[i] ################## compare only images that have associated GT #################### dt_df = dt_df[dt_df.root_image.isin(gt_df.root_image.unique())] ################## get the list of all classes (detected of not) #################### class_list = [] max_name_class_length = 0 for dift_class in gt_df.obj_class.unique(): class_list.append(dift_class) # included by Dom max_name_class_length = max(max_name_class_length, len(dift_class)) # end of included by Dom for dift_class in dt_df.obj_class.unique(): if dift_class not in class_list: class_list.append(dift_class) ########################### generating template dict for per classes metrics ############### mAP_class_dict = {} prec_class_dict = {} for dift_class in class_list: mAP_class_dict[dift_class] = 0 prec_class_dict[dift_class] = 0 ####################### initialize variables ############################# dt_df = dt_df.sort_values(['obj_class']) dt_df['correct'] = 0 dt_df['precision'] = 0.0 dt_df['recall'] = 0.0 gt_df = gt_df.sort_values(['obj_class']) gt_df['used'] = 0 # overlap threshold for acceptance overlap_threshold = 0.5 ####################### comparison of detected objects to GT ################################# for image in dt_df.root_image.unique(): print(image) dt_df_img = dt_df[dt_df['root_image'] == image] gt_df_img = gt_df[gt_df['root_image'] == image] # list different classes present in GT gt_classes = gt_df_img.obj_class.unique() # list out wrong predicted classes dt_df_correct_classes = dt_df_img[dt_df_img['obj_class'].isin( gt_classes)] # for each correct class for dift_class in gt_classes: gt_df_class = gt_df_img[gt_df_img.obj_class == dift_class] dt_df_class = dt_df_correct_classes[dt_df_correct_classes.obj_class == dift_class] for rowA in gt_df_class.iterrows( ): # compare each GT object of this class ... xA = rowA[1].x yA = rowA[1].y wA = rowA[1].w hA = rowA[1].h used = rowA[1].used for rowB in dt_df_class.iterrows( ): # ... with every detected object of this class xB = rowB[1].x yB = rowB[1].y wB = rowB[1].w hB = rowB[1].h IOU = calc_IOU(xA, xB, yA, yB, wA, wB, hA, hB) if IOU > overlap_threshold: # i.e. if detection and GT overlap if used == 0: # gt not found yet indexB, dataB = rowB dt_df.at[indexB, 'correct'] = 1 indexA, dataA = rowA gt_df.at[indexA, 'used'] = 1 df_TP = dt_df[dt_df.correct == 1] df_FP = dt_df[dt_df.correct == 0] df_FN = gt_df[gt_df.used == 0] TP = len(df_TP.correct) FP = len(df_FP.correct) FN = len(df_FN.used) precision_general = float(TP / (TP + FP)) recall_general = float(TP / (TP + FN)) F1_general = 2 * float((precision_general * recall_general) / (precision_general + recall_general)) ########################### mAP ############################# # once we have for each detection its status (TP or FP), we can compute average precision for each class # for this we will need to compute recall and precision for each class # included by Dom gt_classes = gt_df.obj_class.unique() nb_classes = len(gt_classes) df_metrics_classes = pd.DataFrame(pd.np.empty((nb_classes, 10)), columns=[ 'classes', 'precision', 'recall', 'F1', 'mAP', 'GT', 'TP', 'FP', 'FN', 'training_examples' ]) df_metrics_classes['classes'] = gt_classes df_metrics_classes['precision'] = 0 df_metrics_classes['recall'] = 0 df_metrics_classes['F1'] = 0 df_metrics_classes['mAP'] = 0 df_metrics_classes['GT'] = 0 df_metrics_classes['TP'] = 0 df_metrics_classes['FP'] = 0 df_metrics_classes['FN'] = 0 # end of included by Dom df_examples = pd.read_csv(examples_file_path, index_col='name') print('*********') print(df_examples) for dift_class in gt_df.obj_class.unique(): gt_df_class = gt_df[gt_df.obj_class == dift_class] items_to_detect = len(gt_df_class) df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'GT'] = items_to_detect df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'training_examples'] = df_examples.at[ dift_class, 'count'] FN_count = 0 for row in gt_df_class.iterrows(): index, data = row if (data['used'] == 0): FN_count += 1 df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'FN'] = FN_count for dift_class in dt_df.obj_class.unique(): gt_df_class = gt_df[gt_df.obj_class == dift_class] dt_df_class = dt_df[dt_df.obj_class == dift_class] # GT number of objects to find (potential positives = PP) PP = len(gt_df_class) if PP == 0: # if false positive mAP_class_dict[dift_class] = 0 if PP != 0: TP_count = 0 FP_count = 0 row_count = 0 # (for recall) for row in dt_df_class.iterrows(): row_count += 1 index, data = row if (data['correct'] == 1): TP_count += 1 else: FP_count += 1 # if (data['used'] == 0): # FN_count +=1 recall = float(TP_count / PP) #print('recall = ', recall) precision = float(TP_count / row_count) #print('precision = ', precision) dt_df_class.at[index, 'recall'] = recall dt_df_class.at[index, 'precision'] = precision prec = dt_df_class['precision'].tolist() rec = dt_df_class['recall'].tolist() # included by Dom avg_prec_class = mean(prec) if avg_prec_class != 0 and recall != 0: F1_score_class = 2 * float( (avg_prec_class * recall) / (avg_prec_class + recall)) else: F1_score_class = 0 df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'precision'] = avg_prec_class df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'recall'] = recall df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'F1'] = F1_score_class df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'TP'] = TP_count df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'FP'] = FP_count # end of included by Dom ap, mrec, mprec = voc_ap( rec, prec) # according previously defined function mAP_class_dict[dift_class] += ap # this is just a fix for some issues pandas can encounter with .from_dict() function # but sometimes, it does work perfectly fine. new_dict = {k: [v] for k, v in mAP_class_dict.items()} temp_df = pd.DataFrame(new_dict, index=['mAP']) mAP_class_df = temp_df.T df_metrics_classes.training_examples = df_metrics_classes.training_examples.astype( int) for dift_class in dt_df.obj_class.unique(): df_metrics_classes.loc[df_metrics_classes.classes == dift_class, 'mAP'] = mAP_class_df.at[dift_class, 'mAP'] # mAP_class_df = pd.DataFrame.from_dict(mAP_class_dict, orient = 'index', columns = ['mAP']) mAP = float(float(sum(mAP_class_dict.values())) / float(len(class_list))) if print_res: print('\n metrics per class_______________________\n') print(df_metrics_classes) print('\n general metrics _________________\n') print('mAP: \t\t%.4f' % mAP) print('precision: \t%.4f' % precision_general) print('recall: \t%.4f' % recall_general) print('F1: \t\t%.4f' % F1_general) df_metrics_classes.to_csv('test_temp/metrics_per_class.ods', index=False) df_metrics_classes.to_csv('test_temp/metrics_per_class.csv', index=False) df_general_metrics = pd.DataFrame( pd.np.empty((1, 4)), columns=['mAP', 'precision', 'recall', 'F1']) df_general_metrics['mAP'] = mAP df_general_metrics['precision'] = precision_general df_general_metrics['recall'] = recall_general df_general_metrics['F1'] = F1_general df_general_metrics.to_csv('test_temp/general_metrics.csv', index=False) df_general_metrics.to_csv('test_temp/general_metrics.ods', index=False) #################### edit results to csv file #modified by Dom; variables were not defined; maybe to delete? if edit_res: res_list = [['mAP', mAP], ['precision', precision_general], ['recall', recall_general], ['F1', F1_general]] res_df = pd.DataFrame(res_list, columns=['metric', 'value']) # print(res_df.T) # res_df.to_csv('test_temp/general_metrics.csv', index=False) # res_df.to_csv('test_temp/general_metrics.ods', index=False) return TP, FP, FN, precision_general, recall_general, F1_general, mAP, mAP_class_dict