def main(): # ---------- Input all the arguments ----------- % parser = argparse.ArgumentParser( parents=[common.capture_spec_parser, db_export.parser, process_export.parser], add_help=False) parser.add_argument("detection_type", choices=['faces', 'plates']) my_parser = argparse.ArgumentParser( parents=[parser]) my_parser.add_argument("--scaling", type=float, default=1, help=("rescaling of bounding boxes")) my_parser.add_argument("--save", type=float, default=1, help=("rescaling of bounding boxes")) # For training the classifier my_parser.add_argument("--threshold-lower", type=float, default=threshold_lower_dft, help=("lower bound of overlap (FP)")) my_parser.add_argument("--threshold", type=float, default=threshold_dft, help=("overlap threshold (fraction) " "for a hit")) my_parser.add_argument("--op-th", type=float, default=op_th_dft, help=("roc operating threshold")) my_parser.add_argument("--op-th-validation", type=float, default=op_th_dft_validation, help=("roc operating threshold, validation")) my_parser.add_argument("--min-size", type=int, default=min_size_dft, help=("minimum labeled box size in pixels")) my_parser.add_argument("--save_detected", default=True) my_parser.add_argument("--label-type") # agruments for color labeling of hits/ misses / false alarms my_parser.add_argument("--b-label-raster", default=False) my_parser.add_argument("--data-to-label-directory", help=("directory of clean or blurred rasters to label/crop")) # arguments for saving hits/ misses / false alarms # hits / misses / flase alarms will be cropped # from the "--rasters-to-label-directory" my_parser.add_argument("--b-save-boxes", default=False) my_parser.add_argument("--b-crop-labeled-raster", default=False) args = my_parser.parse_args() # detection type and parent box type detection_type = args.detection_type parent_map = {'faces':'people', 'plates':'cars'} parent_type = parent_map[detection_type] label_type = args.label_type if not label_type: label_type = 'clear_'+detection_type if not args.data_to_label_directory: data_to_label_directory = args.data_directory; # read labels and detections capture_ids = common.get_capture_ids(**args.__dict__) (detected_boxes_dict_in, detected_parent_boxes_dict, labeled_boxes_dict) = get_boxes_simplified( args.db_filename, args.data_directory, capture_ids, label_type, detection_type, parent_type, args.x_stride, args.y_offset, 1/args.scaling) labeled_boxes_dict_all = labeled_boxes_dict ''' (dmmy2, dmmy1, labeled_boxes_dict_all) = get_boxes_simplified( args.db_filename, args.data_directory, capture_ids, 'all', detection_type, parent_type, args.x_stride, args.y_offset,1) ''' #print detected_boxes_dict_in # run for a number of thresholds to generate ROC curve score_th = numpy.arange(-12, 4, 1)/10.0; score_th = numpy.append(score_th, args.op_th) score_th.sort() # Commented on 6/31 by tchen, since not using different threshold for validation # score_th_v = numpy.arange(0, 6 , 1)/10.0; # score_th_v = score_th_v if numpy.any(score_th_v == args.op_th_validation) \ # else numpy.append(score_th_v, args.op_th_validation) # score_th_v.sort() score_th_v = numpy.arange(0, 1 , 1); num_tp = numpy.zeros(score_th.size) num_fn = numpy.zeros(score_th.size) num_fp = numpy.zeros(score_th.size) recall = numpy.zeros(score_th.size) precision = numpy.zeros(score_th.size) fpr = numpy.zeros(score_th.size) pixel_fpr = numpy.zeros(score_th.size) # Save for score dictionary, added by tchen keys = detected_boxes_dict_in.keys() keys.sort() score_dict = dict() score_min = -1.2 for capture_id in keys: for box in detected_boxes_dict_in[capture_id]: if float(box[4]) > score_min: score_dict["{:s} {:s} {:s} {:s} ".format(*box)] = box[4] for j in range(score_th_v.size): for i in range(score_th.size): # get detected boxes with score above the threshold (detected_boxes_dict) = get_boxes_with_score_above_th(detected_boxes_dict_in, score_th[i], score_th_v[j]) # split labled boxes into hits (tp) and misses(fn) # split detected boxes into hits(tp_db) and false_alarms(fp_db) b_save_detected_txt = False ## Not sure why/when to set this to true #if score_th[i] == args.op_th: # b_save_detected_txt = True (tp, fn, tp_db, fp_db) = evaluate_detections_all(detected_boxes_dict, labeled_boxes_dict, labeled_boxes_dict_all, args.data_directory, args.detection_type, b_save_detected_txt, args.threshold, args.threshold_lower) # Save label for TP/FP, added by tchen on 7/10 # add scaling on 7/21 if (i == 0) and (args.save_detected): # driveid = args.db_filename[-20:-4] driveid = (args.db_filename.split('/'))[-1][:-4] # labelfname = args.data_directory[0:-36] + driveid + '/' + args.data_directory[-36:-32] + '_' + detection_type + '_TP' + '.txt' directory = args.data_directory.split('/') for i in range(0, len(directory)-1): labelfile = file(labelfname, "w") save_detected_boxes(driveid, labelfname, tp_db, args.scaling) labelfname = args.data_directory[0:-36] + driveid + '/' + args.data_directory[-36:-32] + '_' + detection_type + '_FP' + '.txt' save_detected_boxes(driveid, labelfname, fp_db, args.scaling) ''' for capture_id in tp_db: for bbox in tp_db[capture_id]: labelfile.write(capture_id + " ") tmpstr = "" for row in bbox: tmpstr = tmpstr + str("{:0.0f} {:0.0f} ".format(*row)) words = tmpstr[:-1].split() xc = (int(words[0]) + int(words[2]))/2 yc = (int(words[1]) + int(words[3]))/2 xwid = xc - int(words[0]) ywid = yc - int(words[1]) x0 = int(max(xc-scale*xwid, 1)) y0 = int(max(yc-scale*ywid, 1)) x1 = int(min(xc+scale*xwid, 8192)) y1 = int(min(yc+scale*ywid, 1320)) tmpstr = str(x0)+" "+str(y0)+" "+str(x1)+" "+ str(y1)+" " + score_dict[tmpstr] labelfile.write(tmpstr) labelfile.write("\n") labelfile.close() labelfname = args.data_directory[0:-36] + driveid + '/' + args.data_directory[-36:-32] + '_' + detection_type + '_FP' + '.txt' labelfile = file(labelfname, "w") for capture_id in fp_db: for bbox in fp_db[capture_id]: labelfile.write(capture_id + " ") tmpstr = "" for row in bbox: tmpstr = tmpstr + str("{:0.0f} {:0.0f} ".format(*row)) words = tmpstr[:-1].split() xc = (int(words[0]) + int(words[2]))/2 yc = (int(words[1]) + int(words[3]))/2 xwid = xc - int(words[0]) ywid = yc - int(words[1]) x0 = int(max(xc-scale*xwid, 1)) y0 = int(max(yc-scale*ywid, 1)) x1 = int(min(xc+scale*xwid, 8192)) y1 = int(min(yc+scale*ywid, 1320)) tmpstr = str(x0)+" "+str(y0)+" "+str(x1)+" "+ str(y1)+" " + score_dict[tmpstr] labelfile.write(tmpstr) labelfile.write("\n") labelfile.close() ''' # get ROC stats fname=args.data_directory[:-9]+'/'+args.detection_type+\ '_per_raster_thval'+str(score_th_v[j])+ \ 'th_' + str(score_th[i]) +'.txt' (num_tp[i], num_fn[i], num_fp[i], area_fa) = privacy_evaluation_summary( tp, fn, tp_db, fp_db, fname, detected_boxes_dict.keys(), args.min_size); recall[i] = num_tp[i] / (num_tp[i] + num_fn[i]) * 100.0 precision[i] = num_tp[i] / (num_tp[i] + num_fp[i]) * 100.0 fpr[i] = num_fp[i] / (num_tp[i] + num_fn[i]) pixel_fpr[i]= area_fa/len(labeled_boxes_dict.keys())/(4096*8192)*100.0 # label /crop/ save if args.b_save_boxes \ and score_th[i] == args.op_th \ and score_th_v[j] == args.op_th_validation: save_detections(data_to_label_directory, args.data_directory, detected_boxes_dict.keys(), tp, fn, tp_db, fp_db, args.b_crop_labeled_raster) if args.b_label_raster \ and score_th[i] == args.op_th \ and score_th_v[j] == args.op_th_validation: save_labeled_raster(data_to_label_directory, args.data_directory, detected_boxes_dict.keys(), [args.op_th, args.op_th_validation], tp, fn, tp_db, fp_db) # save ROC stats in file # fname=args.data_directory[:-36]+'/'+detection_type+'_roc_validation_th_' + str(score_th_v[j]) +'.txt' driveid = args.db_filename[-20:-4] fname = args.data_directory[0:-36] + driveid + '/' + args.data_directory[-36:-32] + '_' + detection_type + '_roc' + '.txt' # fname=args.data_directory[:-4]+'_'+detection_type+'_roc'+'.txt' names =numpy.array(['#threshold', 'recall[%]', 'FPR', 'pixel FPR[%]', 'precision[%]', 'TP', 'FN', 'FP']); table_data =numpy.array([score_th, recall, fpr, pixel_fpr, precision, num_tp, num_fn, num_fp]) save_roc_file(fname, names, table_data.transpose(), args, score_th_v[j]) # print on screen if desired if 1 or score_th_v[j] == args.op_th_validation: f = open(fname, 'r') print(f.read()) f.close() # -------------------------------------------------------------------------- # Label/crop/save routines # ------------------------------------------------------------------------- def draw_rect(image, box, color_rgb): ((x0, y0), (x1, y1)) = box if x0<x1 and y0<y1 and x0>=0 and y0>=0 and x1<8192 and y1<4096 : color_bgr = color_rgb[::-1] image[[y0, y1-1], x0:x1] = color_bgr image[y0:y1, [x0, x1-1]] = color_bgr return 1 else : print y0, y1, x0, x1 return 0 def save_box(image, boxn, fname): ((x0, y0), (x1, y1)) = boxn if y1-y0 > 8 and x1-x0 >8 and \ x0>8 and y0>8 and x0<8192-8 and y1<4096-8: patch = image[y0:y1, x0:x1,:]; ret = cv2.imwrite(fname, patch) def expand_box(box, scale_x, scale_y, max_x, max_y,): ((x0, y0), (x1, y1)) = box width = x1-x0 height = y1 - y0 center_x = x0 + width/2.0 center_y = y0 + height/2.0 boxn = box; boxn[0][0] = max(0, center_x - width/2.0*scale_x) boxn[1][0] = min(max_x, center_x + width/2.0*+scale_x) boxn[0][1] = max(0, center_y - height/2.0*scale_y) boxn[1][1] = min(max_y, center_y + height/2.0*scale_y) return boxn def crop_and_save_boxes(boxes, image, dir, capture_id, expand_rx, expand_ry): id = 1; for box in boxes: if expand_rx == 1 and expand_ry == 1: boxn = box; else: boxn = expand_box(box, expand_rx, expand_ry, image.shape[1], image.shape[0]) fname = dir + "%05d_%04d.png" % (capture_id, id) save_box(image, boxn, fname) id = id+1; def label_raster(boxes, image, clr): num = 0; for box in boxes: num = num + draw_rect(image, box, clr) return num def save_detections(data_directory, save_directory, keys, tp, fn, tp_db, fp_db, b_crop_labeled_raster): hits_dir = save_directory+"/hits/" miss_dir = save_directory+"/missed/" fas_dir = save_directory+"/false_alarms/" if os.path.isdir(hits_dir) is False: os.mkdir(hits_dir) if os.path.isdir(miss_dir) is False: os.mkdir(save_directory+"/missed/") if os.path.isdir(fas_dir) is False: os.mkdir(fas_dir) keys.sort() for capture_id in keys: image_fname =os.path.join(data_directory, "%06d" % capture_id, "raster.jpg") if os.path.exists(image_fname) is False: continue; image = cv2.imread(image_fname) if b_crop_labeled_raster is True: label_raster(tp_db[capture_id], image, (0, 255, 0)); # hits - green label_raster(fn[capture_id], image, (255, 0, 0)); # missed - red label_raster(fp_db[capture_id], image, (0, 255, 255)); # false alarms - blue # Save detected boxes marked as hits crop_and_save_boxes(tp_db[capture_id], image, hits_dir, capture_id, 1.0 , 1.0) # Draw/save labeled boxes marked missed crop_and_save_boxes(fn[capture_id], image, miss_dir, capture_id, 1.0 , 1.0) # Draw/save labeled boxes marked false alarms crop_and_save_boxes(fp_db[capture_id], image, fas_dir, capture_id, 1.0 , 1.0) def save_labeled_raster(data_directory, save_directory, keys, op_th, tp, fn, tp_db, fp_db): keys.sort() for capture_id in keys: image_fname =os.path.join(data_directory, "%06d" % capture_id, "raster.jpg") if os.path.exists(image_fname) is False: continue; image = cv2.imread(image_fname) num_tp =label_raster(tp_db[capture_id], image, (0, 255, 0)); #hits - green num_fn =label_raster(fn[capture_id], image, (255, 0, 0)); #missed - red num_fp =label_raster(fp_db[capture_id], image, (0, 255, 255)); #false alarms - blue num_xx =label_raster(tp[capture_id], image, (255, 255, 0)); #hits, as labeled - yellow text = "obj_hit = %d, obj miss = %d, obj false = %d, op_th_det =%.2f, op_th_val = %.2f " \ % (num_xx, num_fn, num_fp, op_th[0], op_th[1] ) cv2.putText(image, text, (3500,100), cv2.FONT_HERSHEY_SIMPLEX, 3, (0,0,0),thickness=4); if data_directory == save_directory: save_filename = os.path.join(save_directory, "%06d" % capture_id, "raster_labeled.jpg") else: save_filename = os.path.join(save_directory, "%06d" % capture_id, "raster.jpg") print save_filename ret = cv2.imwrite(save_filename, image) # -------------------------------------------------------------------------- # Hits, misses and false alarms counting # ------------------------------------------------------------------------- def privacy_evaluation_summary(tp, fn, tp_db, fp, fname, keys, min_size=min_size_dft): ''' Count true positives, false negatives, and false positives for a set of captures. min_size is the minimum number of pixels in a labeled box for it to be considered. ''' # Don't save the raster files # rfile= file( fname, "w" ) # rfile.write( "#" + "-"*70 +"#\n" ) # names =["#Capture ID", "TP(#hits) ", "FN(#miss)", "FA(#fa)", "pixel FPR[%]"] # rfile.write("{: >12} {: >12} {: >12} {: >12} {: >16} ".format(*names)) # rfile.write( "\n#" + "-"*70 +"#\n" ) num_tp = 0 num_fn = 0 num_fp = 0 area_fa = 0; keys.sort() for capture_id in keys: # object box stats ntp = sum(1 for t in tp[capture_id] if area(t) >= min_size) nfn = sum(1 for t in fn[capture_id] if area(t) >= min_size) nfp = len(fp[capture_id]) fa_area = sum(area(t) for t in fp[capture_id]) area_fa += fa_area; num_tp += ntp num_fn += nfn num_fp += nfp # record #data = numpy.array([capture_id, ntp, nfn, nfp, fa_area/(4096*8192/100)]) #rfile.write("{:s} {:s} {:s} {:s} {:s} ".format(*data)) # rfile.write("{:12.0f} {:12.0f} {:12.0f} {:12.0f} {:16.2f} ".format(*data)) #rfile.write("\n") #rfile.close() return (num_tp, num_fn, num_fp, area_fa) def evaluate_detections_all(detected_boxes_dict, labeled_boxes_dict, labeled_boxes_dict_all, data_directory, detection_type, b_save_detected_txt, threshold=threshold_dft, threshold_lower=threshold_lower_dft): ''' Compute true positives, false negatives, and false positives for a set of captures. detected_boxes_dict and labeled_boxes_dict must have the same set of keys. ''' detection_capture_ids = set(detected_boxes_dict.keys()) label_capture_ids = set(labeled_boxes_dict.keys()) label_capture_ids_all = set(labeled_boxes_dict_all.keys()) if label_capture_ids ^ detection_capture_ids: raise ValueError("Detection and label capture-ID-sets differ") true_positives = {} false_negatives = {} true_positives_db = {} false_positives = {} for capture_id in label_capture_ids: if b_save_detected_txt is True: fname = data_directory+"/%s"%(capture_id) +'/detected_'+ detection_type+'_labeled.txt' fnameL= data_directory+"/%s"%(capture_id) +'/labeled_'+ detection_type+'.txt' # fname = data_directory+"/%06d"%(capture_id) +'/detected_'+ detection_type+'_labeled.txt' # fnameL= data_directory+"/%06d"%(capture_id) +'/labeled_'+ detection_type+'.txt' else: fname=[] fnameL=[] (tp, fn, tp_db, fp_tmp) = evaluate_detections_one_capture( detected_boxes_dict[capture_id], labeled_boxes_dict[capture_id], fname, fnameL, threshold) (tp_tmp, fn_tmp, tp_db_tmp, fp) = evaluate_detections_one_capture( detected_boxes_dict[capture_id], labeled_boxes_dict_all[capture_id], fname, fnameL, threshold, threshold_lower) true_positives[capture_id] = tp false_negatives[capture_id] = fn true_positives_db[capture_id] = tp_db false_positives[capture_id] = fp return (true_positives, false_negatives, true_positives_db, false_positives) def evaluate_detections_one_capture(detected_boxes, labeled_boxes, fname, fnameL, threshold=threshold_dft, threshold_lower=threshold_lower_dft): ''' Compute true positives, false negatives, and false positives. `threshold' is what fraction of a labeled box must be detected in order for it to count as a true positive. The return value is (true_positives, false_negatives, false_positives). ''' detected_boxes = numpy.array(detected_boxes) labeled_boxes = numpy.array(labeled_boxes, dtype=int) # get the coordinates only if len(detected_boxes)>0: detected_boxes_coor = detected_boxes[:,:4].astype(int) else: detected_boxes_coor = numpy.array([], dtype=int) detected_boxes_coor = detected_boxes_coor.reshape(-1,2,2) labeled_boxes = labeled_boxes.reshape(-1,2,2) labeled_sizes = area(labeled_boxes) overlaps = compute_all_box_overlaps(detected_boxes_coor, labeled_boxes) hit = overlaps/labeled_sizes.astype(float) >= threshold is_true_positive = numpy.logical_or.reduce(hit, axis=0) # The separate clauses are needed due to a bug in numpy that # prevents an array of length zero from being indexed *at all* (even # by an empty list) if len(labeled_boxes) > 0: true_positives = labeled_boxes[is_true_positive] false_negatives = labeled_boxes[is_true_positive ^ True] else: true_positives = labeled_boxes false_negatives = labeled_boxes # For getting different FP hit = overlaps/labeled_sizes.astype(float) >= threshold_lower is_false_positive = numpy.logical_or.reduce(hit, axis=1) ^ True if len(detected_boxes) > 0: false_positives = detected_boxes_coor[is_false_positive] true_positives_db = detected_boxes_coor[is_false_positive ^ True] else: false_positives = detected_boxes_coor true_positives_db = detected_boxes_coor # save detected boxes as labeled if len(fname)>0: if len(detected_boxes)>0: labels = numpy.ones((len(detected_boxes),1)) labels[is_false_positive]=-1 detected_boxes = numpy.concatenate((labels, detected_boxes),axis=1) rfile= file( fname, "w" ) for row in detected_boxes: data = tuple("{0:.0f}".format(el) for el in row[:5])+\ tuple("{0:.2f}".format(el) for el in row[5:]) rfile.write("\t".join(data)) rfile.write("\n") rfile.close() if len(labeled_boxes)>0: rfile= file( fnameL, "w" ) for row in labeled_boxes: row=numpy.concatenate((numpy.array([1]),row.flatten()),axis=1) data = tuple("{0:.0f}".format(el) for el in row[:5])+\ tuple("{0:.2f}".format(el) for el in row[5:]) rfile.write("\t".join(data)) rfile.write("\n") rfile.close() # true_positives - labeled boxes, marked as hits # false_negatives - labeled boxes, marked as missed # true_positives_db - detected boxes, marked as hits # false_negatives - detected boxes, marked as false alarms return (true_positives, false_negatives, true_positives_db, false_positives) def compute_all_box_overlaps(boxes_A, boxes_B): ''' Given two arrays of boxes (shape (m, 2, 2) and (n, 2, 2)), compute the overlap size of each pair in the cartesian product. The return value has shape (m, n). ''' dim = (numpy.minimum(boxes_A[:,None,1,:], boxes_B[None,:,1,:]) - numpy.maximum(boxes_A[:,None,0,:], boxes_B[None,:,0,:])) numpy.maximum(dim, 0, dim) return dim.prod(axis=2) def area(box): ''' Return the area of a box [(x0, y0), (x1, y1)]. Vectorized. ''' box.reshape(-1,2,2) area = numpy.diff(box, axis=-2).prod(axis=-1) return area.reshape(area.shape[:-1]) def reduce_to_intersection(dict_A, dict_B): ''' Given two dicts, remove all entries whose keys are not common to both. ''' keys_A = set(dict_A.iterkeys()) keys_B = set(dict_B.iterkeys()) keys = keys_A & keys_B del_A = keys_A ^ keys del_B = keys_B ^ keys for key in del_A: del dict_A[key] for key in del_B: del dict_B[key] #----------------------------------------------------------------- # Load detection with desired threshold #------------------------------------------------------------------ ''' Modified by tchen on 6/30: No parent score (i.e. no box[5]) ''' def get_boxes_with_score_above_th(detected_boxes_dict, threshold_d, threshold_v): keys = detected_boxes_dict.keys() keys.sort() detected_boxes_dict_out = dict() for capture_id in keys: detected_boxes_dict_out[capture_id] = [] boxes_out = [] for box in detected_boxes_dict[capture_id]: if float(box[4])> threshold_d: boxes_out.append(box) #if len(box)>5: # if box[5] > threshold_v: # boxes_out.append(box) # else: # boxes_out.append(box) # some development code - intentionally disabled for box in boxes_out: detected_boxes_dict_out[capture_id].append(box) return detected_boxes_dict_out # -------------------------------------------------------------------------- # Load detected boxes and labeled boxes data # ------------------------------------------------------------------------- def get_boxes_simplified(db_filename, data_directory, capture_ids, label_type, detection_type, parent_type, x_stride=db_export.x_stride_dft, y_offset=db_export.y_offset_dft, scale=1): ''' Simple work flow for finding all the detected and labeled boxes given a data directory and database file. ''' ''' Modified codes for loading "detected_boxes_dic", "labeled_boxes_dic" on 6/30 by tchen. detected_boxes_dic : @ Ln 640 labeled_boxes_dic : boxes_from_txt() @ Ln 650 ''' if detection_type is None: detection_type = label_type # Now don't need "label_capture_ids", by tchen on 6/30 label_capture_ids = None ''' if capture_ids is not None: detection_capture_ids = capture_ids label_capture_ids = capture_ids else: detection_capture_ids = process_export.find_capture_ids( data_directory) label_capture_ids = None ''' # Added load_detected_boxes function by tchen on 6/30 # driveid = "HT068_1380264747" driveid = db_filename[-20:-4] detected_boxes_dict = load_detected_boxes(driveid, data_directory, scale) # detected_boxes_dict = dict(load_detected_boxes(driveid, data_directory)) detected_parent_boxes_dict = detected_boxes_dict ''' detected_boxes_dict = dict(process_export.detected_boxes( data_directory, detection_type, detection_capture_ids)) detected_parent_boxes_dict = dict(process_export.detected_boxes( data_directory, parent_type, detection_capture_ids)) ''' if 1: if db_filename[-3:] == "txt": labeled_boxes_dict = boxes_from_txt( db_filename, label_type, label_capture_ids) else: labeled_boxes_dict = db_export.boxes_from_db( db_filename, label_type, x_stride, y_offset, label_capture_ids) # a development option to read clear_lables and all_labels if 0: labeled_boxes_dict = db_export.boxes_from_db( db_filename, 'all_plates', x_stride, y_offset, label_capture_ids) labeled_boxes_dict tmpdict = db_export.boxes_from_db( db_filename, 'clear_plates', x_stride, y_offset, label_capture_ids) for key in tmpdict: if key in labeled_boxes_dict: labeled_boxes_dict[key] += tmpdict[key] else: labeled_boxes_dict[key] = tmpdict[key] if capture_ids is None: reduce_to_intersection(detected_boxes_dict, labeled_boxes_dict) return detected_boxes_dict,detected_parent_boxes_dict, labeled_boxes_dict # Added by tchen on 6/30 def load_detected_boxes(driveid, data_directory,scale): ''' "data_directory" here is a .txt file with all the detection results. Return the detected bbox. The file format of the detection result is: [driveid captureid score x0 y0 x1 y1] ''' f = open(data_directory) all_boxes = {} for line in f: words = line[:-1].split() if (words[0] == driveid): if words[1] not in all_boxes: all_boxes[words[1]] = [] xc = (int(words[3]) + int(words[5]))/2 yc = (int(words[4]) + int(words[6]))/2 xwid = xc - int(words[3]) ywid = yc - int(words[4]) x0 = int(max(xc-scale*xwid, 1)) y0 = int(max(yc-scale*ywid, 1)) x1 = int(min(xc+scale*xwid, 8192)) y1 = int(min(yc+scale*ywid, 1320)) all_boxes[words[1]].append((str(x0), str(y0), str(x1), str(y1), words[2])) return all_boxes def boxes_from_txt(db_filename, detection_type=None, capture_ids=None): ''' Return the training bounding boxes of the specified type from the given database file. The return value is a dictionary { capture_id: [(x0, y0, x1, y1), ...], ... }. ''' ''' Modified on 6/29 by tchen ''' all_boxes = {} # for capture_id in all_captures: # for f in listdir(db_filename) # Check the drive id f = open (db_filename, 'r') for line in f: words = line[:-1].split() capture_id = words[0] x0 = int(words[1]) y0 = int(words[2]) x1 = int(words[3]) y1 = int(words[4]) if capture_id not in all_boxes: all_boxes[capture_id] = [] all_boxes[capture_id].append((x0,y0,x1,y1)) return all_boxes #-------------------------------------------------------------------------------- # SAVE #-------------------------------------------------------------------------------- def save_detected_boxes(driveid, labelfname, printdict, scale): labelfile = file(labelfname, "w") for capture_id in printdict: for bbox in printdict[capture_id]: labelfile.write(capture_id + " ") tmpstr = "" for row in bbox: tmpstr = tmpstr + str("{:0.0f} {:0.0f} ".format(*row)) words = tmpstr[:-1].split() xc = (int(words[0]) + int(words[2]))/2 yc = (int(words[1]) + int(words[3]))/2 xwid = xc - int(words[0]) ywid = yc - int(words[1]) x0 = int(max(xc-scale*xwid, 1)) y0 = int(max(yc-scale*ywid, 1)) x1 = int(min(xc+scale*xwid, 8192)) y1 = int(min(yc+scale*ywid, 1320)) tmpstr = str(x0)+" "+str(y0)+" "+str(x1)+" "+ str(y1)+" " + score_dict[tmpstr] labelfile.write(tmpstr) labelfile.write("\n") labelfile.close() def save_roc_file(fname, names, table_data, args, op_th_v): line="#"+"-"*150+"\n" rfile= file( fname, "w" ) rfile.write(line) rfile.write("# Data : " + args.data_directory+"\n") rfile.write("# Labels: " + args.db_filename+"\n") rfile.write("# Type : " + args.detection_type+"\n") rfile.write("# Validation thrshold:" + str(op_th_v) +"\n") rfile.write("# Number of labels: %d \n"% (table_data[1,5]+ table_data[1,6])) rfile.write(line) rfile.write( "{: >16} {: >16} {: >16} {: >16} {: >16} {: >16} {: >16} {: >16}".format(*names)) rfile.write("\n") rfile.write(line) for row in table_data: rfile.write("{:16.2f} {:16.2f} {:16.2f} {:16.2f} {:16.2f} {:16.0f} {:16.0f} {:16.0f} ".format(*row)) rfile.write("\n") rfile.write("# LEGEND:\n") rfile.write(line), rfile.write("# Detection rule: the label box covered by at least %2.1f%%\n"% (args.threshold*100)) rfile.write("# True positives, i.e. hits: TP\n") rfile.write("# False negatives, i.e. misses: FN\n") rfile.write("# False positives, i.e. false alarms: FP\n") rfile.write("# Recall: TP/(TP+FN)\n") rfile.write("# Precision: TP/(TP+FP)\n") rfile.write("# FPR (false positive rate): FP/(TP+FN)\n") rfile.write("# Pixel FPR (upper bound, does not discount box overlap) : (# false positive pixels) / (# pixels in raster)\n") rfile.close() if __name__ == '__main__': main()
parser.add_argument("data_directory", help=("directory containing all panorama data")) if __name__ == "__main__": my_parser = argparse.ArgumentParser( parents=[common.capture_spec_parser, parser]) my_parser.add_argument("detection_type", choices=["faces", "plates", "people", "cars"]) my_parser.add_argument("--header-format", default="{capture_id:06d}", help=("format string for header field (default " "is \"{capture_id:06d}\" for " "000001, 000002, etc.)")) args = my_parser.parse_args() capture_ids = common.get_capture_ids(**args.__dict__) if capture_ids is None: capture_ids = find_capture_ids(args.data_directory) for (capture_id, boxes) in detected_boxes(args.data_directory, args.detection_type, capture_ids): for box in boxes: print( args.header_format.format(capture_id=capture_id) + " " + " ".join(map(str, box)))
def main(): # ---------- Input all the arguments ----------- % parser = argparse.ArgumentParser(parents=[ common.capture_spec_parser, db_export.parser, process_export.parser ], add_help=False) parser.add_argument("detection_type", choices=['faces', 'plates']) my_parser = argparse.ArgumentParser(parents=[parser]) my_parser.add_argument("--threshold", type=float, default=threshold_dft, help=("overlap threshold (fraction) " "for a hit")) my_parser.add_argument("--op-th", type=float, default=op_th_dft, help=("roc operating threshold")) my_parser.add_argument("--op-th-validation", type=float, default=op_th_dft_validation, help=("roc operating threshold, validation")) my_parser.add_argument("--min-size", type=int, default=min_size_dft, help=("minimum labeled box size in pixels")) my_parser.add_argument("--label-type") # agruments for color labeling of hits/ misses / false alarms my_parser.add_argument("--b-label-raster", default=False) my_parser.add_argument( "--data-to-label-directory", help=("directory of clean or blurred rasters to label/crop")) # arguments for saving hits/ misses / false alarms # hits / misses / flase alarms will be cropped # from the "--rasters-to-label-directory" my_parser.add_argument("--b-save-boxes", default=False) my_parser.add_argument("--b-crop-labeled-raster", default=False) args = my_parser.parse_args() # detection type and parent box type detection_type = args.detection_type parent_map = {'faces': 'people', 'plates': 'cars'} parent_type = parent_map[detection_type] label_type = args.label_type if not label_type: label_type = 'clear_' + detection_type if not args.data_to_label_directory: data_to_label_directory = args.data_directory # read labels and detections capture_ids = common.get_capture_ids(**args.__dict__) (detected_boxes_dict_in, detected_parent_boxes_dict, labeled_boxes_dict) = get_boxes_simplified(args.db_filename, args.data_directory, capture_ids, label_type, detection_type, parent_type, args.x_stride, args.y_offset) (dmmy2, dmmy1, labeled_boxes_dict_all) = get_boxes_simplified( args.db_filename, args.data_directory, capture_ids, 'all', detection_type, parent_type, args.x_stride, args.y_offset) #print detected_boxes_dict_in # run for a number of thresholds to generate ROC curve score_th = numpy.arange(-12, 4, 1) / 10.0 score_th = numpy.append(score_th, args.op_th) score_th.sort() score_th_v = numpy.arange(0, 6, 1) / 10.0 score_th_v = score_th_v if numpy.any(score_th_v == args.op_th_validation) \ else numpy.append(score_th_v, args.op_th_validation) score_th_v.sort() num_tp = numpy.zeros(score_th.size) num_fn = numpy.zeros(score_th.size) num_fp = numpy.zeros(score_th.size) recall = numpy.zeros(score_th.size) precision = numpy.zeros(score_th.size) fpr = numpy.zeros(score_th.size) pixel_fpr = numpy.zeros(score_th.size) for j in range(score_th_v.size): for i in range(score_th.size): # get detected boxes with score above the threshold (detected_boxes_dict) = get_boxes_with_score_above_th( detected_boxes_dict_in, score_th[i], score_th_v[j]) # split labled boxes into hits (tp) and misses(fn) # split detected boxes into hits(tp_db) and false_alarms(fp_db) b_save_detected_txt = False if score_th[i] == args.op_th: b_save_detected_txt = True (tp, fn, tp_db, fp_db) = evaluate_detections_all( detected_boxes_dict, labeled_boxes_dict, labeled_boxes_dict_all, args.data_directory, args.detection_type, b_save_detected_txt, args.threshold) # get ROC stats fname=args.data_directory+'/'+args.detection_type+\ '_per_raster_thval'+str(score_th_v[j])+ \ 'th_' + str(score_th[i]) +'.txt' (num_tp[i], num_fn[i], num_fp[i], area_fa) = privacy_evaluation_summary(tp, fn, tp_db, fp_db, fname, detected_boxes_dict.keys(), args.min_size) recall[i] = num_tp[i] / (num_tp[i] + num_fn[i]) * 100.0 precision[i] = num_tp[i] / (num_tp[i] + num_fp[i]) * 100.0 fpr[i] = num_fp[i] / (num_tp[i] + num_fn[i]) pixel_fpr[i] = area_fa / len( labeled_boxes_dict.keys()) / (4096 * 8192) * 100.0 # label /crop/ save if args.b_save_boxes \ and score_th[i] == args.op_th \ and score_th_v[j] == args.op_th_validation: save_detections(data_to_label_directory, args.data_directory, detected_boxes_dict.keys(), tp, fn, tp_db, fp_db, args.b_crop_labeled_raster) if args.b_label_raster \ and score_th[i] == args.op_th \ and score_th_v[j] == args.op_th_validation: save_labeled_raster(data_to_label_directory, args.data_directory, detected_boxes_dict.keys(), [args.op_th, args.op_th_validation], tp, fn, tp_db, fp_db) # save ROC stats in file fname = args.data_directory + '/' + detection_type + '_roc_validation_th_' + str( score_th_v[j]) + '.txt' names = numpy.array([ '#threshold', 'recall[%]', 'FPR', 'pixel FPR[%]', 'precision[%]', 'TP', 'FN', 'FP' ]) table_data = numpy.array([ score_th, recall, fpr, pixel_fpr, precision, num_tp, num_fn, num_fp ]) save_roc_file(fname, names, table_data.transpose(), args, score_th_v[j]) # print on screen if desired if 1 or score_th_v[j] == args.op_th_validation: f = open(fname, 'r') print(f.read()) f.close()
parser = argparse.ArgumentParser(add_help=False) parser.add_argument("data_directory", help=("directory containing all panorama data")) if __name__ == "__main__": my_parser = argparse.ArgumentParser( parents=[common.capture_spec_parser, parser]) my_parser.add_argument("detection_type", choices=["faces", "plates", "people", "cars"]) my_parser.add_argument("--header-format", default="{capture_id:06d}", help=("format string for header field (default " "is \"{capture_id:06d}\" for " "000001, 000002, etc.)")) args = my_parser.parse_args() capture_ids = common.get_capture_ids(**args.__dict__) if capture_ids is None: capture_ids = find_capture_ids(args.data_directory) for (capture_id, boxes) in detected_boxes(args.data_directory, args.detection_type, capture_ids): for box in boxes: print (args.header_format.format(capture_id=capture_id) + " " + " ".join(map(str, box)))
def main(): # ---------- Input all the arguments ----------- % parser = argparse.ArgumentParser( parents=[common.capture_spec_parser, db_export.parser, process_export.parser], add_help=False) parser.add_argument("detection_type", choices=['faces', 'plates']) my_parser = argparse.ArgumentParser( parents=[parser]) my_parser.add_argument("--threshold", type=float, default=threshold_dft, help=("overlap threshold (fraction) " "for a hit")) my_parser.add_argument("--op-th", type=float, default=op_th_dft, help=("roc operating threshold")) my_parser.add_argument("--op-th-validation", type=float, default=op_th_dft_validation, help=("roc operating threshold, validation")) my_parser.add_argument("--min-size", type=int, default=min_size_dft, help=("minimum labeled box size in pixels")) my_parser.add_argument("--label-type") # agruments for color labeling of hits/ misses / false alarms my_parser.add_argument("--b-label-raster", default=False) my_parser.add_argument("--data-to-label-directory", help=("directory of clean or blurred rasters to label/crop")) # arguments for saving hits/ misses / false alarms # hits / misses / flase alarms will be cropped # from the "--rasters-to-label-directory" my_parser.add_argument("--b-save-boxes", default=False) my_parser.add_argument("--b-crop-labeled-raster", default=False) args = my_parser.parse_args() # detection type and parent box type detection_type = args.detection_type parent_map = {'faces':'people', 'plates':'cars'} parent_type = parent_map[detection_type] label_type = args.label_type if not label_type: label_type = 'clear_'+detection_type if not args.data_to_label_directory: data_to_label_directory = args.data_directory; # read labels and detections capture_ids = common.get_capture_ids(**args.__dict__) (detected_boxes_dict_in, detected_parent_boxes_dict, labeled_boxes_dict) = get_boxes_simplified( args.db_filename, args.data_directory, capture_ids, label_type, detection_type, parent_type, args.x_stride, args.y_offset) (dmmy2, dmmy1, labeled_boxes_dict_all) = get_boxes_simplified( args.db_filename, args.data_directory, capture_ids, 'all', detection_type, parent_type, args.x_stride, args.y_offset) #print detected_boxes_dict_in # run for a number of thresholds to generate ROC curve score_th = numpy.arange(-12, 4, 1)/10.0; score_th = numpy.append(score_th, args.op_th) score_th.sort() score_th_v = numpy.arange(0, 6 , 1)/10.0; score_th_v = score_th_v if numpy.any(score_th_v == args.op_th_validation) \ else numpy.append(score_th_v, args.op_th_validation) score_th_v.sort() num_tp = numpy.zeros(score_th.size) num_fn = numpy.zeros(score_th.size) num_fp = numpy.zeros(score_th.size) recall = numpy.zeros(score_th.size) precision = numpy.zeros(score_th.size) fpr = numpy.zeros(score_th.size) pixel_fpr = numpy.zeros(score_th.size) for j in range(score_th_v.size): for i in range(score_th.size): # get detected boxes with score above the threshold (detected_boxes_dict) = get_boxes_with_score_above_th(detected_boxes_dict_in, score_th[i], score_th_v[j]) # split labled boxes into hits (tp) and misses(fn) # split detected boxes into hits(tp_db) and false_alarms(fp_db) b_save_detected_txt = False if score_th[i] == args.op_th: b_save_detected_txt = True (tp, fn, tp_db, fp_db) = evaluate_detections_all(detected_boxes_dict, labeled_boxes_dict, labeled_boxes_dict_all, args.data_directory, args.detection_type, b_save_detected_txt, args.threshold) # get ROC stats fname=args.data_directory+'/'+args.detection_type+\ '_per_raster_thval'+str(score_th_v[j])+ \ 'th_' + str(score_th[i]) +'.txt' (num_tp[i], num_fn[i], num_fp[i], area_fa) = privacy_evaluation_summary( tp, fn, tp_db, fp_db, fname, detected_boxes_dict.keys(), args.min_size); recall[i] = num_tp[i] / (num_tp[i] + num_fn[i]) * 100.0 precision[i] = num_tp[i] / (num_tp[i] + num_fp[i]) * 100.0 fpr[i] = num_fp[i] / (num_tp[i] + num_fn[i]) pixel_fpr[i]= area_fa/len(labeled_boxes_dict.keys())/(4096*8192)*100.0 # label /crop/ save if args.b_save_boxes \ and score_th[i] == args.op_th \ and score_th_v[j] == args.op_th_validation: save_detections(data_to_label_directory, args.data_directory, detected_boxes_dict.keys(), tp, fn, tp_db, fp_db, args.b_crop_labeled_raster) if args.b_label_raster \ and score_th[i] == args.op_th \ and score_th_v[j] == args.op_th_validation: save_labeled_raster(data_to_label_directory, args.data_directory, detected_boxes_dict.keys(), [args.op_th, args.op_th_validation], tp, fn, tp_db, fp_db) # save ROC stats in file fname=args.data_directory+'/'+detection_type+'_roc_validation_th_' + str(score_th_v[j]) +'.txt' names =numpy.array(['#threshold', 'recall[%]', 'FPR', 'pixel FPR[%]', 'precision[%]', 'TP', 'FN', 'FP']); table_data =numpy.array([score_th, recall, fpr, pixel_fpr, precision, num_tp, num_fn, num_fp]) save_roc_file(fname, names, table_data.transpose(), args, score_th_v[j]) # print on screen if desired if 1 or score_th_v[j] == args.op_th_validation: f = open(fname, 'r') print(f.read()) f.close()