def convert_csv_to_json(input_path, output_path): # Format spec: # # https://github.com/microsoft/CameraTraps/tree/master/api/batch_processing print('Loading csv results...') df = load_api_results(input_path) info = { "detector": "unknown", "detection_completion_time": "unknown", "classifier": "unknown", "classification_completion_time": "unknown" } classification_categories = {} detection_categories = annotation_constants.bbox_category_id_to_name images = [] # iFile = 0; row = df.iloc[iFile] for iFile, row in df.iterrows(): image = {} image['file'] = row['image_path'] image['max_detection_conf'] = row['max_confidence'] src_detections = row['detections'] out_detections = [] for iDetection, detection in enumerate(src_detections): # Our .csv format was ymin/xmin/ymax/xmax # # Our .json format is xmin/ymin/w/h ymin = detection[0] xmin = detection[1] ymax = detection[2] xmax = detection[3] bbox = [xmin, ymin, xmax - xmin, ymax - ymin] conf = detection[4] iClass = detection[5] out_detection = {} out_detection['category'] = str(iClass) out_detection['conf'] = conf out_detection['bbox'] = bbox out_detections.append(out_detection) # ...for each detection image['detections'] = out_detections images.append(image) # ...for each image json_out = {} json_out['info'] = info json_out['detection_categories'] = detection_categories json_out['classification_categories'] = classification_categories json_out['images'] = images json.dump(json_out, open(output_path, 'w'), indent=1)
def find_repeat_detections(inputFilename, outputFilename, options=None): ##%% Input handling if options is None: options = RepeatDetectionOptions() toReturn = RepeatDetectionResults() ##%% Load file detectionResults, otherFields = load_api_results( inputFilename, normalize_paths=True, filename_replacements=options.filenameReplacements) toReturn.detectionResults = detectionResults toReturn.otherFields = otherFields ##%% Separate files into directories # This will be a map from a directory name to smaller data frames rowsByDirectory = {} # This is a mapping back into the rows of the original table filenameToRow = {} # TODO: in the case where we're loading an existing set of FPs after manual filtering, # we should load these data frames too, rather than re-building them from the input. print('Separating files into directories...') # iRow = 0; row = detectionResults.iloc[0] for iRow, row in detectionResults.iterrows(): relativePath = row['file'] dirName = os.path.dirname(relativePath) if options.nDirLevelsFromLeaf > 0: iLevel = 0 while (iLevel < options.nDirLevelsFromLeaf): iLevel += 1 dirName = os.path.dirname(dirName) assert len(dirName) > 0 if not dirName in rowsByDirectory: # Create a new DataFrame with just this row # rowsByDirectory[dirName] = pd.DataFrame(row) rowsByDirectory[dirName] = [] rowsByDirectory[dirName].append(row) assert relativePath not in filenameToRow filenameToRow[relativePath] = iRow # Convert lists of rows to proper DataFrames dirs = list(rowsByDirectory.keys()) for d in dirs: rowsByDirectory[d] = pd.DataFrame(rowsByDirectory[d]) toReturn.rowsByDirectory = rowsByDirectory toReturn.filenameToRow = filenameToRow print('Finished separating {} files into {} directories'.format( len(detectionResults), len(rowsByDirectory))) ##% Look for matches (or load them from file) dirsToSearch = list(rowsByDirectory.keys())[0:options.debugMaxDir] # length-nDirs list of lists of DetectionLocation objects suspiciousDetections = [None] * len(dirsToSearch) # Are we actually looking for matches, or just loading from a file? if len(options.filterFileToLoad) == 0: # We're actually looking for matches... print('Finding similar detections...') allCandidateDetections = [None] * len(dirsToSearch) if not options.bParallelizeComparisons: options.pbar = None # iDir = 0; dirName = dirsToSearch[iDir] for iDir, dirName in enumerate(tqdm(dirsToSearch)): allCandidateDetections[iDir] = find_matches_in_directory( dirName, options, rowsByDirectory) else: options.pbar = tqdm(total=len(dirsToSearch)) allCandidateDetections = Parallel( n_jobs=options.nWorkers, prefer='threads')(delayed(find_matches_in_directory)( dirName, options, rowsByDirectory) for dirName in tqdm(dirsToSearch)) print('\nFinished looking for similar bounding boxes') ##%% Find suspicious locations based on match results print('Filtering out repeat detections...') nImagesWithSuspiciousDetections = 0 nSuspiciousDetections = 0 # For each directory # # iDir = 51 for iDir in range(len(dirsToSearch)): # A list of DetectionLocation objects suspiciousDetectionsThisDir = [] # A list of DetectionLocation objects candidateDetectionsThisDir = allCandidateDetections[iDir] for iLocation, candidateLocation in enumerate( candidateDetectionsThisDir): # occurrenceList is a list of file/detection pairs nOccurrences = len(candidateLocation.instances) if nOccurrences < options.occurrenceThreshold: continue nImagesWithSuspiciousDetections += nOccurrences nSuspiciousDetections += 1 suspiciousDetectionsThisDir.append(candidateLocation) # Find the images corresponding to this bounding box, render boxes suspiciousDetections[iDir] = suspiciousDetectionsThisDir print( 'Finished searching for repeat detections\nFound {} unique detections on {} images that are suspicious' .format(nSuspiciousDetections, nImagesWithSuspiciousDetections)) else: print('Bypassing detection-finding, loading from {}'.format( options.filterFileToLoad)) # Load the filtering file detectionIndexFileName = options.filterFileToLoad sIn = open(detectionIndexFileName, 'r').read() suspiciousDetections = jsonpickle.decode(sIn) filteringBaseDir = os.path.dirname(options.filterFileToLoad) assert len(suspiciousDetections) == len(dirsToSearch) nDetectionsRemoved = 0 nDetectionsLoaded = 0 # We're skipping detection-finding, but to see which images are actually legit false # positives, we may be looking for physical files or loading from a text file. fileList = None if options.filteredFileListToLoad is not None: with open(options.filteredFileListToLoad) as f: fileList = f.readlines() fileList = [x.strip() for x in fileList] nSuspiciousDetections = sum([len(x) for x in suspiciousDetections]) print( 'Loaded false positive list from file, will remove {} of {} suspicious detections' .format(len(fileList), nSuspiciousDetections)) # For each directory # iDir = 0; detections = suspiciousDetections[0] for iDir, detections in enumerate(suspiciousDetections): bValidDetection = [True] * len(detections) nDetectionsLoaded += len(detections) # For each detection that was present before filtering # iDetection = 0; detection = detections[iDetection] for iDetection, detection in enumerate(detections): # Are we checking the directory to see whether detections were actually false positives, # or reading from a list? if fileList is None: # Is the image still there? imageFullPath = os.path.join( filteringBaseDir, detection.sampleImageRelativeFileName) # If not, remove this from the list of suspicious detections if not os.path.isfile(imageFullPath): nDetectionsRemoved += 1 bValidDetection[iDetection] = False else: if detection.sampleImageRelativeFileName not in fileList: nDetectionsRemoved += 1 bValidDetection[iDetection] = False nRemovedThisDir = len(bValidDetection) - sum(bValidDetection) if nRemovedThisDir > 0: print('Removed {} of {} detections from directory {}'.format( nRemovedThisDir, len(detections), iDir)) # ...for each detection detectionsFiltered = list(compress(detections, bValidDetection)) suspiciousDetections[iDir] = detectionsFiltered # ...for each directory print('Removed {} of {} total detections via manual filtering'.format( nDetectionsRemoved, nDetectionsLoaded)) # ...if we are/aren't finding detections (vs. loading from file) toReturn.suspiciousDetections = suspiciousDetections if options.bRenderHtml: # Render problematic locations with html (loop) print('Rendering html') nDirs = len(dirsToSearch) directoryHtmlFiles = [None] * nDirs if options.bParallelizeRendering: # options.pbar = tqdm(total=nDirs) options.pbar = None directoryHtmlFiles = Parallel( n_jobs=options.nWorkers, prefer='threads')(delayed(render_images_for_directory)( iDir, directoryHtmlFiles, suspiciousDetections, options) for iDir in tqdm(range(nDirs))) else: options.pbar = None # For each directory # iDir = 51 for iDir in range(nDirs): # Add this directory to the master list of html files directoryHtmlFiles[iDir] = render_images_for_directory( iDir, directoryHtmlFiles, suspiciousDetections, options) # ...for each directory # Write master html file masterHtmlFile = os.path.join(options.outputBase, 'index.html') os.makedirs(options.outputBase, exist_ok=True) toReturn.masterHtmlFile = masterHtmlFile with open(masterHtmlFile, 'w') as fHtml: fHtml.write('<html><body>\n') fHtml.write( '<h2><b>Repeat detections by directory</b></h2></br>\n') for iDir, dirHtmlFile in enumerate(directoryHtmlFiles): if dirHtmlFile is None: continue relPath = os.path.relpath(dirHtmlFile, options.outputBase) dirName = dirsToSearch[iDir] # Remove unicode characters before formatting relPath = relPath.encode('ascii', 'ignore').decode('ascii') dirName = dirName.encode('ascii', 'ignore').decode('ascii') fHtml.write('<a href={}>{}</a><br/>\n'.format( relPath, dirName)) fHtml.write('</body></html>\n') # ...if we're rendering html toReturn.allRowsFiltered = update_detection_table(toReturn, options, outputFilename) # Create filtering directory if options.bWriteFilteringFolder: print('Creating filtering folder...') dateString = datetime.now().strftime('%Y.%m.%d.%H.%M.%S') filteringDir = os.path.join(options.outputBase, 'filtering_' + dateString) os.makedirs(filteringDir, exist_ok=True) # iDir = 0; suspiciousDetectionsThisDir = suspiciousDetections[iDir] for iDir, suspiciousDetectionsThisDir in enumerate( tqdm(suspiciousDetections)): # suspiciousDetectionsThisDir is a list of DetectionLocation objects for iDetection, detection in enumerate( suspiciousDetectionsThisDir): bbox = detection.bbox instance = detection.instances[0] relativePath = instance.filename inputFullPath = os.path.join(options.imageBase, relativePath) assert (os.path.isfile(inputFullPath) ), 'Not a file: {}'.format(inputFullPath) outputRelativePath = 'dir{:0>4d}_det{:0>4d}.jpg'.format( iDir, iDetection) outputFullPath = os.path.join(filteringDir, outputRelativePath) render_bounding_box(bbox, inputFullPath, outputFullPath, 15) detection.sampleImageRelativeFileName = outputRelativePath # Write out the detection index detectionIndexFileName = os.path.join(filteringDir, 'detectionIndex.json') jsonpickle.set_encoder_options('json', sort_keys=True, indent=4) s = jsonpickle.encode(suspiciousDetections) with open(detectionIndexFileName, 'w') as f: f.write(s) toReturn.filterFile = detectionIndexFileName print('Done') # ...if we're writing filtering info return toReturn
def process_batch_results(options): ppresults = PostProcessingResults() ##%% Expand some options for convenience output_dir = options.output_dir ##%% Prepare output dir os.makedirs(output_dir, exist_ok=True) ##%% Load ground truth if available ground_truth_indexed_db = None if options.ground_truth_json_file and len(options.ground_truth_json_file) > 0: ground_truth_indexed_db = IndexedJsonDb(options.ground_truth_json_file, b_normalize_paths=True, filename_replacements=options.ground_truth_filename_replacements) # Mark images in the ground truth as positive or negative n_negative, n_positive, n_unknown, n_ambiguous = mark_detection_status(ground_truth_indexed_db, negative_classes=options.negative_classes, unknown_classes=options.unlabeled_classes) print('Finished loading and indexing ground truth: {} negative, {} positive, {} unknown, {} ambiguous'.format( n_negative, n_positive, n_unknown, n_ambiguous)) ##%% Load detection results if options.api_detection_results is None: detection_results, other_fields = load_api_results(options.api_output_file, normalize_paths=True, filename_replacements=options.api_output_filename_replacements) ppresults.api_detection_results = detection_results ppresults.api_other_fields = other_fields else: print('Bypassing detection results loading...') assert options.api_other_fields is not None detection_results = options.api_detection_results other_fields = options.api_other_fields detection_categories_map = other_fields['detection_categories'] if 'classification_categories' in other_fields: classification_categories_map = other_fields['classification_categories'] else: classification_categories_map = {} # Add a column (pred_detection_label) to indicate predicted detection status, not separating out the classes if options.include_almost_detections: detection_results['pred_detection_label'] = DetectionStatus.DS_ALMOST confidences = detection_results['max_detection_conf'] detection_results.loc[confidences >= options.confidence_threshold,'pred_detection_label'] = DetectionStatus.DS_POSITIVE detection_results.loc[confidences < options.almost_detection_confidence_threshold,'pred_detection_label'] = DetectionStatus.DS_NEGATIVE else: detection_results['pred_detection_label'] = \ np.where(detection_results['max_detection_conf'] >= options.confidence_threshold, DetectionStatus.DS_POSITIVE, DetectionStatus.DS_NEGATIVE) n_positives = sum(detection_results['pred_detection_label'] == DetectionStatus.DS_POSITIVE) print('Finished loading and preprocessing {} rows from detector output, predicted {} positives'.format( len(detection_results), n_positives)) if options.include_almost_detections: n_almosts = sum(detection_results['pred_detection_label'] == DetectionStatus.DS_ALMOST) print('...and {} almost-positives'.format(n_almosts)) ##%% If we have ground truth, remove images we can't match to ground truth if ground_truth_indexed_db is not None: b_match = [False] * len(detection_results) detector_files = detection_results['file'].tolist() # fn = detector_files[0]; print(fn) for i_fn, fn in enumerate(detector_files): # assert fn in ground_truth_indexed_db.filename_to_id, 'Could not find ground truth for row {} ({})'.format(i_fn,fn) if fn in ground_truth_indexed_db.filename_to_id: b_match[i_fn] = True print('Confirmed filename matches to ground truth for {} of {} files'.format(sum(b_match), len(detector_files))) detection_results = detection_results[b_match] detector_files = detection_results['file'].tolist() assert len(detector_files) > 0, 'No detection files available, possible ground truth path issue?' print('Trimmed detection results to {} files'.format(len(detector_files))) ##%% Sample images for visualization images_to_visualize = detection_results if options.num_images_to_sample > 0 and options.num_images_to_sample <= len(detection_results): images_to_visualize = images_to_visualize.sample(options.num_images_to_sample, random_state=options.sample_seed) output_html_file = '' style_header = """<head> <style type="text/css"> <!-- a { text-decoration:none; } body { font-family:segoe ui, calibri, "trebuchet ms", verdana, arial, sans-serif; } div.contentdiv { margin-left:20px; } --> </style> </head>""" ##%% Fork here depending on whether or not ground truth is available # If we have ground truth, we'll compute precision/recall and sample tp/fp/tn/fn. # # Otherwise we'll just visualize detections/non-detections. if ground_truth_indexed_db is not None: ##%% Detection evaluation: compute precision/recall # numpy array of detection probabilities p_detection = detection_results['max_detection_conf'].values n_detections = len(p_detection) # numpy array of bools (0.0/1.0), and -1 as null value gt_detections = np.zeros(n_detections, dtype=float) for i_detection, fn in enumerate(detector_files): image_id = ground_truth_indexed_db.filename_to_id[fn] image = ground_truth_indexed_db.image_id_to_image[image_id] detection_status = image['_detection_status'] if detection_status == DetectionStatus.DS_NEGATIVE: gt_detections[i_detection] = 0.0 elif detection_status == DetectionStatus.DS_POSITIVE: gt_detections[i_detection] = 1.0 else: gt_detections[i_detection] = -1.0 # Don't include ambiguous/unknown ground truth in precision/recall analysis b_valid_ground_truth = gt_detections >= 0.0 p_detection_pr = p_detection[b_valid_ground_truth] gt_detections_pr = gt_detections[b_valid_ground_truth] print('Including {} of {} values in p/r analysis'.format(np.sum(b_valid_ground_truth), len(b_valid_ground_truth))) precisions, recalls, thresholds = precision_recall_curve(gt_detections_pr, p_detection_pr) # For completeness, include the result at a confidence threshold of 1.0 thresholds = np.append(thresholds, [1.0]) precisions_recalls = pd.DataFrame(data={ 'confidence_threshold': thresholds, 'precision': precisions, 'recall': recalls }) # Compute and print summary statistics average_precision = average_precision_score(gt_detections_pr, p_detection_pr) print('Average precision: {:.1%}'.format(average_precision)) # Thresholds go up throughout precisions/recalls/thresholds; find the last # value where recall is at or above target. That's our precision @ target recall. target_recall = 0.9 b_above_target_recall = np.where(recalls >= target_recall) if not np.any(b_above_target_recall): precision_at_target_recall = 0.0 else: i_target_recall = np.argmax(b_above_target_recall) precision_at_target_recall = precisions[i_target_recall] print('Precision at {:.1%} recall: {:.1%}'.format(target_recall, precision_at_target_recall)) cm = confusion_matrix(gt_detections_pr, np.array(p_detection_pr) > options.confidence_threshold) # Flatten the confusion matrix tn, fp, fn, tp = cm.ravel() precision_at_confidence_threshold = tp / (tp + fp) recall_at_confidence_threshold = tp / (tp + fn) f1 = 2.0 * (precision_at_confidence_threshold * recall_at_confidence_threshold) / \ (precision_at_confidence_threshold + recall_at_confidence_threshold) print('At a confidence threshold of {:.1%}, precision={:.1%}, recall={:.1%}, f1={:.1%}'.format( options.confidence_threshold, precision_at_confidence_threshold, recall_at_confidence_threshold, f1)) ##%% Collect classification results, if they exist classifier_accuracies = [] # Mapping of classnames to idx for the confusion matrix. # # The lambda is actually kind of a hack, because we use assume that # the following code does not reassign classname_to_idx classname_to_idx = collections.defaultdict(lambda: len(classname_to_idx)) # Confusion matrix as defaultdict of defaultdict # # Rows / first index is ground truth, columns / second index is predicted category classifier_cm = collections.defaultdict(lambda: collections.defaultdict(lambda: 0)) # iDetection = 0; fn = detector_files[iDetection]; print(fn) assert len(detector_files) == len(detection_results) for iDetection,fn in enumerate(detector_files): image_id = ground_truth_indexed_db.filename_to_id[fn] image = ground_truth_indexed_db.image_id_to_image[image_id] detections = detection_results['detections'].iloc[iDetection] pred_class_ids = [det['classifications'][0][0] \ for det in detections if 'classifications' in det.keys()] pred_classnames = [classification_categories_map[pd] for pd in pred_class_ids] # If this image has classification predictions, and an unambiguous class # annotated, and is a positive image... if len(pred_classnames) > 0 \ and '_unambiguous_category' in image.keys() \ and image['_detection_status'] == DetectionStatus.DS_POSITIVE: # The unambiguous category, we make this a set for easier handling afterward gt_categories = set([image['_unambiguous_category']]) pred_categories = set(pred_classnames) # Compute the accuracy as intersection of union, # i.e. (# of categories in both prediciton and GT) # divided by (# of categories in either prediction or GT # # In case of only one GT category, the result will be 1.0, if # prediction is one category and this category matches GT # # It is 1.0/(# of predicted top-1 categories), if the GT is # one of the predicted top-1 categories. # # It is 0.0, if none of the predicted categories is correct classifier_accuracies.append( len(gt_categories & pred_categories) / len(gt_categories | pred_categories) ) image['_classification_accuracy'] = classifier_accuracies[-1] # Distribute this accuracy across all predicted categories in the # confusion matrix assert len(gt_categories) == 1 gt_class_idx = classname_to_idx[list(gt_categories)[0]] for pred_category in pred_categories: pred_class_idx = classname_to_idx[pred_category] classifier_cm[gt_class_idx][pred_class_idx] += 1 # ...for each file in the detection results # If we have classification results if len(classifier_accuracies) > 0: # Build confusion matrix as array from classifier_cm all_class_ids = sorted(classname_to_idx.values()) classifier_cm_array = np.array( [[classifier_cm[r_idx][c_idx] for c_idx in all_class_ids] for r_idx in all_class_ids], dtype=float) classifier_cm_array /= (classifier_cm_array.sum(axis=1, keepdims=True) + 1e-7) # Print some statistics print("Finished computation of {} classification results".format(len(classifier_accuracies))) print("Mean accuracy: {}".format(np.mean(classifier_accuracies))) # Prepare confusion matrix output # Get confusion matrix as string sio = io.StringIO() np.savetxt(sio, classifier_cm_array * 100, fmt='%5.1f') cm_str = sio.getvalue() # Get fixed-size classname for each idx idx_to_classname = {v:k for k,v in classname_to_idx.items()} classname_list = [idx_to_classname[idx] for idx in sorted(classname_to_idx.values())] classname_headers = ['{:<5}'.format(cname[:5]) for cname in classname_list] # Prepend class name on each line and add to the top cm_str_lines = [' ' * 16 + ' '.join(classname_headers)] cm_str_lines += ['{:>15}'.format(cn[:15]) + ' ' + cm_line for cn, cm_line in zip(classname_list, cm_str.splitlines())] # Print formatted confusion matrix print("Confusion matrix: ") print(*cm_str_lines, sep='\n') # Plot confusion matrix # To manually add more space at bottom: plt.rcParams['figure.subplot.bottom'] = 0.1 # # Add 0.5 to figsize for every class. For two classes, this will result in # fig = plt.figure(figsize=[4,4]) fig = vis_utils.plot_confusion_matrix( classifier_cm_array, classname_list, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues, vmax=1.0, use_colorbar=True, y_label=True) cm_figure_relative_filename = 'confusion_matrix.png' cm_figure_filename = os.path.join(output_dir, cm_figure_relative_filename) plt.savefig(cm_figure_filename) plt.close(fig) # ...if we have classification results ##%% Render output # Write p/r table to .csv file in output directory pr_table_filename = os.path.join(output_dir, 'prec_recall.csv') precisions_recalls.to_csv(pr_table_filename, index=False) # Write precision/recall plot to .png file in output directory t = 'Precision-Recall curve: AP={:0.1%}, P@{:0.1%}={:0.1%}'.format( average_precision, target_recall, precision_at_target_recall) fig = vis_utils.plot_precision_recall_curve(precisions, recalls, t) pr_figure_relative_filename = 'prec_recall.png' pr_figure_filename = os.path.join(output_dir, pr_figure_relative_filename) plt.savefig(pr_figure_filename) # plt.show(block=False) plt.close(fig) ##%% Sampling # Sample true/false positives/negatives with correct/incorrect top-1 # classification and render to html # Accumulate html image structs (in the format expected by write_html_image_lists) # for each category, e.g. 'tp', 'fp', ..., 'class_bird', ... images_html = collections.defaultdict(lambda: []) # Add default entries by accessing them for the first time [images_html[res] for res in ['tp', 'tpc', 'tpi', 'fp', 'tn', 'fn']] # Siyu: what does this do? This line should have no effect for res in images_html.keys(): os.makedirs(os.path.join(output_dir, res), exist_ok=True) image_count = len(images_to_visualize) # Each element will be a list of 2-tuples, with elements [collection name,html info struct] rendering_results = [] # Each element will be a three-tuple with elements file,max_conf,detections files_to_render = [] # Assemble the information we need for rendering, so we can parallelize without # dealing with Pandas # i_row = 0; row = images_to_visualize.iloc[0] for _, row in images_to_visualize.iterrows(): # Filenames should already have been normalized to either '/' or '\' files_to_render.append([row['file'],row['max_detection_conf'],row['detections']]) def render_image_with_gt(file_info): image_relative_path = file_info[0] max_conf = file_info[1] detections = file_info[2] # This should already have been normalized to either '/' or '\' image_id = ground_truth_indexed_db.filename_to_id.get(image_relative_path, None) if image_id is None: print('Warning: couldn''t find ground truth for image {}'.format(image_relative_path)) return None image = ground_truth_indexed_db.image_id_to_image[image_id] annotations = ground_truth_indexed_db.image_id_to_annotations[image_id] gt_status = image['_detection_status'] gt_presence = bool(gt_status) gt_classes = CameraTrapJsonUtils.annotations_to_classnames( annotations,ground_truth_indexed_db.cat_id_to_name) gt_class_summary = ','.join(gt_classes) if gt_status > DetectionStatus.DS_MAX_DEFINITIVE_VALUE: print('Skipping image {}, does not have a definitive ground truth status (status: {}, classes: {})'.format( image_id, gt_status, gt_class_summary)) return None detected = max_conf > options.confidence_threshold if gt_presence and detected: if '_classification_accuracy' not in image.keys(): res = 'tp' elif np.isclose(1, image['_classification_accuracy']): res = 'tpc' else: res = 'tpi' elif not gt_presence and detected: res = 'fp' elif gt_presence and not detected: res = 'fn' else: res = 'tn' display_name = '<b>Result type</b>: {}, <b>Presence</b>: {}, <b>Class</b>: {}, <b>Max conf</b>: {:0.3f}%, <b>Image</b>: {}'.format( res.upper(), str(gt_presence), gt_class_summary, max_conf * 100, image_relative_path) rendered_image_html_info = render_bounding_boxes(options.image_base_dir, image_relative_path, display_name, detections, res, detection_categories_map, classification_categories_map, options) image_result = None if len(rendered_image_html_info) > 0: image_result = [[res,rendered_image_html_info]] for gt_class in gt_classes: image_result.append(['class_{}'.format(gt_class),rendered_image_html_info]) return image_result # ...def render_image_with_gt(file_info) start_time = time.time() if options.parallelize_rendering: if options.parallelize_rendering_n_cores is None: pool = ThreadPool() else: print('Rendering images with {} workers'.format(options.parallelize_rendering_n_cores)) pool = ThreadPool(options.parallelize_rendering_n_cores) rendering_results = list(tqdm(pool.imap(render_image_with_gt, files_to_render), total=len(files_to_render))) else: # file_info = files_to_render[0] for file_info in tqdm(files_to_render): rendering_results.append(render_image_with_gt(file_info)) elapsed = time.time() - start_time # Map all the rendering results in the list rendering_results into the # dictionary images_html image_rendered_count = 0 for rendering_result in rendering_results: if rendering_result is None: continue image_rendered_count += 1 for assignment in rendering_result: images_html[assignment[0]].append(assignment[1]) # Prepare the individual html image files image_counts = prepare_html_subpages(images_html, output_dir) print('{} images rendered (of {})'.format(image_rendered_count,image_count)) # Write index.html all_tp_count = image_counts['tp'] + image_counts['tpc'] + image_counts['tpi'] total_count = all_tp_count + image_counts['tn'] + image_counts['fp'] + image_counts['fn'] classification_detection_results = """ <a href="tpc.html">with all correct top-1 predictions (TPC)</a> ({})<br/> <a href="tpi.html">with one or more incorrect top-1 prediction (TPI)</a> ({})<br/> <a href="tp.html">without classification evaluation</a><sup>*</sup> ({})<br/>""".format( image_counts['tpc'], image_counts['tpi'], image_counts['tp'] ) index_page = """<html> {} <body> <h2>Evaluation</h2> <h3>Sample images</h3> <div style="margin-left:20px;"> <p>A sample of {} images, annotated with detections above {:.1%} confidence.</p> <a href="tp.html">True positives (TP)</a> ({}) ({:0.1%})<br/> CLASSIFICATION_PLACEHOLDER_1 <a href="tn.html">True negatives (TN)</a> ({}) ({:0.1%})<br/> <a href="fp.html">False positives (FP)</a> ({}) ({:0.1%})<br/> <a href="fn.html">False negatives (FN)</a> ({}) ({:0.1%})<br/> CLASSIFICATION_PLACEHOLDER_2 </div> """.format( style_header, image_count, options.confidence_threshold, all_tp_count, all_tp_count/total_count, image_counts['tn'], image_counts['tn']/total_count, image_counts['fp'], image_counts['fp']/total_count, image_counts['fn'], image_counts['fn']/total_count ) index_page += """ <h3>Detection results</h3> <div class="contentdiv"> <p>At a confidence threshold of {:0.1%}, precision={:0.1%}, recall={:0.1%}</p> <p><strong>Precision/recall summary for all {} images</strong></p><img src="{}"><br/> </div> """.format( options.confidence_threshold, precision_at_confidence_threshold, recall_at_confidence_threshold, len(detection_results), pr_figure_relative_filename ) if len(classifier_accuracies) > 0: index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_1',classification_detection_results) index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_2',"""<p><sup>*</sup>We do not evaluate the classification result of images if the classification information is missing, if the image contains categories like ‘empty’ or ‘human’, or if the image has multiple classification labels.</p>""") else: index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_1','') index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_2','') if len(classifier_accuracies) > 0: index_page += """ <h3>Classification results</h3> <div class="contentdiv"> <p>Classification accuracy: {:.2%}<br> The accuracy is computed only for images with exactly one classification label. The accuracy of an image is computed as 1/(number of unique detected top-1 classes), i.e. if the model detects multiple boxes with different top-1 classes, then the accuracy decreases and the image is put into 'TPI'.</p> <p>Confusion matrix:</p> <p><img src="{}"></p> <div style='font-family:monospace;display:block;'>{}</div> </div> """.format( np.mean(classifier_accuracies), cm_figure_relative_filename, "<br>".join(cm_str_lines).replace(' ', ' ') ) # Show links to each GT class # # We could do this without classification results; currently we don't. if len(classname_to_idx) > 0: index_page += '<h3>Images of specific classes</h3><br/><div class="contentdiv">' # Add links to all available classes for cname in sorted(classname_to_idx.keys()): index_page += "<a href='class_{0}.html'>{0}</a> ({1})<br>".format( cname, len(images_html['class_{}'.format(cname)])) index_page += "</div>" # Close body and html tags index_page += "</body></html>" output_html_file = os.path.join(output_dir, 'index.html') with open(output_html_file, 'w') as f: f.write(index_page) print('Finished writing html to {}'.format(output_html_file)) # ...for each image ##%% Otherwise, if we don't have ground truth... else: ##%% Sample detections/non-detections # Accumulate html image structs (in the format expected by write_html_image_list) # for each category images_html = collections.defaultdict(lambda: []) # Add default entries by accessing them for the first time [images_html[res] for res in ['detections', 'non_detections']] if options.include_almost_detections: images_html['almost_detections'] # Create output directories for res in images_html.keys(): os.makedirs(os.path.join(output_dir, res), exist_ok=True) image_count = len(images_to_visualize) has_classification_info = False # Each element will be a list of 2-tuples, with elements [collection name,html info struct] rendering_results = [] # Each element will be a three-tuple with elements [file,max_conf,detections] files_to_render = [] # Assemble the information we need for rendering, so we can parallelize without # dealing with Pandas # i_row = 0; row = images_to_visualize.iloc[0] for _, row in images_to_visualize.iterrows(): # Filenames should already have been normalized to either '/' or '\' files_to_render.append([row['file'], row['max_detection_conf'], row['detections']]) # Local function for parallelization def render_image_no_gt(file_info): image_relative_path = file_info[0] max_conf = file_info[1] detections = file_info[2] detection_status = DetectionStatus.DS_UNASSIGNED if max_conf >= options.confidence_threshold: detection_status = DetectionStatus.DS_POSITIVE else: if options.include_almost_detections: if max_conf >= options.almost_detection_confidence_threshold: detection_status = DetectionStatus.DS_ALMOST else: detection_status = DetectionStatus.DS_NEGATIVE else: detection_status = DetectionStatus.DS_NEGATIVE if detection_status == DetectionStatus.DS_POSITIVE: res = 'detections' elif detection_status == DetectionStatus.DS_NEGATIVE: res = 'non_detections' else: assert detection_status == DetectionStatus.DS_ALMOST res = 'almost_detections' display_name = '<b>Result type</b>: {}, <b>Image</b>: {}, <b>Max conf</b>: {:0.3f}'.format( res, image_relative_path, max_conf) rendering_options = copy.copy(options) if detection_status == DetectionStatus.DS_ALMOST: rendering_options.confidence_threshold = rendering_options.almost_detection_confidence_threshold rendered_image_html_info = render_bounding_boxes(options.image_base_dir, image_relative_path, display_name, detections, res, detection_categories_map, classification_categories_map, rendering_options) image_result = None if len(rendered_image_html_info) > 0: image_result = [[res,rendered_image_html_info]] for det in detections: if 'classifications' in det: top1_class = classification_categories_map[det['classifications'][0][0]] image_result.append(['class_{}'.format(top1_class),rendered_image_html_info]) return image_result # ...def render_image_no_gt(file_info): start_time = time.time() if options.parallelize_rendering: if options.parallelize_rendering_n_cores is None: pool = ThreadPool() else: print('Rendering images with {} workers'.format(options.parallelize_rendering_n_cores)) pool = ThreadPool(options.parallelize_rendering_n_cores) rendering_results = list(tqdm(pool.imap(render_image_no_gt, files_to_render), total=len(files_to_render))) else: for file_info in tqdm(files_to_render): rendering_results.append(render_image_no_gt(file_info)) elapsed = time.time() - start_time # Map all the rendering results in the list rendering_results into the # dictionary images_html image_rendered_count = 0 for rendering_result in rendering_results: if rendering_result is None: continue image_rendered_count += 1 for assignment in rendering_result: if 'class' in assignment[0]: has_classification_info = True images_html[assignment[0]].append(assignment[1]) # Prepare the individual html image files image_counts = prepare_html_subpages(images_html, output_dir) if image_rendered_count == 0: seconds_per_image = 0 else: seconds_per_image = elapsed/image_rendered_count print('Rendered {} images (of {}) in {} ({} per image)'.format(image_rendered_count, image_count,humanfriendly.format_timespan(elapsed), humanfriendly.format_timespan(seconds_per_image))) # Write index.HTML total_images = image_counts['detections'] + image_counts['non_detections'] if options.include_almost_detections: total_images += image_counts['almost_detections'] assert total_images == image_count, \ 'Error: image_count is {}, total_images is {}'.format(image_count,total_images) almost_detection_string = '' if options.include_almost_detections: almost_detection_string = ' (“almost detection” threshold at {:.1%})'.format(options.almost_detection_confidence_threshold) index_page = """<html>{}<body> <h2>Visualization of results</h2> <p>A sample of {} images, annotated with detections above {:.1%} confidence{}.</p> <h3>Sample images</h3> <div class="contentdiv"> <a href="detections.html">detections</a> ({}, {:.1%})<br/> <a href="non_detections.html">non-detections</a> ({}, {:.1%})<br/>""".format( style_header,image_count, options.confidence_threshold, almost_detection_string, image_counts['detections'], image_counts['detections']/total_images, image_counts['non_detections'], image_counts['non_detections']/total_images ) if options.include_almost_detections: index_page += """<a href="almost_detections.html">almost-detections</a> ({}, {:.1%})<br/>""".format( image_counts['almost_detections'], image_counts['almost_detections']/total_images) index_page += '</div>\n' if has_classification_info: index_page += "<h3>Images of detected classes</h3>" index_page += "<p>The same image might appear under multiple classes if multiple species were detected.</p>\n<div class='contentdiv'>\n" # Add links to all available classes for cname in sorted(classification_categories_map.values()): ccount = len(images_html['class_{}'.format(cname)]) if ccount > 0: index_page += "<a href='class_{}.html'>{}</a> ({})<br/>\n".format(cname, cname.lower(), ccount) index_page += "</div>\n" index_page += "</body></html>" output_html_file = os.path.join(output_dir, 'index.html') with open(output_html_file, 'w') as f: f.write(index_page) print('Finished writing html to {}'.format(output_html_file)) # os.startfile(output_html_file) # ...if we do/don't have ground truth ppresults.output_html_file = output_html_file return ppresults
def process_batch_results(options): ##%% Expand some options for convenience output_dir = options.output_dir confidence_threshold = options.confidence_threshold ##%% Prepare output dir os.makedirs(output_dir, exist_ok=True) ##%% Load ground truth if available ground_truth_indexed_db = None if options.ground_truth_json_file and len( options.ground_truth_json_file) > 0: ground_truth_indexed_db = IndexedJsonDb( options.ground_truth_json_file, b_normalize_paths=True, filename_replacements=options.ground_truth_filename_replacements) # Mark images in the ground truth as positive or negative n_negative, n_positive, n_unknown, n_ambiguous = mark_detection_status( ground_truth_indexed_db, negative_classes=options.negative_classes, unknown_classes=options.unlabeled_classes) print( 'Finished loading and indexing ground truth: {} negative, {} positive, {} unknown, {} ambiguous' .format(n_negative, n_positive, n_unknown, n_ambiguous)) ##%% Load detection results detection_results, other_fields = load_api_results( options.api_output_file, normalize_paths=True, filename_replacements=options.api_output_filename_replacements) detection_categories_map = other_fields['detection_categories'] if 'classification_categories' in other_fields: classification_categories_map = other_fields[ 'classification_categories'] else: classification_categories_map = {} # Add a column (pred_detection_label) to indicate predicted detection status, not separating out the classes detection_results['pred_detection_label'] = \ np.where(detection_results['max_detection_conf'] >= options.confidence_threshold, DetectionStatus.DS_POSITIVE, DetectionStatus.DS_NEGATIVE) n_positives = sum(detection_results['pred_detection_label'] == DetectionStatus.DS_POSITIVE) print( 'Finished loading and preprocessing {} rows from detector output, predicted {} positives' .format(len(detection_results), n_positives)) ##%% If we have ground truth, remove images we can't match to ground truth # ground_truth_indexed_db.db['images'][0] if ground_truth_indexed_db is not None: b_match = [False] * len(detection_results) detector_files = detection_results['file'].tolist() for i_fn, fn in enumerate(detector_files): # assert fn in ground_truth_indexed_db.filename_to_id, 'Could not find ground truth for row {} ({})'.format(i_fn,fn) if fn in fn in ground_truth_indexed_db.filename_to_id: b_match[i_fn] = True print('Confirmed filename matches to ground truth for {} of {} files'. format(sum(b_match), len(detector_files))) detection_results = detection_results[b_match] detector_files = detection_results['file'].tolist() print('Trimmed detection results to {} files'.format( len(detector_files))) ##%% Sample images for visualization images_to_visualize = detection_results if options.num_images_to_sample > 0 and options.num_images_to_sample <= len( detection_results): images_to_visualize = images_to_visualize.sample( options.num_images_to_sample, random_state=options.sample_seed) ##%% Fork here depending on whether or not ground truth is available output_html_file = '' # If we have ground truth, we'll compute precision/recall and sample tp/fp/tn/fn. # # Otherwise we'll just visualize detections/non-detections. if ground_truth_indexed_db is not None: ##%% DETECTION EVALUATION: Compute precision/recall # numpy array of detection probabilities p_detection = detection_results['max_detection_conf'].values n_detections = len(p_detection) # numpy array of bools (0.0/1.0), and -1 as null value gt_detections = np.zeros(n_detections, dtype=float) for i_detection, fn in enumerate(detector_files): image_id = ground_truth_indexed_db.filename_to_id[fn] image = ground_truth_indexed_db.image_id_to_image[image_id] detection_status = image['_detection_status'] if detection_status == DetectionStatus.DS_NEGATIVE: gt_detections[i_detection] = 0.0 elif detection_status == DetectionStatus.DS_POSITIVE: gt_detections[i_detection] = 1.0 else: gt_detections[i_detection] = -1.0 # Don't include ambiguous/unknown ground truth in precision/recall analysis b_valid_ground_truth = gt_detections >= 0.0 p_detection_pr = p_detection[b_valid_ground_truth] gt_detections_pr = gt_detections[b_valid_ground_truth] print('Including {} of {} values in p/r analysis'.format( np.sum(b_valid_ground_truth), len(b_valid_ground_truth))) precisions, recalls, thresholds = precision_recall_curve( gt_detections_pr, p_detection_pr) # For completeness, include the result at a confidence threshold of 1.0 thresholds = np.append(thresholds, [1.0]) precisions_recalls = pd.DataFrame( data={ 'confidence_threshold': thresholds, 'precision': precisions, 'recall': recalls }) # Compute and print summary statistics average_precision = average_precision_score(gt_detections_pr, p_detection_pr) print('Average precision: {:.1%}'.format(average_precision)) # Thresholds go up throughout precisions/recalls/thresholds; find the last # value where recall is at or above target. That's our precision @ target recall. target_recall = 0.9 b_above_target_recall = np.where(recalls >= target_recall) if not np.any(b_above_target_recall): precision_at_target_recall = 0.0 else: i_target_recall = np.argmax(b_above_target_recall) precision_at_target_recall = precisions[i_target_recall] print('Precision at {:.1%} recall: {:.1%}'.format( target_recall, precision_at_target_recall)) cm = confusion_matrix(gt_detections_pr, np.array(p_detection_pr) > confidence_threshold) # Flatten the confusion matrix tn, fp, fn, tp = cm.ravel() precision_at_confidence_threshold = tp / (tp + fp) recall_at_confidence_threshold = tp / (tp + fn) f1 = 2.0 * (precision_at_confidence_threshold * recall_at_confidence_threshold) / \ (precision_at_confidence_threshold + recall_at_confidence_threshold) print( 'At a confidence threshold of {:.1%}, precision={:.1%}, recall={:.1%}, f1={:.1%}' .format(confidence_threshold, precision_at_confidence_threshold, recall_at_confidence_threshold, f1)) ##%% CLASSIFICATION evaluation classifier_accuracies = [] # Mapping of classnames to idx for the confusion matrix. # The lambda is actually kind of a hack, because we use assume that # the following code does not reassign classname_to_idx classname_to_idx = collections.defaultdict( lambda: len(classname_to_idx)) # Confusion matrix as defaultdict of defaultdict # Rows / first index is ground truth, columns / second index is predicted category classifier_cm = collections.defaultdict( lambda: collections.defaultdict(lambda: 0)) for iDetection, fn in enumerate(detector_files): image_id = ground_truth_indexed_db.filename_to_id[fn] image = ground_truth_indexed_db.image_id_to_image[image_id] pred_class_ids = [det['classifications'][0][0] \ for det in detection_results['detections'][iDetection] if 'classifications' in det.keys()] pred_classnames = [ classification_categories_map[pd] for pd in pred_class_ids ] # If this image has classification predictions, and an unambiguous class # annotated, and is a positive image if len(pred_classnames) > 0 \ and '_unambiguous_category' in image.keys() \ and image['_detection_status'] == DetectionStatus.DS_POSITIVE: # The unambiguous category, we make this a set for easier handling afterward # TODO: actually we can replace the unambiguous category by all annotated # categories. However, then the confusion matrix doesn't make sense anymore # TODO: make sure we are using the class names as strings in both, not IDs gt_categories = set([image['_unambiguous_category']]) pred_categories = set(pred_classnames) # Compute the accuracy as intersection of union, # i.e. (# of categories in both prediciton and GT) # divided by (# of categories in either prediction or GT # In case of only one GT category, the result will be 1.0, if # prediction is one category and this category matches GT # It is 1.0/(# of predicted top-1 categories), if the GT is # one of the predicted top-1 categories. # It is 0.0, if none of the predicted categories is correct classifier_accuracies.append( len(gt_categories & pred_categories) / len(gt_categories | pred_categories)) image['_classification_accuracy'] = classifier_accuracies[-1] # Distribute this accuracy across all predicted categories in the # confusion matrix assert len(gt_categories) == 1 gt_class_idx = classname_to_idx[list(gt_categories)[0]] for pred_category in pred_categories: pred_class_idx = classname_to_idx[pred_category] classifier_cm[gt_class_idx][pred_class_idx] += 1 # If we have classification results if len(classifier_accuracies) > 0: # Build confusion matrix as array from classifier_cm all_class_ids = sorted(classname_to_idx.values()) classifier_cm_array = np.array( [[classifier_cm[r_idx][c_idx] for c_idx in all_class_ids] for r_idx in all_class_ids], dtype=float) classifier_cm_array /= ( classifier_cm_array.sum(axis=1, keepdims=True) + 1e-7) # Print some statistics print("Finished computation of {} classification results".format( len(classifier_accuracies))) print("Mean accuracy: {}".format(np.mean(classifier_accuracies))) # Prepare confusion matrix output # Get CM matrix as string sio = io.StringIO() np.savetxt(sio, classifier_cm_array * 100, fmt='%5.1f') cm_str = sio.getvalue() # Get fixed-size classname for each idx idx_to_classname = {v: k for k, v in classname_to_idx.items()} classname_list = [ idx_to_classname[idx] for idx in sorted(classname_to_idx.values()) ] classname_headers = [ '{:<5}'.format(cname[:5]) for cname in classname_list ] # Prepend class name on each line and add to the top cm_str_lines = [' ' * 16 + ' '.join(classname_headers)] cm_str_lines += [ '{:>15}'.format(cn[:15]) + ' ' + cm_line for cn, cm_line in zip(classname_list, cm_str.splitlines()) ] # print formatted confusion matrix print("Confusion matrix: ") print(*cm_str_lines, sep='\n') # Plot confusion matrix # To manually add more space at bottom: plt.rcParams['figure.subplot.bottom'] = 0.1 # Add 0.5 to figsize for every class. For two classes, this will result in # fig = plt.figure(figsize=[4,4]) fig = vis_utils.plot_confusion_matrix(classifier_cm_array, classname_list, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues, vmax=1.0, use_colorbar=True, y_label=True) cm_figure_relative_filename = 'confusion_matrix.png' cm_figure_filename = os.path.join(output_dir, cm_figure_relative_filename) plt.savefig(cm_figure_filename) plt.close(fig) ##%% Render output # Write p/r table to .csv file in output directory pr_table_filename = os.path.join(output_dir, 'prec_recall.csv') precisions_recalls.to_csv(pr_table_filename, index=False) # Write precision/recall plot to .png file in output directory t = 'Precision-Recall curve: AP={:0.1%}, P@{:0.1%}={:0.1%}'.format( average_precision, target_recall, precision_at_target_recall) fig = vis_utils.plot_precision_recall_curve(precisions, recalls, t) pr_figure_relative_filename = 'prec_recall.png' pr_figure_filename = os.path.join(output_dir, pr_figure_relative_filename) plt.savefig(pr_figure_filename) # plt.show(block=False) plt.close(fig) ##%% Sample true/false positives/negatives with correct/incorrect top-1 # classification and render to html # Accumulate html image structs (in the format expected by write_html_image_lists) # for each category, e.g. 'tp', 'fp', ..., 'class_bird', ... images_html = collections.defaultdict(lambda: []) # Add default entries by accessing them for the first time [images_html[res] for res in ['tp', 'tpc', 'tpi', 'fp', 'tn', 'fn']] for res in images_html.keys(): os.makedirs(os.path.join(output_dir, res), exist_ok=True) count = 0 # i_row = 0; row = images_to_visualize.iloc[0] for i_row, row in tqdm(images_to_visualize.iterrows(), total=len(images_to_visualize)): image_relative_path = row['file'] # This should already have been normalized to either '/' or '\' image_id = ground_truth_indexed_db.filename_to_id.get( image_relative_path, None) if image_id is None: print('Warning: couldn' 't find ground truth for image {}'.format( image_relative_path)) continue image = ground_truth_indexed_db.image_id_to_image[image_id] annotations = ground_truth_indexed_db.image_id_to_annotations[ image_id] gt_status = image['_detection_status'] if gt_status > DetectionStatus.DS_MAX_DEFINITIVE_VALUE: print( 'Skipping image {}, does not have a definitive ground truth status' .format(i_row, gt_status)) continue gt_presence = bool(gt_status) gt_classes = CameraTrapJsonUtils.annotations_to_classnames( annotations, ground_truth_indexed_db.cat_id_to_name) gt_class_summary = ','.join(gt_classes) max_conf = row['max_detection_conf'] detections = row['detections'] detected = max_conf > confidence_threshold if gt_presence and detected: if '_classification_accuracy' not in image.keys(): res = 'tp' elif np.isclose(1, image['_classification_accuracy']): res = 'tpc' else: res = 'tpi' elif not gt_presence and detected: res = 'fp' elif gt_presence and not detected: res = 'fn' else: res = 'tn' display_name = '<b>Result type</b>: {}, <b>Presence</b>: {}, <b>Class</b>: {}, <b>Max conf</b>: {:0.2f}%, <b>Image</b>: {}'.format( res.upper(), str(gt_presence), gt_class_summary, max_conf * 100, image_relative_path) rendered_image_html_info = render_bounding_boxes( options.image_base_dir, image_relative_path, display_name, detections, res, detection_categories_map, classification_categories_map, options) if len(rendered_image_html_info) > 0: images_html[res].append(rendered_image_html_info) for gt_class in gt_classes: images_html['class_{}'.format(gt_class)].append( rendered_image_html_info) count += 1 # ...for each image in our sample print('{} images rendered'.format(count)) # Prepare the individual html image files image_counts = prepare_html_subpages(images_html, output_dir) # Write index.HTML all_tp_count = image_counts['tp'] + image_counts['tpc'] + image_counts[ 'tpi'] total_count = all_tp_count + image_counts['tn'] + image_counts[ 'fp'] + image_counts['fn'] index_page = """<html><body> <h2>Evaluation</h2> <h3>Sample images</h3> <p>A sample of {} images, annotated with detections above {:.1%} confidence.</p> True positives (TP) ({} or {:0.1%})<br/> -- <a href="tpc.html">with all correct top-1 predictions (TPC)</a> ({})<br/> -- <a href="tpi.html">with one or more incorrect top-1 prediction (TPI)</a> ({})<br/> -- <a href="tp.html">without classification evaluation</a> (*) ({})<br/> <a href="tn.html">True negatives (TN)</a> ({} or {:0.1%})<br/> <a href="fp.html">False positives (FP)</a> ({} or {:0.1%})<br/> <a href="fn.html">False negatives (FN)</a> ({} or {:0.1%})<br/> <p>(*) We do not evaluate the classification result of images, if the classification information is missing, if the image contains categories like 'empty' or 'human', or if the image has multiple classification labels.</p>""".format( count, confidence_threshold, all_tp_count, all_tp_count / total_count, image_counts['tpc'], image_counts['tpi'], image_counts['tp'], image_counts['tn'], image_counts['tn'] / total_count, image_counts['fp'], image_counts['fp'] / total_count, image_counts['fn'], image_counts['fn'] / total_count) index_page += """ <h3>Detection results</h3> <p>At a confidence threshold of {:0.1%}, precision={:0.1%}, recall={:0.1%}</p> <p><strong>Precision/recall summary for all {} images</strong></p><img src="{}"><br/> """.format(confidence_threshold, precision_at_confidence_threshold, recall_at_confidence_threshold, len(detection_results), pr_figure_relative_filename) if len(classifier_accuracies) > 0: index_page += """ <h3>Classification results</h3> <p>Classification accuracy: {:.2%}<br> The accuracy is computed only for images with exactly one classification label. The accuracy of an image is computed as 1/(number of unique detected top-1 classes), i.e. if the model detects multiple boxes with different top-1 classes, then the accuracy decreases and the image is put into 'TPI'.</p> <p>Confusion matrix:</p> <p><img src="{}"></p> <div style='font-family:monospace;display:block;'>{}</div> """.format(np.mean(classifier_accuracies), cm_figure_relative_filename, "<br>".join(cm_str_lines).replace(' ', ' ')) # Show links to each GT class index_page += "<h3>Images of specific classes:</h3>" # Add links to all available classes for cname in sorted(classname_to_idx.keys()): index_page += "<a href='class_{0}.html'>{0}</a> ({1})<br>".format( cname, len(images_html['class_{}'.format(cname)])) # Close body and html tag index_page += "</body></html>" output_html_file = os.path.join(output_dir, 'index.html') with open(output_html_file, 'w') as f: f.write(index_page) print('Finished writing html to {}'.format(output_html_file)) ##%% Otherwise, if we don't have ground truth... else: ##%% Sample detections/non-detections os.makedirs(os.path.join(output_dir, 'detections'), exist_ok=True) os.makedirs(os.path.join(output_dir, 'non_detections'), exist_ok=True) # Accumulate html image structs (in the format expected by write_html_image_lists) # for each category images_html = collections.defaultdict(lambda: []) # Add default entries by accessing them for the first time [images_html[res] for res in ['detections', 'non_detections']] for res in images_html.keys(): os.makedirs(os.path.join(output_dir, res), exist_ok=True) count = 0 has_classification_info = False # i_row = 0; row = images_to_visualize.iloc[0] for i_row, row in tqdm(images_to_visualize.iterrows(), total=len(images_to_visualize)): image_relative_path = row['file'] # This should already have been normalized to either '/' or '\' max_conf = row['max_detection_conf'] detections = row['detections'] detected = True if max_conf > confidence_threshold else False if detected: res = 'detections' else: res = 'non_detections' display_name = '<b>Result type</b>: {}, <b>Image</b>: {}, <b>Max conf</b>: {}'.format( res, image_relative_path, max_conf) rendered_image_html_info = render_bounding_boxes( options.image_base_dir, image_relative_path, display_name, detections, res, detection_categories_map, classification_categories_map, options) if len(rendered_image_html_info) > 0: images_html[res].append(rendered_image_html_info) for det in detections: if 'classifications' in det: has_classification_info = True top1_class = classification_categories_map[ det['classifications'][0][0]] images_html['class_{}'.format(top1_class)].append( rendered_image_html_info) count += 1 # ...for each image in our sample print('{} images rendered'.format(count)) # Prepare the individual html image files image_counts = prepare_html_subpages(images_html, output_dir) # Write index.HTML total_images = image_counts['detections'] + image_counts[ 'non_detections'] index_page = """<html><body> <h2>Visualization of results</h2> <p>A sample of {} images, annotated with detections above {:.1%} confidence.</p> <h3>Sample images</h3> <a href="detections.html">Detections</a> ({}, {:.1%})<br/> <a href="non_detections.html">Non-detections</a> ({}, {:.1%})<br/>""".format( count, confidence_threshold, image_counts['detections'], image_counts['detections'] / total_images, image_counts['non_detections'], image_counts['non_detections'] / total_images) if has_classification_info: index_page += "<h3>Images of detected classes</h3>" index_page += "<p>The same image might appear under multiple classes if multiple species were detected.</p>" # Add links to all available classes for cname in sorted(classification_categories_map.values()): ccount = len(images_html['class_{}'.format(cname)]) if ccount > 0: index_page += "<a href='class_{0}.html'>{0}</a> ({1})<br>".format( cname, ccount) index_page += "</body></html>" output_html_file = os.path.join(output_dir, 'index.html') with open(output_html_file, 'w') as f: f.write(index_page) print('Finished writing html to {}'.format(output_html_file)) # ...if we do/don't have ground truth ppresults = PostProcessingResults() ppresults.output_html_file = output_html_file return ppresults
def process_batch_results(options): ##%% Expand some options for convenience output_dir = options.output_dir confidence_threshold = options.confidence_threshold ##%% Prepare output dir os.makedirs(output_dir, exist_ok=True) ##%% Load ground truth if available ground_truth_indexed_db = None if options.ground_truth_json_file and len( options.ground_truth_json_file) > 0: ground_truth_indexed_db = IndexedJsonDb( options.ground_truth_json_file, b_normalize_paths=True, filename_replacements=options.ground_truth_filename_replacements) # Mark images in the ground truth as positive or negative (nNegative, nPositive, nUnknown, nAmbiguous) = mark_detection_status( ground_truth_indexed_db, negative_classes=options.negative_classes, unknown_classes=options.unlabeled_classes) print( 'Finished loading and indexing ground truth: {} negative, {} positive, {} unknown, {} ambiguous' .format(nNegative, nPositive, nUnknown, nAmbiguous)) ##%% Load detection results detection_results = load_api_results( options.detector_output_file, normalize_paths=True, filename_replacements=options.detector_output_filename_replacements) # Add a column (pred_detection_label) to indicate predicted detection status import numpy as np detection_results['pred_detection_label'] = \ np.where(detection_results['max_confidence'] >= options.confidence_threshold, DetectionStatus.DS_POSITIVE, DetectionStatus.DS_NEGATIVE) nPositives = sum(detection_results['pred_detection_label'] == DetectionStatus.DS_POSITIVE) print( 'Finished loading and preprocessing {} rows from detector output, predicted {} positives' .format(len(detection_results), nPositives)) ##%% If we have ground truth, remove images we can't match to ground truth # ground_truth_indexed_db.db['images'][0] if ground_truth_indexed_db is not None: b_match = [False] * len(detection_results) detector_files = detection_results['image_path'].tolist() for iFn, fn in enumerate(detector_files): # assert fn in ground_truth_indexed_db.filename_to_id, 'Could not find ground truth for row {} ({})'.format(iFn,fn) if fn in fn in ground_truth_indexed_db.filename_to_id: b_match[iFn] = True print('Confirmed filename matches to ground truth for {} of {} files'. format(sum(b_match), len(detector_files))) detection_results = detection_results[b_match] detector_files = detection_results['image_path'].tolist() print('Trimmed detection results to {} files'.format( len(detector_files))) ##%% Sample images for visualization images_to_visualize = detection_results if options.num_images_to_sample > 0 and options.num_images_to_sample < len( detection_results): images_to_visualize = images_to_visualize.sample( options.num_images_to_sample, random_state=options.sample_seed) ##%% Fork here depending on whether or not ground truth is available output_html_file = '' # If we have ground truth, we'll compute precision/recall and sample tp/fp/tn/fn. # # Otherwise we'll just visualize detections/non-detections. if ground_truth_indexed_db is not None: ##%% Compute precision/recall # numpy array of detection probabilities p_detection = detection_results['max_confidence'].values n_detections = len(p_detection) # numpy array of bools (0.0/1.0) gt_detections = np.zeros(n_detections, dtype=float) for iDetection, fn in enumerate(detector_files): image_id = ground_truth_indexed_db.filename_to_id[fn] image = ground_truth_indexed_db.image_id_to_image[image_id] detection_status = image['_detection_status'] if detection_status == DetectionStatus.DS_NEGATIVE: gt_detections[iDetection] = 0.0 elif detection_status == DetectionStatus.DS_POSITIVE: gt_detections[iDetection] = 1.0 else: gt_detections[iDetection] = -1.0 # Don't include ambiguous/unknown ground truth in precision/recall analysis b_valid_ground_truth = gt_detections >= 0.0 p_detection_pr = p_detection[b_valid_ground_truth] gt_detections_pr = gt_detections[b_valid_ground_truth] print('Including {} of {} values in p/r analysis'.format( np.sum(b_valid_ground_truth), len(b_valid_ground_truth))) precisions, recalls, thresholds = precision_recall_curve( gt_detections_pr, p_detection_pr) # For completeness, include the result at a confidence threshold of 1.0 thresholds = np.append(thresholds, [1.0]) precisions_recalls = pd.DataFrame( data={ 'confidence_threshold': thresholds, 'precision': precisions, 'recall': recalls }) # Compute and print summary statistics average_precision = average_precision_score(gt_detections_pr, p_detection_pr) print('Average precision: {:.2f}'.format(average_precision)) # Thresholds go up throughout precisions/recalls/thresholds; find the last # value where recall is at or above target. That's our precision @ target recall. target_recall = 0.9 b_above_target_recall = np.where(recalls >= target_recall) if not np.any(b_above_target_recall): precision_at_target_recall = 0.0 else: i_target_recall = np.argmax(b_above_target_recall) precision_at_target_recall = precisions[i_target_recall] print('Precision at {:.2f} recall: {:.2f}'.format( target_recall, precision_at_target_recall)) cm = confusion_matrix(gt_detections_pr, np.array(p_detection_pr) > confidence_threshold) # Flatten the confusion matrix tn, fp, fn, tp = cm.ravel() precision_at_confidence_threshold = tp / (tp + fp) recall_at_confidence_threshold = tp / (tp + fn) f1 = 2.0 * (precision_at_confidence_threshold * recall_at_confidence_threshold) / \ (precision_at_confidence_threshold + recall_at_confidence_threshold) print( 'At a confidence threshold of {:.2f}, precision={:.2f}, recall={:.2f}, f1={:.2f}' .format(confidence_threshold, precision_at_confidence_threshold, recall_at_confidence_threshold, f1)) ##%% Render output # Write p/r table to .csv file in output directory pr_table_filename = os.path.join(output_dir, 'prec_recall.csv') precisions_recalls.to_csv(pr_table_filename, index=False) # Write precision/recall plot to .png file in output directory step_kwargs = ({'step': 'post'}) fig = plt.figure() plt.step(recalls, precisions, color='b', alpha=0.2, where='post') plt.fill_between(recalls, precisions, alpha=0.2, color='b', **step_kwargs) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.05]) t = 'Precision-Recall curve: AP={:0.2f}, P@{:0.2f}={:0.2f}'.format( average_precision, target_recall, precision_at_target_recall) plt.title(t) pr_figure_relative_filename = 'prec_recall.png' pr_figure_filename = os.path.join(output_dir, pr_figure_relative_filename) plt.savefig(pr_figure_filename) # plt.show(block=False) plt.close(fig) ##%% Sample true/false positives/negatives and render to html os.makedirs(os.path.join(output_dir, 'tp'), exist_ok=True) os.makedirs(os.path.join(output_dir, 'fp'), exist_ok=True) os.makedirs(os.path.join(output_dir, 'tn'), exist_ok=True) os.makedirs(os.path.join(output_dir, 'fn'), exist_ok=True) # Accumulate html image structs (in the format expected by write_html_image_lists) # for each category images_html = {'tp': [], 'fp': [], 'tn': [], 'fn': []} count = 0 # i_row = 0; row = images_to_visualize.iloc[0] for i_row, row in tqdm(images_to_visualize.iterrows(), total=len(images_to_visualize)): image_relative_path = row['image_path'] # This should already have been normalized to either '/' or '\' image_id = ground_truth_indexed_db.filename_to_id.get( image_relative_path, None) if image_id is None: print('Warning: couldn' 't find ground truth for image {}'.format( image_relative_path)) continue image_info = ground_truth_indexed_db.image_id_to_image[image_id] annotations = ground_truth_indexed_db.image_id_to_annotations[ image_id] gt_status = image_info['_detection_status'] if gt_status > DetectionStatus.DS_MAX_DEFINITIVE_VALUE: print( 'Skipping image {}, does not have a definitive ground truth status' .format(i_row, gt_status)) continue gt_presence = bool(gt_status) gt_class_name = CameraTrapJsonUtils.annotationsToString( annotations, ground_truth_indexed_db.cat_id_to_name) max_conf = row['max_confidence'] boxes_and_scores = row['detections'] detected = True if max_conf > confidence_threshold else False if gt_presence and detected: res = 'tp' elif not gt_presence and detected: res = 'fp' elif gt_presence and not detected: res = 'fn' else: res = 'tn' display_name = '<b>Result type</b>: {}, <b>Presence</b>: {}, <b>Class</b>: {}, <b>Max conf</b>: {:0.2f}%, <b>Image</b>: {}'.format( res.upper(), str(gt_presence), gt_class_name, max_conf * 100, image_relative_path) rendered_image_html_info = render_bounding_boxes( options.image_base_dir, image_relative_path, display_name, boxes_and_scores, res, options) if len(rendered_image_html_info) > 0: images_html[res].append(rendered_image_html_info) count += 1 # ...for each image in our sample print('{} images rendered'.format(count)) # Prepare the individual html image files image_counts = prepare_html_subpages(images_html, output_dir) # Write index.HTML index_page = """<html><body> <p><strong>A sample of {} images, annotated with detections above {:.1f}% confidence.</strong></p> <a href="tp.html">True positives (tp)</a> ({})<br/> <a href="tn.html">True negatives (tn)</a> ({})<br/> <a href="fp.html">False positives (fp)</a> ({})<br/> <a href="fn.html">False negatives (fn)</a> ({})<br/> <p>At a confidence threshold of {:0.1f}%, precision={:0.2f}, recall={:0.2f}</p> <p><strong>Precision/recall summary for all {} images</strong></p><img src="{}"><br/> </body></html>""".format( count, confidence_threshold * 100, image_counts['tp'], image_counts['tn'], image_counts['fp'], image_counts['fn'], confidence_threshold * 100, precision_at_confidence_threshold, recall_at_confidence_threshold, len(detection_results), pr_figure_relative_filename) output_html_file = os.path.join(output_dir, 'index.html') with open(output_html_file, 'w') as f: f.write(index_page) print('Finished writing html to {}'.format(output_html_file)) ##%% Otherwise, if we don't have ground truth... else: ##%% Sample detections/non-detections os.makedirs(os.path.join(output_dir, 'detections'), exist_ok=True) os.makedirs(os.path.join(output_dir, 'non_detections'), exist_ok=True) # Accumulate html image structs (in the format expected by write_html_image_lists) # for each category images_html = { 'detections': [], 'non_detections': [], } count = 0 # i_row = 0; row = images_to_visualize.iloc[0] for i_row, row in tqdm(images_to_visualize.iterrows(), total=len(images_to_visualize)): image_relative_path = row['image_path'] # This should already have been normalized to either '/' or '\' max_conf = row['max_confidence'] boxes_and_scores = row['detections'] detected = True if max_conf > confidence_threshold else False if detected: res = 'detections' else: res = 'non_detections' display_name = '<b>Result type</b>: {}, <b>Image</b>: {}, <b>Max conf</b>: {}'.format( res, image_relative_path, max_conf) rendered_image_html_info = render_bounding_boxes( options.image_base_dir, image_relative_path, display_name, boxes_and_scores, res, options) if len(rendered_image_html_info) > 0: images_html[res].append(rendered_image_html_info) count += 1 # ...for each image in our sample print('{} images rendered'.format(count)) # Prepare the individual html image files image_counts = prepare_html_subpages(images_html, output_dir) # Write index.HTML index_page = """<html><body> <p><strong>A sample of {} images, annotated with detections above {:.1f}% confidence.</strong></p> <a href="detections.html">Detections</a> ({})<br/> <a href="non_detections.html">Non-detections</a> ({})<br/> </body></html>""".format(count, confidence_threshold * 100, image_counts['detections'], image_counts['non_detections']) output_html_file = os.path.join(output_dir, 'index.html') with open(output_html_file, 'w') as f: f.write(index_page) print('Finished writing html to {}'.format(output_html_file)) # ...if we do/don't have ground truth ppresults = PostProcessingResults() ppresults.output_html_file = output_html_file return ppresults
def find_repeat_detections(inputFilename, outputFilename=None, options=None): ##%% Input handling if options is None: options = RepeatDetectionOptions() # Validate some options if options.customDirNameFunction is not None: assert options.nDirLevelsFromLeaf == 0, 'Cannot mix custom dir name functions with nDirLevelsFromLeaf' if options.nDirLevelsFromLeaf != 0: assert options.customDirNameFunction is None, 'Cannot mix custom dir name functions with nDirLevelsFromLeaf' if options.filterFileToLoad is not None and len( options.filterFileToLoad) > 0: print('Bypassing detection-finding, loading from {}'.format( options.filterFileToLoad)) # Load the filtering file detectionIndexFileName = options.filterFileToLoad sIn = open(detectionIndexFileName, 'r').read() detectionInfo = jsonpickle.decode(sIn) filteringBaseDir = os.path.dirname(options.filterFileToLoad) suspiciousDetections = detectionInfo['suspiciousDetections'] # Load the same options we used when finding repeat detections options = detectionInfo['options'] # ...except for things that explicitly tell this function not to # find repeat detections. options.filterFileToLoad = detectionIndexFileName options.bWriteFilteringFolder = False options.bRenderHtml = False # ...if we're loading from an existing filtering file toReturn = RepeatDetectionResults() # Check early to avoid problems with the output folder if options.bWriteFilteringFolder or options.bRenderHtml: assert options.outputBase is not None and len(options.outputBase) > 0 os.makedirs(options.outputBase, exist_ok=True) # Load file detectionResults, otherFields = load_api_results( inputFilename, normalize_paths=True, filename_replacements=options.filenameReplacements) toReturn.detectionResults = detectionResults toReturn.otherFields = otherFields # detectionResults[detectionResults['failure'].notna()] # Before doing any real work, make sure we can *probably* access images # This is just a cursory check on the first image, but it heads off most # problems related to incorrect mount points, etc. Better to do this before # spending 20 minutes finding repeat detections. if options.bWriteFilteringFolder or options.bRenderHtml: if not is_sas_url(options.imageBase): row = detectionResults.iloc[0] relativePath = row['file'] for s in options.filenameReplacements.keys(): relativePath = relativePath.replace( s, options.filenameReplacements[s]) absolutePath = os.path.join(options.imageBase, relativePath) assert os.path.isfile( absolutePath), 'Could not find file {}'.format(absolutePath) ##%% Separate files into directories # This will be a map from a directory name to smaller data frames rowsByDirectory = {} # This is a mapping back into the rows of the original table filenameToRow = {} # TODO: in the case where we're loading an existing set of FPs after # manual filtering, we should load these data frames too, rather than # re-building them from the input. print('Separating files into directories...') nCustomDirReplacements = 0 # iRow = 0; row = detectionResults.iloc[0] for iRow, row in detectionResults.iterrows(): relativePath = row['file'] if options.customDirNameFunction is not None: basicDirName = os.path.dirname(relativePath.replace('\\', '/')) dirName = options.customDirNameFunction(relativePath) if basicDirName != dirName: nCustomDirReplacements += 1 else: dirName = os.path.dirname(relativePath) if len(dirName) == 0: assert options.nDirLevelsFromLeaf == 0, 'Can' 't use the dirLevelsFromLeaf option with flat filenames' else: if options.nDirLevelsFromLeaf > 0: iLevel = 0 while (iLevel < options.nDirLevelsFromLeaf): iLevel += 1 dirName = os.path.dirname(dirName) assert len(dirName) > 0 if not dirName in rowsByDirectory: # Create a new DataFrame with just this row # rowsByDirectory[dirName] = pd.DataFrame(row) rowsByDirectory[dirName] = [] rowsByDirectory[dirName].append(row) assert relativePath not in filenameToRow filenameToRow[relativePath] = iRow # ...for each unique detection if options.customDirNameFunction is not None: print('Custom dir name function made {} replacements (of {} images)'. format(nCustomDirReplacements, len(detectionResults))) # Convert lists of rows to proper DataFrames dirs = list(rowsByDirectory.keys()) for d in dirs: rowsByDirectory[d] = pd.DataFrame(rowsByDirectory[d]) toReturn.rowsByDirectory = rowsByDirectory toReturn.filenameToRow = filenameToRow print('Finished separating {} files into {} directories'.format( len(detectionResults), len(rowsByDirectory))) ##% Look for matches (or load them from file) dirsToSearch = list(rowsByDirectory.keys()) if options.debugMaxDir > 0: dirsToSearch = dirsToSearch[0:options.debugMaxDir] # Are we actually looking for matches, or just loading from a file? if len(options.filterFileToLoad) == 0: # length-nDirs list of lists of DetectionLocation objects suspiciousDetections = [None] * len(dirsToSearch) # We're actually looking for matches... print('Finding similar detections...') allCandidateDetections = [None] * len(dirsToSearch) if not options.bParallelizeComparisons: options.pbar = None # iDir = 4; dirName = dirsToSearch[iDir] # for iDir, dirName in enumerate(tqdm(dirsToSearch)): for iDir, dirName in enumerate(dirsToSearch): print('Processing dir {} of {}: {}'.format( iDir, len(dirsToSearch), dirName)) allCandidateDetections[iDir] = find_matches_in_directory( dirName, options, rowsByDirectory) else: options.pbar = tqdm(total=len(dirsToSearch)) allCandidateDetections = Parallel( n_jobs=options.nWorkers, prefer='threads')(delayed(find_matches_in_directory)( dirName, options, rowsByDirectory) for dirName in tqdm(dirsToSearch)) print('\nFinished looking for similar bounding boxes') ##%% Find suspicious locations based on match results print('Filtering out repeat detections...') nImagesWithSuspiciousDetections = 0 nSuspiciousDetections = 0 # For each directory # # iDir = 51 for iDir in range(len(dirsToSearch)): # A list of DetectionLocation objects suspiciousDetectionsThisDir = [] # A list of DetectionLocation objects candidateDetectionsThisDir = allCandidateDetections[iDir] for iLocation, candidateLocation in enumerate( candidateDetectionsThisDir): # occurrenceList is a list of file/detection pairs nOccurrences = len(candidateLocation.instances) if nOccurrences < options.occurrenceThreshold: continue nImagesWithSuspiciousDetections += nOccurrences nSuspiciousDetections += 1 suspiciousDetectionsThisDir.append(candidateLocation) suspiciousDetections[iDir] = suspiciousDetectionsThisDir # Sort the above-threshold detections for easier review if options.smartSort is not None: suspiciousDetections[iDir] = sort_detections_for_directory( suspiciousDetections[iDir], options) print('Found {} suspicious detections in directory {} ({})'.format( len(suspiciousDetections[iDir]), iDir, dirsToSearch[iDir])) # ...for each directory print( 'Finished searching for repeat detections\nFound {} unique detections on {} images that are suspicious' .format(nSuspiciousDetections, nImagesWithSuspiciousDetections)) # If we're just loading detections from a file... else: assert len(suspiciousDetections) == len(dirsToSearch) nDetectionsRemoved = 0 nDetectionsLoaded = 0 # We're skipping detection-finding, but to see which images are actually legit false # positives, we may be looking for physical files or loading from a text file. fileList = None if options.filteredFileListToLoad is not None: with open(options.filteredFileListToLoad) as f: fileList = f.readlines() fileList = [x.strip() for x in fileList] nSuspiciousDetections = sum([len(x) for x in suspiciousDetections]) print( 'Loaded false positive list from file, will remove {} of {} suspicious detections' .format(len(fileList), nSuspiciousDetections)) # For each directory # iDir = 0; detections = suspiciousDetections[0] # # suspiciousDetections is an array of DetectionLocation objects, # one per directory. for iDir, detections in enumerate(suspiciousDetections): bValidDetection = [True] * len(detections) nDetectionsLoaded += len(detections) # For each detection that was present before filtering # iDetection = 0; detection = detections[iDetection] for iDetection, detection in enumerate(detections): # Are we checking the directory to see whether detections were actually false # positives, or reading from a list? if fileList is None: # Is the image still there? imageFullPath = os.path.join( filteringBaseDir, detection.sampleImageRelativeFileName) # If not, remove this from the list of suspicious detections if not os.path.isfile(imageFullPath): nDetectionsRemoved += 1 bValidDetection[iDetection] = False else: if detection.sampleImageRelativeFileName not in fileList: nDetectionsRemoved += 1 bValidDetection[iDetection] = False # ...for each detection nRemovedThisDir = len(bValidDetection) - sum(bValidDetection) if nRemovedThisDir > 0: print('Removed {} of {} detections from directory {}'.format( nRemovedThisDir, len(detections), iDir)) detectionsFiltered = list(compress(detections, bValidDetection)) suspiciousDetections[iDir] = detectionsFiltered # ...for each directory print('Removed {} of {} total detections via manual filtering'.format( nDetectionsRemoved, nDetectionsLoaded)) # ...if we are/aren't finding detections (vs. loading from file) toReturn.suspiciousDetections = suspiciousDetections if options.bRenderHtml: # Render problematic locations with html (loop) print('Rendering html') nDirs = len(dirsToSearch) directoryHtmlFiles = [None] * nDirs if options.bParallelizeRendering: # options.pbar = tqdm(total=nDirs) options.pbar = None directoryHtmlFiles = Parallel( n_jobs=options.nWorkers, prefer='threads')(delayed(render_images_for_directory)( iDir, directoryHtmlFiles, suspiciousDetections, options) for iDir in tqdm(range(nDirs))) else: options.pbar = None # For each directory # iDir = 51 for iDir in range(nDirs): # Add this directory to the master list of html files directoryHtmlFiles[iDir] = render_images_for_directory( iDir, directoryHtmlFiles, suspiciousDetections, options) # ...for each directory # Write master html file masterHtmlFile = os.path.join(options.outputBase, 'index.html') os.makedirs(options.outputBase, exist_ok=True) toReturn.masterHtmlFile = masterHtmlFile with open(masterHtmlFile, 'w') as fHtml: fHtml.write('<html><body>\n') fHtml.write( '<h2><b>Repeat detections by directory</b></h2></br>\n') for iDir, dirHtmlFile in enumerate(directoryHtmlFiles): if dirHtmlFile is None: continue relPath = os.path.relpath(dirHtmlFile, options.outputBase) dirName = dirsToSearch[iDir] # Remove unicode characters before formatting relPath = relPath.encode('ascii', 'ignore').decode('ascii') dirName = dirName.encode('ascii', 'ignore').decode('ascii') fHtml.write('<a href={}>{}</a><br/>\n'.format( relPath, dirName)) fHtml.write('</body></html>\n') # ...if we're rendering html toReturn.allRowsFiltered = update_detection_table(toReturn, options, outputFilename) # Create filtering directory if options.bWriteFilteringFolder: print('Creating filtering folder...') dateString = datetime.now().strftime('%Y.%m.%d.%H.%M.%S') filteringDir = os.path.join(options.outputBase, 'filtering_' + dateString) os.makedirs(filteringDir, exist_ok=True) # iDir = 0; suspiciousDetectionsThisDir = suspiciousDetections[iDir] for iDir, suspiciousDetectionsThisDir in enumerate( tqdm(suspiciousDetections)): # suspiciousDetectionsThisDir is a list of DetectionLocation objects # iDetection = 0; detection = suspiciousDetectionsThisDir[0] for iDetection, detection in enumerate( suspiciousDetectionsThisDir): instance = detection.instances[0] relativePath = instance.filename if detection.clusterLabel is not None: clusterString = '_c{:0>4d}'.format(detection.clusterLabel) else: clusterString = '' outputRelativePath = 'dir{:0>4d}_det{:0>4d}{}_n{:0>4d}.jpg'.format( iDir, iDetection, clusterString, len(detection.instances)) outputFullPath = os.path.join(filteringDir, outputRelativePath) if is_sas_url(options.imageBase): inputFullPath = relative_sas_url(options.imageBase, relativePath) else: inputFullPath = os.path.join(options.imageBase, relativePath) assert (os.path.isfile(inputFullPath) ), 'Not a file: {}'.format(inputFullPath) try: render_bounding_box(detection, inputFullPath, outputFullPath, lineWidth=options.lineThickness, expansion=options.boxExpansion) except Exception as e: print( 'Warning: error rendering bounding box from {} to {}: {}' .format(inputFullPath, outputFullPath, e)) if options.bFailOnRenderError: raise detection.sampleImageRelativeFileName = outputRelativePath # Write out the detection index detectionIndexFileName = os.path.join(filteringDir, DETECTION_INDEX_FILE_NAME) jsonpickle.set_encoder_options('json', sort_keys=True, indent=4) detectionInfo = {} detectionInfo['suspiciousDetections'] = suspiciousDetections options.pbar = None detectionInfo['options'] = options s = jsonpickle.encode(detectionInfo, make_refs=False) with open(detectionIndexFileName, 'w') as f: f.write(s) toReturn.filterFile = detectionIndexFileName print('Done') # ...if we're writing filtering info return toReturn