def process_batch_results(options):
ppresults = PostProcessingResults()
##%% Expand some options for convenience
output_dir = options.output_dir
##%% Prepare output dir
os.makedirs(output_dir, exist_ok=True)
##%% Load ground truth if available
ground_truth_indexed_db = None
if options.ground_truth_json_file and len(options.ground_truth_json_file) > 0:
ground_truth_indexed_db = IndexedJsonDb(options.ground_truth_json_file, b_normalize_paths=True,
filename_replacements=options.ground_truth_filename_replacements)
# Mark images in the ground truth as positive or negative
n_negative, n_positive, n_unknown, n_ambiguous = mark_detection_status(ground_truth_indexed_db,
negative_classes=options.negative_classes, unknown_classes=options.unlabeled_classes)
print('Finished loading and indexing ground truth: {} negative, {} positive, {} unknown, {} ambiguous'.format(
n_negative, n_positive, n_unknown, n_ambiguous))
##%% Load detection results
if options.api_detection_results is None:
detection_results, other_fields = load_api_results(options.api_output_file,
normalize_paths=True,
filename_replacements=options.api_output_filename_replacements)
ppresults.api_detection_results = detection_results
ppresults.api_other_fields = other_fields
else:
print('Bypassing detection results loading...')
assert options.api_other_fields is not None
detection_results = options.api_detection_results
other_fields = options.api_other_fields
detection_categories_map = other_fields['detection_categories']
if 'classification_categories' in other_fields:
classification_categories_map = other_fields['classification_categories']
else:
classification_categories_map = {}
# Add a column (pred_detection_label) to indicate predicted detection status, not separating out the classes
if options.include_almost_detections:
detection_results['pred_detection_label'] = DetectionStatus.DS_ALMOST
confidences = detection_results['max_detection_conf']
detection_results.loc[confidences >= options.confidence_threshold,'pred_detection_label'] = DetectionStatus.DS_POSITIVE
detection_results.loc[confidences < options.almost_detection_confidence_threshold,'pred_detection_label'] = DetectionStatus.DS_NEGATIVE
else:
detection_results['pred_detection_label'] = \
np.where(detection_results['max_detection_conf'] >= options.confidence_threshold,
DetectionStatus.DS_POSITIVE, DetectionStatus.DS_NEGATIVE)
n_positives = sum(detection_results['pred_detection_label'] == DetectionStatus.DS_POSITIVE)
print('Finished loading and preprocessing {} rows from detector output, predicted {} positives'.format(
len(detection_results), n_positives))
if options.include_almost_detections:
n_almosts = sum(detection_results['pred_detection_label'] == DetectionStatus.DS_ALMOST)
print('...and {} almost-positives'.format(n_almosts))
##%% If we have ground truth, remove images we can't match to ground truth
if ground_truth_indexed_db is not None:
b_match = [False] * len(detection_results)
detector_files = detection_results['file'].tolist()
# fn = detector_files[0]; print(fn)
for i_fn, fn in enumerate(detector_files):
# assert fn in ground_truth_indexed_db.filename_to_id, 'Could not find ground truth for row {} ({})'.format(i_fn,fn)
if fn in ground_truth_indexed_db.filename_to_id:
b_match[i_fn] = True
print('Confirmed filename matches to ground truth for {} of {} files'.format(sum(b_match), len(detector_files)))
detection_results = detection_results[b_match]
detector_files = detection_results['file'].tolist()
assert len(detector_files) > 0, 'No detection files available, possible ground truth path issue?'
print('Trimmed detection results to {} files'.format(len(detector_files)))
##%% Sample images for visualization
images_to_visualize = detection_results
if options.num_images_to_sample > 0 and options.num_images_to_sample <= len(detection_results):
images_to_visualize = images_to_visualize.sample(options.num_images_to_sample, random_state=options.sample_seed)
output_html_file = ''
style_header = """<head>
<style type="text/css">
<!--
a { text-decoration:none; }
body { font-family:segoe ui, calibri, "trebuchet ms", verdana, arial, sans-serif; }
div.contentdiv { margin-left:20px; }
-->
</style>
</head>"""
##%% Fork here depending on whether or not ground truth is available
# If we have ground truth, we'll compute precision/recall and sample tp/fp/tn/fn.
#
# Otherwise we'll just visualize detections/non-detections.
if ground_truth_indexed_db is not None:
##%% Detection evaluation: compute precision/recall
# numpy array of detection probabilities
p_detection = detection_results['max_detection_conf'].values
n_detections = len(p_detection)
# numpy array of bools (0.0/1.0), and -1 as null value
gt_detections = np.zeros(n_detections, dtype=float)
for i_detection, fn in enumerate(detector_files):
image_id = ground_truth_indexed_db.filename_to_id[fn]
image = ground_truth_indexed_db.image_id_to_image[image_id]
detection_status = image['_detection_status']
if detection_status == DetectionStatus.DS_NEGATIVE:
gt_detections[i_detection] = 0.0
elif detection_status == DetectionStatus.DS_POSITIVE:
gt_detections[i_detection] = 1.0
else:
gt_detections[i_detection] = -1.0
# Don't include ambiguous/unknown ground truth in precision/recall analysis
b_valid_ground_truth = gt_detections >= 0.0
p_detection_pr = p_detection[b_valid_ground_truth]
gt_detections_pr = gt_detections[b_valid_ground_truth]
print('Including {} of {} values in p/r analysis'.format(np.sum(b_valid_ground_truth),
len(b_valid_ground_truth)))
precisions, recalls, thresholds = precision_recall_curve(gt_detections_pr, p_detection_pr)
# For completeness, include the result at a confidence threshold of 1.0
thresholds = np.append(thresholds, [1.0])
precisions_recalls = pd.DataFrame(data={
'confidence_threshold': thresholds,
'precision': precisions,
'recall': recalls
})
# Compute and print summary statistics
average_precision = average_precision_score(gt_detections_pr, p_detection_pr)
print('Average precision: {:.1%}'.format(average_precision))
# Thresholds go up throughout precisions/recalls/thresholds; find the last
# value where recall is at or above target. That's our precision @ target recall.
target_recall = 0.9
b_above_target_recall = np.where(recalls >= target_recall)
if not np.any(b_above_target_recall):
precision_at_target_recall = 0.0
else:
i_target_recall = np.argmax(b_above_target_recall)
precision_at_target_recall = precisions[i_target_recall]
print('Precision at {:.1%} recall: {:.1%}'.format(target_recall, precision_at_target_recall))
cm = confusion_matrix(gt_detections_pr, np.array(p_detection_pr) > options.confidence_threshold)
# Flatten the confusion matrix
tn, fp, fn, tp = cm.ravel()
precision_at_confidence_threshold = tp / (tp + fp)
recall_at_confidence_threshold = tp / (tp + fn)
f1 = 2.0 * (precision_at_confidence_threshold * recall_at_confidence_threshold) / \
(precision_at_confidence_threshold + recall_at_confidence_threshold)
print('At a confidence threshold of {:.1%}, precision={:.1%}, recall={:.1%}, f1={:.1%}'.format(
options.confidence_threshold, precision_at_confidence_threshold, recall_at_confidence_threshold, f1))
##%% Collect classification results, if they exist
classifier_accuracies = []
# Mapping of classnames to idx for the confusion matrix.
#
# The lambda is actually kind of a hack, because we use assume that
# the following code does not reassign classname_to_idx
classname_to_idx = collections.defaultdict(lambda: len(classname_to_idx))
# Confusion matrix as defaultdict of defaultdict
#
# Rows / first index is ground truth, columns / second index is predicted category
classifier_cm = collections.defaultdict(lambda: collections.defaultdict(lambda: 0))
# iDetection = 0; fn = detector_files[iDetection]; print(fn)
assert len(detector_files) == len(detection_results)
for iDetection,fn in enumerate(detector_files):
image_id = ground_truth_indexed_db.filename_to_id[fn]
image = ground_truth_indexed_db.image_id_to_image[image_id]
detections = detection_results['detections'].iloc[iDetection]
pred_class_ids = [det['classifications'][0][0] \
for det in detections if 'classifications' in det.keys()]
pred_classnames = [classification_categories_map[pd] for pd in pred_class_ids]
# If this image has classification predictions, and an unambiguous class
# annotated, and is a positive image...
if len(pred_classnames) > 0 \
and '_unambiguous_category' in image.keys() \
and image['_detection_status'] == DetectionStatus.DS_POSITIVE:
# The unambiguous category, we make this a set for easier handling afterward
gt_categories = set([image['_unambiguous_category']])
pred_categories = set(pred_classnames)
# Compute the accuracy as intersection of union,
# i.e. (# of categories in both prediciton and GT)
# divided by (# of categories in either prediction or GT
#
# In case of only one GT category, the result will be 1.0, if
# prediction is one category and this category matches GT
#
# It is 1.0/(# of predicted top-1 categories), if the GT is
# one of the predicted top-1 categories.
#
# It is 0.0, if none of the predicted categories is correct
classifier_accuracies.append(
len(gt_categories & pred_categories)
/ len(gt_categories | pred_categories)
)
image['_classification_accuracy'] = classifier_accuracies[-1]
# Distribute this accuracy across all predicted categories in the
# confusion matrix
assert len(gt_categories) == 1
gt_class_idx = classname_to_idx[list(gt_categories)[0]]
for pred_category in pred_categories:
pred_class_idx = classname_to_idx[pred_category]
classifier_cm[gt_class_idx][pred_class_idx] += 1
# ...for each file in the detection results
# If we have classification results
if len(classifier_accuracies) > 0:
# Build confusion matrix as array from classifier_cm
all_class_ids = sorted(classname_to_idx.values())
classifier_cm_array = np.array(
[[classifier_cm[r_idx][c_idx] for c_idx in all_class_ids] for r_idx in all_class_ids], dtype=float)
classifier_cm_array /= (classifier_cm_array.sum(axis=1, keepdims=True) + 1e-7)
# Print some statistics
print("Finished computation of {} classification results".format(len(classifier_accuracies)))
print("Mean accuracy: {}".format(np.mean(classifier_accuracies)))
# Prepare confusion matrix output
# Get confusion matrix as string
sio = io.StringIO()
np.savetxt(sio, classifier_cm_array * 100, fmt='%5.1f')
cm_str = sio.getvalue()
# Get fixed-size classname for each idx
idx_to_classname = {v:k for k,v in classname_to_idx.items()}
classname_list = [idx_to_classname[idx] for idx in sorted(classname_to_idx.values())]
classname_headers = ['{:<5}'.format(cname[:5]) for cname in classname_list]
# Prepend class name on each line and add to the top
cm_str_lines = [' ' * 16 + ' '.join(classname_headers)]
cm_str_lines += ['{:>15}'.format(cn[:15]) + ' ' + cm_line for cn, cm_line in zip(classname_list, cm_str.splitlines())]
# Print formatted confusion matrix
print("Confusion matrix: ")
print(*cm_str_lines, sep='\n')
# Plot confusion matrix
# To manually add more space at bottom: plt.rcParams['figure.subplot.bottom'] = 0.1
#
# Add 0.5 to figsize for every class. For two classes, this will result in
# fig = plt.figure(figsize=[4,4])
fig = vis_utils.plot_confusion_matrix(
classifier_cm_array,
classname_list,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues,
vmax=1.0,
use_colorbar=True,
y_label=True)
cm_figure_relative_filename = 'confusion_matrix.png'
cm_figure_filename = os.path.join(output_dir, cm_figure_relative_filename)
plt.savefig(cm_figure_filename)
plt.close(fig)
# ...if we have classification results
##%% Render output
# Write p/r table to .csv file in output directory
pr_table_filename = os.path.join(output_dir, 'prec_recall.csv')
precisions_recalls.to_csv(pr_table_filename, index=False)
# Write precision/recall plot to .png file in output directory
t = 'Precision-Recall curve: AP={:0.1%}, P@{:0.1%}={:0.1%}'.format(
average_precision, target_recall, precision_at_target_recall)
fig = vis_utils.plot_precision_recall_curve(precisions, recalls, t)
pr_figure_relative_filename = 'prec_recall.png'
pr_figure_filename = os.path.join(output_dir, pr_figure_relative_filename)
plt.savefig(pr_figure_filename)
# plt.show(block=False)
plt.close(fig)
##%% Sampling
# Sample true/false positives/negatives with correct/incorrect top-1
# classification and render to html
# Accumulate html image structs (in the format expected by write_html_image_lists)
# for each category, e.g. 'tp', 'fp', ..., 'class_bird', ...
images_html = collections.defaultdict(lambda: [])
# Add default entries by accessing them for the first time
[images_html[res] for res in ['tp', 'tpc', 'tpi', 'fp', 'tn', 'fn']] # Siyu: what does this do? This line should have no effect
for res in images_html.keys():
os.makedirs(os.path.join(output_dir, res), exist_ok=True)
image_count = len(images_to_visualize)
# Each element will be a list of 2-tuples, with elements [collection name,html info struct]
rendering_results = []
# Each element will be a three-tuple with elements file,max_conf,detections
files_to_render = []
# Assemble the information we need for rendering, so we can parallelize without
# dealing with Pandas
# i_row = 0; row = images_to_visualize.iloc[0]
for _, row in images_to_visualize.iterrows():
# Filenames should already have been normalized to either '/' or '\'
files_to_render.append([row['file'],row['max_detection_conf'],row['detections']])
def render_image_with_gt(file_info):
image_relative_path = file_info[0]
max_conf = file_info[1]
detections = file_info[2]
# This should already have been normalized to either '/' or '\'
image_id = ground_truth_indexed_db.filename_to_id.get(image_relative_path, None)
if image_id is None:
print('Warning: couldn''t find ground truth for image {}'.format(image_relative_path))
return None
image = ground_truth_indexed_db.image_id_to_image[image_id]
annotations = ground_truth_indexed_db.image_id_to_annotations[image_id]
gt_status = image['_detection_status']
gt_presence = bool(gt_status)
gt_classes = CameraTrapJsonUtils.annotations_to_classnames(
annotations,ground_truth_indexed_db.cat_id_to_name)
gt_class_summary = ','.join(gt_classes)
if gt_status > DetectionStatus.DS_MAX_DEFINITIVE_VALUE:
print('Skipping image {}, does not have a definitive ground truth status (status: {}, classes: {})'.format(
image_id, gt_status, gt_class_summary))
return None
detected = max_conf > options.confidence_threshold
if gt_presence and detected:
if '_classification_accuracy' not in image.keys():
res = 'tp'
elif np.isclose(1, image['_classification_accuracy']):
res = 'tpc'
else:
res = 'tpi'
elif not gt_presence and detected:
res = 'fp'
elif gt_presence and not detected:
res = 'fn'
else:
res = 'tn'
display_name = '<b>Result type</b>: {}, <b>Presence</b>: {}, <b>Class</b>: {}, <b>Max conf</b>: {:0.3f}%, <b>Image</b>: {}'.format(
res.upper(), str(gt_presence), gt_class_summary,
max_conf * 100, image_relative_path)
rendered_image_html_info = render_bounding_boxes(options.image_base_dir,
image_relative_path,
display_name,
detections,
res,
detection_categories_map,
classification_categories_map,
options)
image_result = None
if len(rendered_image_html_info) > 0:
image_result = [[res,rendered_image_html_info]]
for gt_class in gt_classes:
image_result.append(['class_{}'.format(gt_class),rendered_image_html_info])
return image_result
# ...def render_image_with_gt(file_info)
start_time = time.time()
if options.parallelize_rendering:
if options.parallelize_rendering_n_cores is None:
pool = ThreadPool()
else:
print('Rendering images with {} workers'.format(options.parallelize_rendering_n_cores))
pool = ThreadPool(options.parallelize_rendering_n_cores)
rendering_results = list(tqdm(pool.imap(render_image_with_gt, files_to_render), total=len(files_to_render)))
else:
# file_info = files_to_render[0]
for file_info in tqdm(files_to_render):
rendering_results.append(render_image_with_gt(file_info))
elapsed = time.time() - start_time
# Map all the rendering results in the list rendering_results into the
# dictionary images_html
image_rendered_count = 0
for rendering_result in rendering_results:
if rendering_result is None:
continue
image_rendered_count += 1
for assignment in rendering_result:
images_html[assignment[0]].append(assignment[1])
# Prepare the individual html image files
image_counts = prepare_html_subpages(images_html, output_dir)
print('{} images rendered (of {})'.format(image_rendered_count,image_count))
# Write index.html
all_tp_count = image_counts['tp'] + image_counts['tpc'] + image_counts['tpi']
total_count = all_tp_count + image_counts['tn'] + image_counts['fp'] + image_counts['fn']
classification_detection_results = """ <a href="tpc.html">with all correct top-1 predictions (TPC)</a> ({})<br/>
<a href="tpi.html">with one or more incorrect top-1 prediction (TPI)</a> ({})<br/>
<a href="tp.html">without classification evaluation</a><sup>*</sup> ({})<br/>""".format(
image_counts['tpc'],
image_counts['tpi'],
image_counts['tp']
)
index_page = """<html>
{}
<body>
<h2>Evaluation</h2>
<h3>Sample images</h3>
<div style="margin-left:20px;">
<p>A sample of {} images, annotated with detections above {:.1%} confidence.</p>
<a href="tp.html">True positives (TP)</a> ({}) ({:0.1%})<br/>
CLASSIFICATION_PLACEHOLDER_1
<a href="tn.html">True negatives (TN)</a> ({}) ({:0.1%})<br/>
<a href="fp.html">False positives (FP)</a> ({}) ({:0.1%})<br/>
<a href="fn.html">False negatives (FN)</a> ({}) ({:0.1%})<br/>
CLASSIFICATION_PLACEHOLDER_2
</div>
""".format(
style_header,
image_count, options.confidence_threshold,
all_tp_count, all_tp_count/total_count,
image_counts['tn'], image_counts['tn']/total_count,
image_counts['fp'], image_counts['fp']/total_count,
image_counts['fn'], image_counts['fn']/total_count
)
index_page += """
<h3>Detection results</h3>
<div class="contentdiv">
<p>At a confidence threshold of {:0.1%}, precision={:0.1%}, recall={:0.1%}</p>
<p><strong>Precision/recall summary for all {} images</strong></p><img src="{}"><br/>
</div>
""".format(
options.confidence_threshold, precision_at_confidence_threshold, recall_at_confidence_threshold,
len(detection_results), pr_figure_relative_filename
)
if len(classifier_accuracies) > 0:
index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_1',classification_detection_results)
index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_2',"""<p><sup>*</sup>We do not evaluate the classification result of images
if the classification information is missing, if the image contains
categories like ‘empty’ or ‘human’, or if the image has multiple
classification labels.</p>""")
else:
index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_1','')
index_page = index_page.replace('CLASSIFICATION_PLACEHOLDER_2','')
if len(classifier_accuracies) > 0:
index_page += """
<h3>Classification results</h3>
<div class="contentdiv">
<p>Classification accuracy: {:.2%}<br>
The accuracy is computed only for images with exactly one classification label.
The accuracy of an image is computed as 1/(number of unique detected top-1 classes),
i.e. if the model detects multiple boxes with different top-1 classes, then the accuracy
decreases and the image is put into 'TPI'.</p>
<p>Confusion matrix:</p>
<p><img src="{}"></p>
<div style='font-family:monospace;display:block;'>{}</div>
</div>
""".format(
np.mean(classifier_accuracies),
cm_figure_relative_filename,
"<br>".join(cm_str_lines).replace(' ', ' ')
)
# Show links to each GT class
#
# We could do this without classification results; currently we don't.
if len(classname_to_idx) > 0:
index_page += '<h3>Images of specific classes</h3><br/><div class="contentdiv">'
# Add links to all available classes
for cname in sorted(classname_to_idx.keys()):
index_page += "<a href='class_{0}.html'>{0}</a> ({1})<br>".format(
cname,
len(images_html['class_{}'.format(cname)]))
index_page += "</div>"
# Close body and html tags
index_page += "</body></html>"
output_html_file = os.path.join(output_dir, 'index.html')
with open(output_html_file, 'w') as f:
f.write(index_page)
print('Finished writing html to {}'.format(output_html_file))
# ...for each image
##%% Otherwise, if we don't have ground truth...
else:
##%% Sample detections/non-detections
# Accumulate html image structs (in the format expected by write_html_image_list)
# for each category
images_html = collections.defaultdict(lambda: [])
# Add default entries by accessing them for the first time
[images_html[res] for res in ['detections', 'non_detections']]
if options.include_almost_detections:
images_html['almost_detections']
# Create output directories
for res in images_html.keys():
os.makedirs(os.path.join(output_dir, res), exist_ok=True)
image_count = len(images_to_visualize)
has_classification_info = False
# Each element will be a list of 2-tuples, with elements [collection name,html info struct]
rendering_results = []
# Each element will be a three-tuple with elements [file,max_conf,detections]
files_to_render = []
# Assemble the information we need for rendering, so we can parallelize without
# dealing with Pandas
# i_row = 0; row = images_to_visualize.iloc[0]
for _, row in images_to_visualize.iterrows():
# Filenames should already have been normalized to either '/' or '\'
files_to_render.append([row['file'],
row['max_detection_conf'],
row['detections']])
# Local function for parallelization
def render_image_no_gt(file_info):
image_relative_path = file_info[0]
max_conf = file_info[1]
detections = file_info[2]
detection_status = DetectionStatus.DS_UNASSIGNED
if max_conf >= options.confidence_threshold:
detection_status = DetectionStatus.DS_POSITIVE
else:
if options.include_almost_detections:
if max_conf >= options.almost_detection_confidence_threshold:
detection_status = DetectionStatus.DS_ALMOST
else:
detection_status = DetectionStatus.DS_NEGATIVE
else:
detection_status = DetectionStatus.DS_NEGATIVE
if detection_status == DetectionStatus.DS_POSITIVE:
res = 'detections'
elif detection_status == DetectionStatus.DS_NEGATIVE:
res = 'non_detections'
else:
assert detection_status == DetectionStatus.DS_ALMOST
res = 'almost_detections'
display_name = '<b>Result type</b>: {}, <b>Image</b>: {}, <b>Max conf</b>: {:0.3f}'.format(
res, image_relative_path, max_conf)
rendering_options = copy.copy(options)
if detection_status == DetectionStatus.DS_ALMOST:
rendering_options.confidence_threshold = rendering_options.almost_detection_confidence_threshold
rendered_image_html_info = render_bounding_boxes(options.image_base_dir,
image_relative_path,
display_name,
detections,
res,
detection_categories_map,
classification_categories_map,
rendering_options)
image_result = None
if len(rendered_image_html_info) > 0:
image_result = [[res,rendered_image_html_info]]
for det in detections:
if 'classifications' in det:
top1_class = classification_categories_map[det['classifications'][0][0]]
image_result.append(['class_{}'.format(top1_class),rendered_image_html_info])
return image_result
# ...def render_image_no_gt(file_info):
start_time = time.time()
if options.parallelize_rendering:
if options.parallelize_rendering_n_cores is None:
pool = ThreadPool()
else:
print('Rendering images with {} workers'.format(options.parallelize_rendering_n_cores))
pool = ThreadPool(options.parallelize_rendering_n_cores)
rendering_results = list(tqdm(pool.imap(render_image_no_gt, files_to_render), total=len(files_to_render)))
else:
for file_info in tqdm(files_to_render):
rendering_results.append(render_image_no_gt(file_info))
elapsed = time.time() - start_time
# Map all the rendering results in the list rendering_results into the
# dictionary images_html
image_rendered_count = 0
for rendering_result in rendering_results:
if rendering_result is None:
continue
image_rendered_count += 1
for assignment in rendering_result:
if 'class' in assignment[0]:
has_classification_info = True
images_html[assignment[0]].append(assignment[1])
# Prepare the individual html image files
image_counts = prepare_html_subpages(images_html, output_dir)
if image_rendered_count == 0:
seconds_per_image = 0
else:
seconds_per_image = elapsed/image_rendered_count
print('Rendered {} images (of {}) in {} ({} per image)'.format(image_rendered_count,
image_count,humanfriendly.format_timespan(elapsed),
humanfriendly.format_timespan(seconds_per_image)))
# Write index.HTML
total_images = image_counts['detections'] + image_counts['non_detections']
if options.include_almost_detections:
total_images += image_counts['almost_detections']
assert total_images == image_count, \
'Error: image_count is {}, total_images is {}'.format(image_count,total_images)
almost_detection_string = ''
if options.include_almost_detections:
almost_detection_string = ' (“almost detection” threshold at {:.1%})'.format(options.almost_detection_confidence_threshold)
index_page = """<html>{}<body>
<h2>Visualization of results</h2>
<p>A sample of {} images, annotated with detections above {:.1%} confidence{}.</p>
<h3>Sample images</h3>
<div class="contentdiv">
<a href="detections.html">detections</a> ({}, {:.1%})<br/>
<a href="non_detections.html">non-detections</a> ({}, {:.1%})<br/>""".format(
style_header,image_count, options.confidence_threshold, almost_detection_string,
image_counts['detections'], image_counts['detections']/total_images,
image_counts['non_detections'], image_counts['non_detections']/total_images
)
if options.include_almost_detections:
index_page += """<a href="almost_detections.html">almost-detections</a> ({}, {:.1%})<br/>""".format(
image_counts['almost_detections'], image_counts['almost_detections']/total_images)
index_page += '</div>\n'
if has_classification_info:
index_page += "<h3>Images of detected classes</h3>"
index_page += "<p>The same image might appear under multiple classes if multiple species were detected.</p>\n<div class='contentdiv'>\n"
# Add links to all available classes
for cname in sorted(classification_categories_map.values()):
ccount = len(images_html['class_{}'.format(cname)])
if ccount > 0:
index_page += "<a href='class_{}.html'>{}</a> ({})<br/>\n".format(cname, cname.lower(), ccount)
index_page += "</div>\n"
index_page += "</body></html>"
output_html_file = os.path.join(output_dir, 'index.html')
with open(output_html_file, 'w') as f:
f.write(index_page)
print('Finished writing html to {}'.format(output_html_file))
# os.startfile(output_html_file)
# ...if we do/don't have ground truth
ppresults.output_html_file = output_html_file
return ppresults
def process_batch_results(options):
##%% Expand some options for convenience
output_dir = options.output_dir
confidence_threshold = options.confidence_threshold
##%% Prepare output dir
os.makedirs(output_dir, exist_ok=True)
##%% Load ground truth if available
ground_truth_indexed_db = None
if options.ground_truth_json_file and len(
options.ground_truth_json_file) > 0:
ground_truth_indexed_db = IndexedJsonDb(
options.ground_truth_json_file,
b_normalize_paths=True,
filename_replacements=options.ground_truth_filename_replacements)
# Mark images in the ground truth as positive or negative
n_negative, n_positive, n_unknown, n_ambiguous = mark_detection_status(
ground_truth_indexed_db,
negative_classes=options.negative_classes,
unknown_classes=options.unlabeled_classes)
print(
'Finished loading and indexing ground truth: {} negative, {} positive, {} unknown, {} ambiguous'
.format(n_negative, n_positive, n_unknown, n_ambiguous))
##%% Load detection results
detection_results, other_fields = load_api_results(
options.api_output_file,
normalize_paths=True,
filename_replacements=options.api_output_filename_replacements)
detection_categories_map = other_fields['detection_categories']
if 'classification_categories' in other_fields:
classification_categories_map = other_fields[
'classification_categories']
else:
classification_categories_map = {}
# Add a column (pred_detection_label) to indicate predicted detection status, not separating out the classes
detection_results['pred_detection_label'] = \
np.where(detection_results['max_detection_conf'] >= options.confidence_threshold,
DetectionStatus.DS_POSITIVE, DetectionStatus.DS_NEGATIVE)
n_positives = sum(detection_results['pred_detection_label'] ==
DetectionStatus.DS_POSITIVE)
print(
'Finished loading and preprocessing {} rows from detector output, predicted {} positives'
.format(len(detection_results), n_positives))
##%% If we have ground truth, remove images we can't match to ground truth
# ground_truth_indexed_db.db['images'][0]
if ground_truth_indexed_db is not None:
b_match = [False] * len(detection_results)
detector_files = detection_results['file'].tolist()
for i_fn, fn in enumerate(detector_files):
# assert fn in ground_truth_indexed_db.filename_to_id, 'Could not find ground truth for row {} ({})'.format(i_fn,fn)
if fn in fn in ground_truth_indexed_db.filename_to_id:
b_match[i_fn] = True
print('Confirmed filename matches to ground truth for {} of {} files'.
format(sum(b_match), len(detector_files)))
detection_results = detection_results[b_match]
detector_files = detection_results['file'].tolist()
print('Trimmed detection results to {} files'.format(
len(detector_files)))
##%% Sample images for visualization
images_to_visualize = detection_results
if options.num_images_to_sample > 0 and options.num_images_to_sample <= len(
detection_results):
images_to_visualize = images_to_visualize.sample(
options.num_images_to_sample, random_state=options.sample_seed)
##%% Fork here depending on whether or not ground truth is available
output_html_file = ''
# If we have ground truth, we'll compute precision/recall and sample tp/fp/tn/fn.
#
# Otherwise we'll just visualize detections/non-detections.
if ground_truth_indexed_db is not None:
##%% DETECTION EVALUATION: Compute precision/recall
# numpy array of detection probabilities
p_detection = detection_results['max_detection_conf'].values
n_detections = len(p_detection)
# numpy array of bools (0.0/1.0), and -1 as null value
gt_detections = np.zeros(n_detections, dtype=float)
for i_detection, fn in enumerate(detector_files):
image_id = ground_truth_indexed_db.filename_to_id[fn]
image = ground_truth_indexed_db.image_id_to_image[image_id]
detection_status = image['_detection_status']
if detection_status == DetectionStatus.DS_NEGATIVE:
gt_detections[i_detection] = 0.0
elif detection_status == DetectionStatus.DS_POSITIVE:
gt_detections[i_detection] = 1.0
else:
gt_detections[i_detection] = -1.0
# Don't include ambiguous/unknown ground truth in precision/recall analysis
b_valid_ground_truth = gt_detections >= 0.0
p_detection_pr = p_detection[b_valid_ground_truth]
gt_detections_pr = gt_detections[b_valid_ground_truth]
print('Including {} of {} values in p/r analysis'.format(
np.sum(b_valid_ground_truth), len(b_valid_ground_truth)))
precisions, recalls, thresholds = precision_recall_curve(
gt_detections_pr, p_detection_pr)
# For completeness, include the result at a confidence threshold of 1.0
thresholds = np.append(thresholds, [1.0])
precisions_recalls = pd.DataFrame(
data={
'confidence_threshold': thresholds,
'precision': precisions,
'recall': recalls
})
# Compute and print summary statistics
average_precision = average_precision_score(gt_detections_pr,
p_detection_pr)
print('Average precision: {:.1%}'.format(average_precision))
# Thresholds go up throughout precisions/recalls/thresholds; find the last
# value where recall is at or above target. That's our precision @ target recall.
target_recall = 0.9
b_above_target_recall = np.where(recalls >= target_recall)
if not np.any(b_above_target_recall):
precision_at_target_recall = 0.0
else:
i_target_recall = np.argmax(b_above_target_recall)
precision_at_target_recall = precisions[i_target_recall]
print('Precision at {:.1%} recall: {:.1%}'.format(
target_recall, precision_at_target_recall))
cm = confusion_matrix(gt_detections_pr,
np.array(p_detection_pr) > confidence_threshold)
# Flatten the confusion matrix
tn, fp, fn, tp = cm.ravel()
precision_at_confidence_threshold = tp / (tp + fp)
recall_at_confidence_threshold = tp / (tp + fn)
f1 = 2.0 * (precision_at_confidence_threshold * recall_at_confidence_threshold) / \
(precision_at_confidence_threshold + recall_at_confidence_threshold)
print(
'At a confidence threshold of {:.1%}, precision={:.1%}, recall={:.1%}, f1={:.1%}'
.format(confidence_threshold, precision_at_confidence_threshold,
recall_at_confidence_threshold, f1))
##%% CLASSIFICATION evaluation
classifier_accuracies = []
# Mapping of classnames to idx for the confusion matrix.
# The lambda is actually kind of a hack, because we use assume that
# the following code does not reassign classname_to_idx
classname_to_idx = collections.defaultdict(
lambda: len(classname_to_idx))
# Confusion matrix as defaultdict of defaultdict
# Rows / first index is ground truth, columns / second index is predicted category
classifier_cm = collections.defaultdict(
lambda: collections.defaultdict(lambda: 0))
for iDetection, fn in enumerate(detector_files):
image_id = ground_truth_indexed_db.filename_to_id[fn]
image = ground_truth_indexed_db.image_id_to_image[image_id]
pred_class_ids = [det['classifications'][0][0] \
for det in detection_results['detections'][iDetection] if 'classifications' in det.keys()]
pred_classnames = [
classification_categories_map[pd] for pd in pred_class_ids
]
# If this image has classification predictions, and an unambiguous class
# annotated, and is a positive image
if len(pred_classnames) > 0 \
and '_unambiguous_category' in image.keys() \
and image['_detection_status'] == DetectionStatus.DS_POSITIVE:
# The unambiguous category, we make this a set for easier handling afterward
# TODO: actually we can replace the unambiguous category by all annotated
# categories. However, then the confusion matrix doesn't make sense anymore
# TODO: make sure we are using the class names as strings in both, not IDs
gt_categories = set([image['_unambiguous_category']])
pred_categories = set(pred_classnames)
# Compute the accuracy as intersection of union,
# i.e. (# of categories in both prediciton and GT)
# divided by (# of categories in either prediction or GT
# In case of only one GT category, the result will be 1.0, if
# prediction is one category and this category matches GT
# It is 1.0/(# of predicted top-1 categories), if the GT is
# one of the predicted top-1 categories.
# It is 0.0, if none of the predicted categories is correct
classifier_accuracies.append(
len(gt_categories & pred_categories) /
len(gt_categories | pred_categories))
image['_classification_accuracy'] = classifier_accuracies[-1]
# Distribute this accuracy across all predicted categories in the
# confusion matrix
assert len(gt_categories) == 1
gt_class_idx = classname_to_idx[list(gt_categories)[0]]
for pred_category in pred_categories:
pred_class_idx = classname_to_idx[pred_category]
classifier_cm[gt_class_idx][pred_class_idx] += 1
# If we have classification results
if len(classifier_accuracies) > 0:
# Build confusion matrix as array from classifier_cm
all_class_ids = sorted(classname_to_idx.values())
classifier_cm_array = np.array(
[[classifier_cm[r_idx][c_idx] for c_idx in all_class_ids]
for r_idx in all_class_ids],
dtype=float)
classifier_cm_array /= (
classifier_cm_array.sum(axis=1, keepdims=True) + 1e-7)
# Print some statistics
print("Finished computation of {} classification results".format(
len(classifier_accuracies)))
print("Mean accuracy: {}".format(np.mean(classifier_accuracies)))
# Prepare confusion matrix output
# Get CM matrix as string
sio = io.StringIO()
np.savetxt(sio, classifier_cm_array * 100, fmt='%5.1f')
cm_str = sio.getvalue()
# Get fixed-size classname for each idx
idx_to_classname = {v: k for k, v in classname_to_idx.items()}
classname_list = [
idx_to_classname[idx]
for idx in sorted(classname_to_idx.values())
]
classname_headers = [
'{:<5}'.format(cname[:5]) for cname in classname_list
]
# Prepend class name on each line and add to the top
cm_str_lines = [' ' * 16 + ' '.join(classname_headers)]
cm_str_lines += [
'{:>15}'.format(cn[:15]) + ' ' + cm_line
for cn, cm_line in zip(classname_list, cm_str.splitlines())
]
# print formatted confusion matrix
print("Confusion matrix: ")
print(*cm_str_lines, sep='\n')
# Plot confusion matrix
# To manually add more space at bottom: plt.rcParams['figure.subplot.bottom'] = 0.1
# Add 0.5 to figsize for every class. For two classes, this will result in
# fig = plt.figure(figsize=[4,4])
fig = vis_utils.plot_confusion_matrix(classifier_cm_array,
classname_list,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues,
vmax=1.0,
use_colorbar=True,
y_label=True)
cm_figure_relative_filename = 'confusion_matrix.png'
cm_figure_filename = os.path.join(output_dir,
cm_figure_relative_filename)
plt.savefig(cm_figure_filename)
plt.close(fig)
##%% Render output
# Write p/r table to .csv file in output directory
pr_table_filename = os.path.join(output_dir, 'prec_recall.csv')
precisions_recalls.to_csv(pr_table_filename, index=False)
# Write precision/recall plot to .png file in output directory
t = 'Precision-Recall curve: AP={:0.1%}, P@{:0.1%}={:0.1%}'.format(
average_precision, target_recall, precision_at_target_recall)
fig = vis_utils.plot_precision_recall_curve(precisions, recalls, t)
pr_figure_relative_filename = 'prec_recall.png'
pr_figure_filename = os.path.join(output_dir,
pr_figure_relative_filename)
plt.savefig(pr_figure_filename)
# plt.show(block=False)
plt.close(fig)
##%% Sample true/false positives/negatives with correct/incorrect top-1
# classification and render to html
# Accumulate html image structs (in the format expected by write_html_image_lists)
# for each category, e.g. 'tp', 'fp', ..., 'class_bird', ...
images_html = collections.defaultdict(lambda: [])
# Add default entries by accessing them for the first time
[images_html[res] for res in ['tp', 'tpc', 'tpi', 'fp', 'tn', 'fn']]
for res in images_html.keys():
os.makedirs(os.path.join(output_dir, res), exist_ok=True)
count = 0
# i_row = 0; row = images_to_visualize.iloc[0]
for i_row, row in tqdm(images_to_visualize.iterrows(),
total=len(images_to_visualize)):
image_relative_path = row['file']
# This should already have been normalized to either '/' or '\'
image_id = ground_truth_indexed_db.filename_to_id.get(
image_relative_path, None)
if image_id is None:
print('Warning: couldn'
't find ground truth for image {}'.format(
image_relative_path))
continue
image = ground_truth_indexed_db.image_id_to_image[image_id]
annotations = ground_truth_indexed_db.image_id_to_annotations[
image_id]
gt_status = image['_detection_status']
if gt_status > DetectionStatus.DS_MAX_DEFINITIVE_VALUE:
print(
'Skipping image {}, does not have a definitive ground truth status'
.format(i_row, gt_status))
continue
gt_presence = bool(gt_status)
gt_classes = CameraTrapJsonUtils.annotations_to_classnames(
annotations, ground_truth_indexed_db.cat_id_to_name)
gt_class_summary = ','.join(gt_classes)
max_conf = row['max_detection_conf']
detections = row['detections']
detected = max_conf > confidence_threshold
if gt_presence and detected:
if '_classification_accuracy' not in image.keys():
res = 'tp'
elif np.isclose(1, image['_classification_accuracy']):
res = 'tpc'
else:
res = 'tpi'
elif not gt_presence and detected:
res = 'fp'
elif gt_presence and not detected:
res = 'fn'
else:
res = 'tn'
display_name = '<b>Result type</b>: {}, <b>Presence</b>: {}, <b>Class</b>: {}, <b>Max conf</b>: {:0.2f}%, <b>Image</b>: {}'.format(
res.upper(), str(gt_presence), gt_class_summary,
max_conf * 100, image_relative_path)
rendered_image_html_info = render_bounding_boxes(
options.image_base_dir, image_relative_path, display_name,
detections, res, detection_categories_map,
classification_categories_map, options)
if len(rendered_image_html_info) > 0:
images_html[res].append(rendered_image_html_info)
for gt_class in gt_classes:
images_html['class_{}'.format(gt_class)].append(
rendered_image_html_info)
count += 1
# ...for each image in our sample
print('{} images rendered'.format(count))
# Prepare the individual html image files
image_counts = prepare_html_subpages(images_html, output_dir)
# Write index.HTML
all_tp_count = image_counts['tp'] + image_counts['tpc'] + image_counts[
'tpi']
total_count = all_tp_count + image_counts['tn'] + image_counts[
'fp'] + image_counts['fn']
index_page = """<html><body>
<h2>Evaluation</h2>
<h3>Sample images</h3>
<p>A sample of {} images, annotated with detections above {:.1%} confidence.</p>
True positives (TP) ({} or {:0.1%})<br/>
-- <a href="tpc.html">with all correct top-1 predictions (TPC)</a> ({})<br/>
-- <a href="tpi.html">with one or more incorrect top-1 prediction (TPI)</a> ({})<br/>
-- <a href="tp.html">without classification evaluation</a> (*) ({})<br/>
<a href="tn.html">True negatives (TN)</a> ({} or {:0.1%})<br/>
<a href="fp.html">False positives (FP)</a> ({} or {:0.1%})<br/>
<a href="fn.html">False negatives (FN)</a> ({} or {:0.1%})<br/>
<p>(*) We do not evaluate the classification result of images, if the classification
information is missing, if the image contains
categories like 'empty' or 'human', or if the image has multiple classification
labels.</p>""".format(
count, confidence_threshold, all_tp_count,
all_tp_count / total_count, image_counts['tpc'],
image_counts['tpi'], image_counts['tp'], image_counts['tn'],
image_counts['tn'] / total_count, image_counts['fp'],
image_counts['fp'] / total_count, image_counts['fn'],
image_counts['fn'] / total_count)
index_page += """
<h3>Detection results</h3>
<p>At a confidence threshold of {:0.1%}, precision={:0.1%}, recall={:0.1%}</p>
<p><strong>Precision/recall summary for all {} images</strong></p><img src="{}"><br/>
""".format(confidence_threshold, precision_at_confidence_threshold,
recall_at_confidence_threshold, len(detection_results),
pr_figure_relative_filename)
if len(classifier_accuracies) > 0:
index_page += """
<h3>Classification results</h3>
<p>Classification accuracy: {:.2%}<br>
The accuracy is computed only for images with exactly one classification label.
The accuracy of an image is computed as 1/(number of unique detected top-1 classes),
i.e. if the model detects multiple boxes with different top-1 classes, then the accuracy
decreases and the image is put into 'TPI'.</p>
<p>Confusion matrix:</p>
<p><img src="{}"></p>
<div style='font-family:monospace;display:block;'>{}</div>
""".format(np.mean(classifier_accuracies),
cm_figure_relative_filename,
"<br>".join(cm_str_lines).replace(' ', ' '))
# Show links to each GT class
index_page += "<h3>Images of specific classes:</h3>"
# Add links to all available classes
for cname in sorted(classname_to_idx.keys()):
index_page += "<a href='class_{0}.html'>{0}</a> ({1})<br>".format(
cname, len(images_html['class_{}'.format(cname)]))
# Close body and html tag
index_page += "</body></html>"
output_html_file = os.path.join(output_dir, 'index.html')
with open(output_html_file, 'w') as f:
f.write(index_page)
print('Finished writing html to {}'.format(output_html_file))
##%% Otherwise, if we don't have ground truth...
else:
##%% Sample detections/non-detections
os.makedirs(os.path.join(output_dir, 'detections'), exist_ok=True)
os.makedirs(os.path.join(output_dir, 'non_detections'), exist_ok=True)
# Accumulate html image structs (in the format expected by write_html_image_lists)
# for each category
images_html = collections.defaultdict(lambda: [])
# Add default entries by accessing them for the first time
[images_html[res] for res in ['detections', 'non_detections']]
for res in images_html.keys():
os.makedirs(os.path.join(output_dir, res), exist_ok=True)
count = 0
has_classification_info = False
# i_row = 0; row = images_to_visualize.iloc[0]
for i_row, row in tqdm(images_to_visualize.iterrows(),
total=len(images_to_visualize)):
image_relative_path = row['file']
# This should already have been normalized to either '/' or '\'
max_conf = row['max_detection_conf']
detections = row['detections']
detected = True if max_conf > confidence_threshold else False
if detected:
res = 'detections'
else:
res = 'non_detections'
display_name = '<b>Result type</b>: {}, <b>Image</b>: {}, <b>Max conf</b>: {}'.format(
res, image_relative_path, max_conf)
rendered_image_html_info = render_bounding_boxes(
options.image_base_dir, image_relative_path, display_name,
detections, res, detection_categories_map,
classification_categories_map, options)
if len(rendered_image_html_info) > 0:
images_html[res].append(rendered_image_html_info)
for det in detections:
if 'classifications' in det:
has_classification_info = True
top1_class = classification_categories_map[
det['classifications'][0][0]]
images_html['class_{}'.format(top1_class)].append(
rendered_image_html_info)
count += 1
# ...for each image in our sample
print('{} images rendered'.format(count))
# Prepare the individual html image files
image_counts = prepare_html_subpages(images_html, output_dir)
# Write index.HTML
total_images = image_counts['detections'] + image_counts[
'non_detections']
index_page = """<html><body>
<h2>Visualization of results</h2>
<p>A sample of {} images, annotated with detections above {:.1%} confidence.</p>
<h3>Sample images</h3>
<a href="detections.html">Detections</a> ({}, {:.1%})<br/>
<a href="non_detections.html">Non-detections</a> ({}, {:.1%})<br/>""".format(
count, confidence_threshold, image_counts['detections'],
image_counts['detections'] / total_images,
image_counts['non_detections'],
image_counts['non_detections'] / total_images)
if has_classification_info:
index_page += "<h3>Images of detected classes</h3>"
index_page += "<p>The same image might appear under multiple classes if multiple species were detected.</p>"
# Add links to all available classes
for cname in sorted(classification_categories_map.values()):
ccount = len(images_html['class_{}'.format(cname)])
if ccount > 0:
index_page += "<a href='class_{0}.html'>{0}</a> ({1})<br>".format(
cname, ccount)
index_page += "</body></html>"
output_html_file = os.path.join(output_dir, 'index.html')
with open(output_html_file, 'w') as f:
f.write(index_page)
print('Finished writing html to {}'.format(output_html_file))
# ...if we do/don't have ground truth
ppresults = PostProcessingResults()
ppresults.output_html_file = output_html_file
return ppresults