def run_evaluation(labelmap, groundtruth, detections, exclusions): """Runs evaluations given input files. Args: labelmap: file object containing map of labels to consider, in pbtxt format groundtruth: file object detections: file object exclusions: file object or None. """ categories, class_whitelist = read_labelmap(labelmap) logging.info("CATEGORIES (%d):\n%s", len(categories), pprint.pformat(categories, indent=2)) excluded_keys = read_exclusions(exclusions) pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator( categories) # Reads the ground truth data. boxes, labels, _ = read_csv(groundtruth, class_whitelist, 0) start = time.time() for image_key in boxes: if image_key in excluded_keys: logging.info(("Found excluded timestamp in ground truth: %s. " "It will be ignored."), image_key) continue pascal_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: np.array(boxes[image_key], dtype=float), standard_fields.InputDataFields.groundtruth_classes: np.array(labels[image_key], dtype=int), standard_fields.InputDataFields.groundtruth_difficult: np.zeros(len(boxes[image_key]), dtype=bool) }) print_time("convert groundtruth", start) # Reads detections data. boxes, labels, scores = read_csv(detections, class_whitelist, 50) start = time.time() for image_key in boxes: if image_key in excluded_keys: logging.info(("Found excluded timestamp in detections: %s. " "It will be ignored."), image_key) continue pascal_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: np.array(boxes[image_key], dtype=float), standard_fields.DetectionResultFields.detection_classes: np.array(labels[image_key], dtype=int), standard_fields.DetectionResultFields.detection_scores: np.array(scores[image_key], dtype=float) }) print_time("convert detections", start) start = time.time() metrics = pascal_evaluator.evaluate() print_time("run_evaluator", start) pprint.pprint(metrics, indent=2)
def __init__(self): this_dir = os.path.dirname(__file__) lib_path = os.path.join(this_dir, '../external/ActivityNet/Evaluation') sys.path.insert(0, lib_path) from ava import object_detection_evaluation from ava import standard_fields sys.path.pop(0) self.sf = standard_fields categories = [{"id": i, "name": i} for i in range(1, 81)] self.evaluator = object_detection_evaluation.PascalDetectionEvaluator( categories)
def __init__(self): this_dir = os.path.dirname(__file__) lib_path = os.path.join(this_dir, '../external/ActivityNet/Evaluation') sys.path.insert(0, lib_path) from ava import object_detection_evaluation from ava import standard_fields sys.path.pop(0) self.sf = standard_fields categories = [{"id": i, "name": i} for i in range(1, 81)] self.evaluator = object_detection_evaluation.PascalDetectionEvaluator( categories) top60path = '../external/ActivityNet/Evaluation/ava/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt' top60path = os.path.join(this_dir, top60path) with open(top60path) as f: self.top60 = [int(x) for x in re.findall('[0-9]+', f.read())]
def run_evaluation(labelmap, groundtruth, exclusions, iou): root_dir = '../../../data/AVA/files/' test_dir = "../test_outputs/" # Make sure not to mess this up experiments_filters = {} experiments_detections = {} experiments_filters['pose'] = ['Pose'] experiments_detections['pose'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb') ] experiments_filters['rgb-streams-aug'] = ['RGB', 'Crop', 'Gauss', 'Fovea'] experiments_detections['rgb-streams-aug'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] experiments_filters['flow vs flowcrop'] = ['Flow', 'Flowcrop'] experiments_detections['flow vs flowcrop'] = [ open(test_dir + "/flow/output_test_flowcrop.csv", 'rb'), ] #all_detections.append(open(test_dir + "/flow/output_test_flow.csv", 'rb')) experiments_filters['two-streams'] = [ 'Two-Stream-RGB', 'Two-Stream-Crop', 'Two-Stream-Gauss', 'Two-Stream-Fovea' ] experiments_detections['two-streams'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] experiments_filters['two-streams-aug'] = ['RGB', 'Crop', 'Gauss', 'Fovea'] experiments_detections['two-streams-aug'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] experiments_filters['mlp vs lstm'] = ['RGB', 'Crop', 'Gauss', 'Fovea'] experiments_detections['mlp vs lstm'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] experiments_filters['lstmA vs lstmB'] = ['RGB', 'Crop', 'Gauss', 'Fovea'] experiments_detections['lstmA vs lstmB'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] experiments_filters['context-fusion mlp vs lstm'] = [ 'RGB', 'Crop', 'Gauss', 'Fovea' ] experiments_detections['context-fusion mlp vs lstm'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] experiments_filters['balancing sampling'] = [ 'RGB', 'Crop', 'Gauss', 'Fovea' ] experiments_detections['balancing sampling'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] experiments_filters['balancing weights'] = [ 'RGB', 'Crop', 'Gauss', 'Fovea' ] experiments_detections['balancing weights'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] experiments_filters['balancing prior'] = ['RGB', 'Crop', 'Gauss', 'Fovea'] experiments_detections['balancing prior'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] # experiment = filters = [] # filters.append("pose") # filters.append("rgb-base") # filters.append("rgb-prior") # filters.append("rgb-sampling") # filters.append("rgb-weights") # filters.append("rgb-kinetics") # filters.append("flow-kinetics") # filters.append("rgb") # filters.append("crop") # filters.append("gauss") # filters.append("fovea") # filters.append("flowcrop") # filters.append("flow") # filters.append("MLP") #filters.append("best case scenario thresh 0.1") #filters.append("two pass scenario thresh 0.1") filters.append("fovea") filters.append("dense-gt") #filters.append("sampling no aug") filters.append("dense-2pass") #filters.append("weights no aug") # filters.append("LSTM5-A-512") # filters.append("random") # filters.append("LSTM5-B-512") # filters.append("LSTM10") # filters.append("2st(rgb)") # filters.append("2st(crop)") # filters.append("2st(gauss)") # filters.append("2st(fovea)") #filters.append("2st(crop) + flowcrop") #filters.append("2st(gauss) + flowcrop") #filters.append("2st(fovea) + flowcrop") #filters.append("2st(fovea) + mlp") #filters.append("2st(crop) + mlp") #filters.append("2st(gauss) + mlp") # filters.append("2stream") #filters.append("2stream + lstm (extra pass)") # filters.append("gauss") #filters.append("gauss aug") #filters.append("LSTMA 512 5 2") #filters.append("LSTMA 512 5 3") #filters.append("LSTMA 512 5 3") #filters.append("LSTMA 1024 5 3") #filters.append("LSTMA 2048 5 3") #filters.append("LSTMB 512 3 3") #filters.append("LSTMB 1024 3 3") #filters.append("LSTMB 2048 3 3") # filters.append("2st(gauss) + lstm") all_detections = [] #all_detections.append(open(test_dir + "context/lstmA/output_test_ctx_lstm_512_5_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmA/output_test_ctx_lstm_1024_5_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmA/output_test_ctx_lstm_2048_5_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_512_3_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_1024_3_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_2048_3_3.csv", 'rb')) # Pose # all_detections.append(open(test_dir + "output_test_flowcrop.csv", 'rb')) # Balancing #all_detections.append(open(test_dir + "output_test_flowcrop.csv", 'rb')) #all_detections.append(open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb')) #all_detections.append(open(test_dir + "/augmentation/predictions_rgb_gauss_1807241628_1000.csv", 'rb')) #all_detections.append(open(test_dir + "/augmentation/output_test_sampling_gauss_1809221859.csv", 'rb')) #all_detections.append(open(test_dir + "/augmentation/output_test_weights_gauss_1809221904.csv", 'rb')) # RGB Streams #all_detections.append(open(test_dir + "/kinetics_init/output_test_rgb_kineticsinit_gauss_1809220212.csv", 'rb')) #all_detections.append(open(test_dir + "/kinetics_init/output_test_flow_kineticsinit_1809220244.csv", 'rb')) # Flow Streams # Context (LSTMs) #filters.append("LSTMB 512 3 3") #filters.append("LSTMB 512 3 2") #filters.append("LSTMB 512 3 1") #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_512_3_1.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_512_3_2.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_512_3_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmA/output_test_ctx_lstm_512_5_1.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmA/output_test_ctx_lstm_512_5_2.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmA/output_test_ctx_lstm_512_5_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_32_3_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_32_5_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_32_10_3.csv", 'rb')) #all_detections.append(open(test_dir + "context/mlp/output_test_ctx.csv", 'rb')) #all_detections.append(open(test_dir + "context/mlp/output_test_ctx_mlp_1809212356.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmA/output_test_ctx_lstm_512_5_3_1809220010.csv", 'rb')) #all_detections.append(open(test_dir + "random/output_test_random_1809221552.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_512_5_3_1809211924.csv", 'rb')) #all_detections.append(open(test_dir + "context/lstmB/output_test_ctx_lstm_128_10_3_1809211930.csv", 'rb')) # 6 2-streams + baseline #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_rgb_1809220100.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_crop.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_gauss.csv", 'rb')) all_detections.append( open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_flowcrop_crop_1809220117.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_flowcrop_gauss_1809220152.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_flowcrop_fovea_1809220136.csv", 'rb')) # Context Fusions # all_detections.append(open(test_dir + "/context_fusion/output_test_3stream_fovea.csv", 'rb')) # all_detections.append(open(test_dir + "/context_fusion/output_test_3stream_crop.csv", 'rb')) # all_detections.append(open(test_dir + "/context_fusion/output_test_3stream_gauss.csv", 'rb')) all_detections.append( open( test_dir + "/context_fusion/output_test_LSTM_FCfusion_contextGT_gauss_1810011737.csv", 'rb')) all_detections.append( open( test_dir + "/context_fusion/output_test_LSTM_FCfusion_context_secondpass_gauss_1810011754.csv", 'rb')) # LSTMs #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_rgb_1809220100.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_crop.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_gauss.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb')) #all_detections.append(open(test_dir + "/context_fusion/output_test_ctx_lstm_fusion_thresh_512_5_3_1809242315.csv", 'rb')) #all_detections.append(open(test_dir + "/context_fusion/output_test_ctx_lstm_512_5_3_1809242252.csv", 'rb')) #all_detections.append(open(test_dir + "/context_fusion/output_test_ctx_lstmavggoodpedro_512_5_3_1809242338.csv", 'rb')) #all_detections.append(open(test_dir + "/context_fusion/output_test_ctx_lstmavg_twophase_thresh02_512_5_3_1809281219.csv", 'rb')) #all_detections.append(open(test_dir + "/context_fusion/output_test_ctx_lstmavg_threephase_512_5_3_1809281317.csv", 'rb')) #all_detections.append(open(test_dir + "/context_fusion/output_test_ctx_lstm_fusion_thresh01_512_5_3_1809281400.csv", 'rb')) #all_detections.append(open(test_dir + "/context_fusion/output_test_ctx_lstmavg_twophase_thresh01_512_5_3_1809281423.csv", 'rb')) #all_detections.append(open(test_dir + "rgb_gauss/output_test_gauss.csv", 'rb')) #all_detections.append(open(test_dir + "augmentation/output_test_sampling_gauss_1809221859.csv", 'rb')) #all_detections.append(open(test_dir + "augmentation/output_test_samplingnoaug_gauss_1809281439.csv", 'rb')) #all_detections.append(open(test_dir + "augmentation/output_test_weightsnew_gauss_1809291104.csv", 'rb')) #all_detections.append(open(test_dir + "augmentation/output_test_weightsaug_gauss_1809261228.csv", 'rb')) # output_test_ctx_lstm_512_5_3_1809242252.csv #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_flowcrop_crop_1809220117.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_flowcrop_gauss_1809220152.csv", 'rb')) #all_detections.append(open(test_dir + "/two-streams/output_test_2stream_flowcrop_fovea_1809220136.csv", 'rb')) # --------------------------------- # New run to compare new flow #all_detections.append(open(test_dir + "/flow/output_test_flowcrop.csv", 'rb')) #all_detections.append(open(test_dir + "/flow/output_test_flow.csv", 'rb')) # New 2 and 3 streams # all_detections.append(open(test_dir + "output_test_gauss.csv", 'rb')) # all_detections.append(open(test_dir + "output_test_gauss_extra.csv", 'rb')) # all_detections.append(open(test_dir + "output_test_3stream_gauss.csv", 'rb')) # all_detections.append(open(test_dir + "output_test_3stream_crop.csv", 'rb')) # Flow, context, 2-stream, 3-stream run #all_detections.append(open(test_dir + "output_test_ctx.csv", 'rb')) #all_detections.append(open(test_dir + "output_test_flow.csv", 'rb')) #all_detections.append(open(test_dir + "output_test_2stream.csv", 'rb')) #all_detections.append(open(test_dir + "output_test_3stream.csv", 'rb')) # RGB run # all_detections.append(open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb')) # all_detections.append(open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb')) # all_detections.append(open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb')) # all_detections.append(open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb')) balancing = False all_gndtruths = [] for i in range(len(all_detections)): if balancing is False: all_gndtruths.append( open(root_dir + "AVA_Test_Custom_Corrected.csv", 'rb')) else: all_gndtruths.append( open(root_dir + "AVA_Test_Custom_Corrected_Balanced.csv", 'rb')) """Runs evaluations given input files. Args: labelmap: file object containing map of labels to consider, in pbtxt format groundtruth: file object detections: file object exclusions: file object or None. """ categories, class_whitelist = read_labelmap(labelmap) logging.info("CATEGORIES (%d):\n%s", len(categories), pprint.pformat(categories, indent=2)) excluded_keys = read_exclusions(exclusions) # Reads detections data. x_axis = [] xpose_ax = [] xobj_ax = [] xhuman_ax = [] ypose_ax = [] yobj_ax = [] yhuman_ax = [] colors_pose = [] colors_obj = [] colors_human = [] finalmAPs = [] colors = [] maxY = -1.0 for detections, gndtruth, filter_type in zip(all_detections, all_gndtruths, filters): pascal_evaluator = None metrics = None actions = None start = 0 pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator( categories, matching_iou_threshold=iou) # Reads the ground truth data. boxes, labels, _ = read_csv(gndtruth, class_whitelist) start = time.time() for image_key in boxes: if image_key in excluded_keys: logging.info(("Found excluded timestamp in ground truth: %s. " "It will be ignored."), image_key) continue pascal_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: np.array(boxes[image_key], dtype=float), standard_fields.InputDataFields.groundtruth_classes: np.array(labels[image_key], dtype=int), standard_fields.InputDataFields.groundtruth_difficult: np.zeros(len(boxes[image_key]), dtype=bool) }) print_time("convert groundtruth", start) # Run evaluation boxes, labels, scores = read_csv(detections, class_whitelist) start = time.time() for image_key in boxes: if image_key in excluded_keys: logging.info(("Found excluded timestamp in detections: %s. " "It will be ignored."), image_key) continue pascal_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: np.array(boxes[image_key], dtype=float), standard_fields.DetectionResultFields.detection_classes: np.array(labels[image_key], dtype=int), standard_fields.DetectionResultFields.detection_scores: np.array(scores[image_key], dtype=float) }) print_time("convert detections", start) start = time.time() metrics = pascal_evaluator.evaluate() print_time("run_evaluator", start) # TODO Show a pretty histogram here besides pprint actions = list(metrics.keys()) final_value = 0.0 for m in actions: ms = m.split("/")[-1] if ms == 'mAP@' + str(iou) + 'IOU': final_value = metrics[m] finalmAPs.append(final_value) else: # x_axis.append(ms) # y_axis.append(metrics[m]) for cat in categories: if cat['name'].split("/")[-1] == ms: if maxY < metrics[m]: maxY = metrics[m] if cat['id'] <= 10: xpose_ax.append("[" + filter_type + "] " + ms) ypose_ax.append(metrics[m]) colors_pose.append('pose') elif cat['id'] <= 22: xobj_ax.append("[" + filter_type + "] " + ms) yobj_ax.append(metrics[m]) colors_obj.append('human-object') else: xhuman_ax.append("[" + filter_type + "] " + ms) yhuman_ax.append(metrics[m]) colors_human.append('human-human') # Make a confusion matrix for this run pascal_evaluator = None x_axis = split_interleave(xpose_ax) + split_interleave( xobj_ax) + split_interleave(xhuman_ax) y_axis = split_interleave(ypose_ax) + split_interleave( yobj_ax) + split_interleave(yhuman_ax) colors = split_interleave(colors_pose) + split_interleave( colors_obj) + split_interleave(colors_human) plt.ylabel('frame-mAP') top = maxY + 0.1 # offset a bit so it looks good sns.set_style("whitegrid") g = sns.barplot(y_axis, x_axis, hue=colors, palette=['red', 'blue', 'green']) ax = g #ax.legend(loc='lower right') # annotate axis = seaborn axis # for p in ax.patches: # ax.annotate("%.3f" % p.get_height(), (p.get_x() + p.get_width() / 2., p.get_height()), # ha='center', va='center', fontsize=10, color='gray', rotation=90, xytext=(0, 20), # textcoords='offset points') # ax.set_ylim(-1, len(y_axis)) sns.set() ax.tick_params(labelsize=6) for p in ax.patches: p.set_height(p.get_height() * 3) ax.annotate("%.3f" % p.get_width(), (p.get_x() + p.get_width(), p.get_y()), xytext=(5, -5), fontsize=8, color='gray', textcoords='offset points') _ = g.set_xlim(0, top) # To make space for the annotations pprint.pprint(metrics, indent=2) ax.set(ylabel="", xlabel="AP") plt.xticks(rotation=0) title = "" file = open("results.txt", "w") for filter_type, mAP in zip(filters, finalmAPs): ft = filter_type + ': mAP@' + str(iou) + 'IOU = ' + str(mAP) + '\n' title += ft file.write(ft) file.close() # ax.figure.tight_layout() ax.figure.subplots_adjust(left=0.2) # change 0.3 to suit your needs. plt.title(title) plt.gca().xaxis.grid(True) plt.show() if len(all_detections) == 1: sz = 2 grid_sz = [1, 1] elif len(all_detections) == 2: sz = 3 grid_sz = [1, 2] elif len(all_detections) == 3: sz = 4 grid_sz = [2, 2] else: sz = 5 grid_sz = [2, 2] for i in range(1, sz): print(i) plt.subplot(grid_sz[0], grid_sz[1], i) if i <= len(all_detections): # Confusion matrix classes = [] for k in categories: classes.append(k['name']) cm = confusion_matrix(all_gndtruths[i - 1], all_detections[i - 1], x_axis) g = sns.heatmap(cm, annot=True, fmt="d", xticklabels=classes[:10], yticklabels=classes[:10], linewidths=0.5, linecolor='black', cbar=True) #t = 0 # for ytick_label, xtick_label in zip(g.axes.get_yticklabels(), g.axes.get_xticklabels()): # if t <= 9: # ytick_label.set_color("r") # xtick_label.set_color("r") # elif t <= 22: # ytick_label.set_color("b") # xtick_label.set_color("b") # else: # ytick_label.set_color("g") # xtick_label.set_color("g") # t += 1 plt.xticks(rotation=-90) plt.title("Pose Confusion Matrix (" + filters[i - 1] + ")") plt.show()
def run_evaluation_threshold(labelmap, groundtruth, exclusions, iou): # sns.palplot(sns.diverging_palette(128, 240, n=10)) # seq_col_brew = sns.color_palette("Blues_r", 4) # For sequential, blue gradient in reverse # Qualitative data palette # current_palette = sns.color_palette("Paired") # sns.set_palette(current_palette) # Make sure not to mess this up filters = [] filters.append("0.1") filters.append("0.2") filters.append("0.3") filters.append("0.4") filters.append("0.5") filters.append("0.6") filters.append("0.7") filters.append("0.8") filters.append("0.9") root_dir = '../../../data/AVA/files/' ftype = "fusion" all_detections = [] ts = "1809281055" for f in filters: all_detections.append( open( "../thresholds/context_" + ftype + "/predictions_fusion_avg_fovea_" + ts + "_" + f + ".csv", 'rb')) all_gndtruths = [] for i in range(len(filters)): all_gndtruths.append( open(root_dir + "AVA_Val_Custom_Corrected.csv", 'rb')) #all_gndtruths.append(open("AVA_Test_Custom_Corrected.csv", 'rb')) #all_gndtruths.append(open("AVA_Test_Custom_Corrected.csv", 'rb')) """Runs evaluations given input files. Args: labelmap: file object containing map of labels to consider, in pbtxt format groundtruth: file object detections: file object exclusions: file object or None. """ categories, class_whitelist = read_labelmap(labelmap) logging.info("CATEGORIES (%d):\n%s", len(categories), pprint.pformat(categories, indent=2)) excluded_keys = read_exclusions(exclusions) # Reads detections data. x_axis = [] xpose_ax = [] xobj_ax = [] xhuman_ax = [] ypose_ax = [] yobj_ax = [] yhuman_ax = [] colors_pose = [] colors_obj = [] colors_human = [] finalmAPs = [] colors = [] maxY = -1.0 for detections, gndtruth, filter_type in zip(all_detections, all_gndtruths, filters): pascal_evaluator = None metrics = None actions = None start = 0 pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator( categories, matching_iou_threshold=iou) # Reads the ground truth data. boxes, labels, _ = read_csv(gndtruth, class_whitelist) start = time.time() for image_key in boxes: if image_key in excluded_keys: logging.info(("Found excluded timestamp in ground truth: %s. " "It will be ignored."), image_key) continue pascal_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: np.array(boxes[image_key], dtype=float), standard_fields.InputDataFields.groundtruth_classes: np.array(labels[image_key], dtype=int), standard_fields.InputDataFields.groundtruth_difficult: np.zeros(len(boxes[image_key]), dtype=bool) }) print_time("convert groundtruth", start) # Run evaluation boxes, labels, scores = read_csv(detections, class_whitelist) start = time.time() for image_key in boxes: if image_key in excluded_keys: logging.info(("Found excluded timestamp in detections: %s. " "It will be ignored."), image_key) continue pascal_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: np.array(boxes[image_key], dtype=float), standard_fields.DetectionResultFields.detection_classes: np.array(labels[image_key], dtype=int), standard_fields.DetectionResultFields.detection_scores: np.array(scores[image_key], dtype=float) }) print_time("convert detections", start) start = time.time() metrics = pascal_evaluator.evaluate() print_time("run_evaluator", start) # TODO Show a pretty histogram here besides pprint actions = list(metrics.keys()) final_value = 0.0 for m in actions: ms = m.split("/")[-1] if ms == 'mAP@' + str(iou) + 'IOU': final_value = metrics[m] finalmAPs.append(final_value) else: # x_axis.append(ms) # y_axis.append(metrics[m]) for cat in categories: if cat['name'].split("/")[-1] == ms: if maxY < metrics[m]: maxY = metrics[m] if cat['id'] <= 10: xpose_ax.append("(" + filter_type + ") " + ms) ypose_ax.append(metrics[m]) colors_pose.append('red') elif cat['id'] <= 22: xobj_ax.append("(" + filter_type + ") " + ms) yobj_ax.append(metrics[m]) colors_obj.append('blue') else: xhuman_ax.append("(" + filter_type + ") " + ms) yhuman_ax.append(metrics[m]) colors_human.append('green') # Make a confusion matrix for this run pascal_evaluator = None x_axis = split_interleave(xpose_ax) + split_interleave( xobj_ax) + split_interleave(xhuman_ax) y_axis = split_interleave(ypose_ax) + split_interleave( yobj_ax) + split_interleave(yhuman_ax) colors = split_interleave(colors_pose) + split_interleave( colors_obj) + split_interleave(colors_human) print(filters) print(finalmAPs) plt.ylabel('frame-mAP') top = 0.1 # offset a bit so it looks good sns.set_style("whitegrid") clrs = ['blue' if (x < max(finalmAPs)) else 'red' for x in finalmAPs] g = sns.barplot(filters, finalmAPs, palette=clrs) ax = g # annotate axis = seaborn axis for p in ax.patches: ax.annotate("%.4f" % p.get_height(), (p.get_x() + p.get_width() / 2., p.get_height()), ha='center', va='center', fontsize=10, color='gray', rotation=90, xytext=(0, 20), textcoords='offset points') _ = g.set_ylim(0, top) # To make space for the annotations plt.show()
def run_evaluation(labelmap, groundtruth, exclusions, iou): root_dir = '../../../data/AVA/files/' test_dir = "../test_outputs/" # Make sure not to mess this up experiments_filters = {} experiments_detections = {} experiment = 'balancing' # Baseline experiments_filters['baseline'] = ['RGB', 'Flow', 'RGB+Flow'] experiments_detections['baseline'] = [ open(test_dir + "/rgb_rgb/output_test_rgb.csv", 'rb'), open(test_dir + "/flow/output_test_flow.csv", 'rb'), open(test_dir + "/two-streams/output_test_2stream_rgb_1809220100.csv", 'rb') ] # RGBS experiments_filters['rgb-streams-aug'] = ['Crop', 'Gauss', 'Fovea'] experiments_detections['rgb-streams-aug'] = [ open(test_dir + "/rgb_crop/output_test_crop.csv", 'rb'), open(test_dir + "/rgb_gauss/output_test_gauss.csv", 'rb'), open(test_dir + "/rgb_fovea/output_test_fovea.csv", 'rb') ] # Flows experiments_filters['flow vs flowcrop'] = ['Flow', 'Flowcrop'] experiments_detections['flow vs flowcrop'] = [ open(test_dir + "/flow/output_test_flow.csv", 'rb'), open(test_dir + "/flow/output_test_flowcrop.csv", 'rb'), ] # Two-streams experiments_filters['two-streams'] = [ 'Two-Stream-Crop', 'Two-Stream-GBB', 'Two-Stream-Fovea' ] experiments_detections['two-streams'] = [ open(test_dir + "/two-streams/output_test_2stream_crop_1807252254.csv", 'rb'), open( test_dir + "/two-streams/output_test_2stream_gauss_1807252309.csv", 'rb'), open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb') ] #experiments_filters['two-streams'] = ['Two-Stream-Crop', 'Two-Stream-Fovea'] #experiments_detections['two-streams'] = [open(test_dir + "/two-streams/output_test_2stream_crop_1807252254.csv", 'rb'), open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb')] experiments_filters['two-streams-flowcrop'] = [ 'Two-Stream-Crop (Flowcrop)', 'Two-Stream-Gauss (Flowcrop)', 'Two-Stream-Fovea (Flowcrop)' ] experiments_detections['two-streams-flowcrop'] = [ open( test_dir + "/two-streams/output_test_2stream_flowcrop_crop_1809220117.csv", 'rb'), open( test_dir + "/two-streams/output_test_2stream_flowcrop_gauss_1809220152.csv", 'rb'), open( test_dir + "/two-streams/output_test_2stream_flowcrop_fovea_1809220136.csv", 'rb') ] # MLP VS LSTM experiments_filters['mlp vs lstm'] = ['MLP', 'LSTMA', 'LSTMB'] experiments_detections['mlp vs lstm'] = [ open(test_dir + "context/mlp/output_test_ctx_mlp_1809212356.csv", 'rb'), open(test_dir + "context/lstmA/output_test_ctx_lstm_128_3_3.csv", 'rb'), open(test_dir + "context/lstmB/output_test_ctx_lstm_128_3_3.csv", 'rb') ] # LSTMS nhu = 512 neighbs = 3 tws = 3 experiments_filters['lstmA vs lstmB'] = ['LSTM A', 'LSTM B'] experiments_detections['lstmA vs lstmB'] = [ open( test_dir + "context/lstmA/output_test_ctx_lstm_" + str(nhu) + "_" + str(tws) + "_" + str(neighbs) + ".csv", 'rb'), open( test_dir + "context/lstmB/output_test_ctx_lstm_" + str(nhu) + "_" + str(tws) + "_" + str(neighbs) + ".csv", 'rb') ] # Fusions experiments_filters['class-score-fusion-gt'] = [ '2-stream Fovea', 'Class Score Fusion (GT)' ] experiments_detections['class-score-fusion-gt'] = [ open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb'), open( test_dir + "/context_fusion/output_test_ctx_lstm_fusion_512_5_3_1809242338.csv", 'rb') ] experiments_filters['class-score-fusion-two-pass'] = [ '2-stream Fovea', 'Class Score Fusion (Two Pass)' ] experiments_detections['class-score-fusion-two-pass'] = [ open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb'), open( test_dir + "/context_fusion/output_test_ctx_lstmavg_twophase_512_5_3_1809281149.csv", 'rb') ] experiments_filters['dense-fusion-gt'] = ['2-stream Fovea', 'Dense Fusion'] experiments_detections['dense-fusion-gt'] = [ open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb'), open( test_dir + "/context_fusion/output_test_LSTM_FCfusion_contextGT_fovea_1810011737.csv", 'rb') ] experiments_filters['dense-fusion-two-pass'] = [ '2-stream Fovea', 'Dense Fusion (Two Pass)' ] experiments_detections['dense-fusion-two-pass'] = [ open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb'), open( test_dir + "/context_fusion/output_test_LSTM_FCfusion_context_secondpass_fovea_1810011754.csv", 'rb') ] # Voting experiments_filters['class-score-fusion-gt-voting'] = [ 'Class Score Fusion (0.2)', 'Class Score Fusion (0.1)' ] experiments_detections['class-score-fusion-gt-voting'] = [ open( test_dir + "/context_fusion/output_test_ctx_lstm_fusion_thresh02_512_5_3_1809242315.csv", 'rb'), open( test_dir + "/context_fusion/output_test_ctx_lstm_fusion_thresh01_512_5_3_1809281400.csv", 'rb') ] experiments_filters['class-score-fusion-two-pass-voting'] = [ 'Class Score Fusion (Two-pass) (0.2)', 'Class Score Fusion (Two-pass) (0.1)' ] experiments_detections['class-score-fusion-two-pass-voting'] = [ open( test_dir + "/context_fusion/output_test_ctx_lstmavg_twophase_thresh02_512_5_3_1809281219.csv", 'rb'), open( test_dir + "/context_fusion/output_test_ctx_lstmavg_twophase_thresh01_512_5_3_1809281423.csv", 'rb') ] # Balancing experiments_filters['balancing'] = ['Imbalanced', 'Oversampling'] experiments_detections['balancing'] = [ open(test_dir + "rgb_gauss/output_test_gauss.csv", 'rb'), open( test_dir + "augmentation/output_test_samplingnoaug_gauss_1809281439.csv", 'rb') ] # Extra experiments experiments_filters['context-fusion mlp'] = [ '2-stream Fovea', 'Dense Fusion MLP' ] experiments_detections['context-fusion mlp'] = [ open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb'), open(test_dir + "/context_fusion/output_test_3stream_fovea.csv", 'rb') ] experiments_filters['context-fusion extra pass'] = [ 'Class Score (Two Pass)', 'Class Score (Extra Pass)' ] experiments_detections['context-fusion extra pass'] = [ open( test_dir + "/context_fusion/output_test_ctx_lstmavg_twophase_512_5_3_1809281149.csv", 'rb'), open( test_dir + "/context_fusion/output_test_ctx_lstmavg_threephase_512_5_3_1809281317.csv", 'rb') ] experiments_filters['random'] = ['Random guessing'] experiments_detections['random'] = [ open(test_dir + "random/output_test_random_1809221552.csv", 'rb') ] # Best experiments_filters['best'] = [ '2-stream Fovea', 'Class Score Fusion (GT)', 'Class Score Fusion (GT, v=0.1)' ] experiments_detections['best'] = [ open(test_dir + "/two-streams/output_test_2stream_fovea.csv", 'rb'), open( test_dir + "/context_fusion/output_test_ctx_lstm_fusion_512_5_3_1809242338.csv", 'rb'), open( test_dir + "/context_fusion/output_test_ctx_lstm_fusion_thresh01_512_5_3_1809281400.csv", 'rb') ] filters = experiments_filters[experiment] all_detections = experiments_detections[experiment] balancing = False all_gndtruths = [] for i in range(len(all_detections)): if balancing is False: all_gndtruths.append( open(root_dir + "AVA_Test_Custom_Corrected.csv", 'rb')) else: all_gndtruths.append( open(root_dir + "AVA_Test_Custom_Corrected_Balanced.csv", 'rb')) """Runs evaluations given input files. Args: labelmap: file object containing map of labels to consider, in pbtxt format groundtruth: file object detections: file object exclusions: file object or None. """ categories, class_whitelist = read_labelmap(labelmap) logging.info("CATEGORIES (%d):\n%s", len(categories), pprint.pformat(categories, indent=2)) excluded_keys = read_exclusions(exclusions) # Reads detections data. x_axis = [] xpose_ax = [] xobj_ax = [] xhuman_ax = [] ypose_ax = [] yobj_ax = [] yhuman_ax = [] colors_pose = [] colors_obj = [] colors_human = [] finalmAPs = [] colors = [] maxY = -1.0 for detections, gndtruth, filter_type in zip(all_detections, all_gndtruths, filters): pascal_evaluator = None metrics = None actions = None start = 0 pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator( categories, matching_iou_threshold=iou) # Reads the ground truth data. boxes, labels, _ = read_csv(gndtruth, class_whitelist) start = time.time() for image_key in boxes: if image_key in excluded_keys: logging.info(("Found excluded timestamp in ground truth: %s. " "It will be ignored."), image_key) continue pascal_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: np.array(boxes[image_key], dtype=float), standard_fields.InputDataFields.groundtruth_classes: np.array(labels[image_key], dtype=int), standard_fields.InputDataFields.groundtruth_difficult: np.zeros(len(boxes[image_key]), dtype=bool) }) print_time("convert groundtruth", start) # Run evaluation boxes, labels, scores = read_csv(detections, class_whitelist) start = time.time() for image_key in boxes: if image_key in excluded_keys: logging.info(("Found excluded timestamp in detections: %s. " "It will be ignored."), image_key) continue pascal_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: np.array(boxes[image_key], dtype=float), standard_fields.DetectionResultFields.detection_classes: np.array(labels[image_key], dtype=int), standard_fields.DetectionResultFields.detection_scores: np.array(scores[image_key], dtype=float) }) print_time("convert detections", start) start = time.time() metrics = pascal_evaluator.evaluate() print_time("run_evaluator", start) # TODO Show a pretty histogram here besides pprint actions = list(metrics.keys()) final_value = 0.0 for m in actions: ms = m.split("/")[-1] if ms == 'mAP@' + str(iou) + 'IOU': final_value = metrics[m] finalmAPs.append(final_value) else: # x_axis.append(ms) # y_axis.append(metrics[m]) for cat in categories: if cat['name'].split("/")[-1] == ms: if maxY < metrics[m]: maxY = metrics[m] if cat['id'] <= 10: xpose_ax.append("[" + filter_type + "] " + ms) ypose_ax.append(metrics[m]) colors_pose.append('pose') elif cat['id'] <= 22: xobj_ax.append("[" + filter_type + "] " + ms) yobj_ax.append(metrics[m]) colors_obj.append('human-object') else: xhuman_ax.append("[" + filter_type + "] " + ms) yhuman_ax.append(metrics[m]) colors_human.append('human-human') # Make a confusion matrix for this run pascal_evaluator = None parts = len(filters) x_axis = split_interleave(xpose_ax, parts) + split_interleave( xobj_ax, parts) + split_interleave(xhuman_ax, parts) y_axis = split_interleave(ypose_ax, parts) + split_interleave( yobj_ax, parts) + split_interleave(yhuman_ax, parts) colors = split_interleave(colors_pose, parts) + split_interleave( colors_obj, parts) + split_interleave(colors_human, parts) plt.ylabel('frame-mAP') top = maxY + 0.1 # offset a bit so it looks good sns.set_style("whitegrid") g = sns.barplot(y_axis, x_axis, hue=colors, palette=['red', 'blue', 'green']) ax = g # ax.legend(loc='lower right') # annotate axis = seaborn axis # for p in ax.patches: # ax.annotate("%.3f" % p.get_height(), (p.get_x() + p.get_width() / 2., p.get_height()), # ha='center', va='center', fontsize=10, color='gray', rotation=90, xytext=(0, 20), # textcoords='offset points') # ax.set_ylim(-1, len(y_axis)) sns.set() ax.tick_params(labelsize=6) for p in ax.patches: p.set_height(p.get_height() * 3) ax.annotate("%.3f" % p.get_width(), (p.get_x() + p.get_width(), p.get_y()), xytext=(5, -5), fontsize=8, color='gray', textcoords='offset points') _ = g.set_xlim(0, top) # To make space for the annotations pprint.pprint(metrics, indent=2) ax.set(ylabel="", xlabel="AP") plt.xticks(rotation=0) title = "" file = open("results.txt", "w") for filter_type, mAP in zip(filters, finalmAPs): ft = filter_type + ': mAP@' + str(iou) + 'IOU = ' + str(mAP) + '\n' title += ft file.write(ft) file.close() # ax.figure.tight_layout() ax.figure.subplots_adjust(left=0.2) # change 0.3 to suit your needs. plt.title(title) plt.gca().xaxis.grid(True) plt.show() if len(all_detections) == 1: sz = 2 grid_sz = [1, 1] elif len(all_detections) == 2: sz = 3 grid_sz = [1, 2] elif len(all_detections) == 3: sz = 4 grid_sz = [2, 2] else: sz = 5 grid_sz = [2, 2] for i in range(1, sz): print(i) plt.subplot(grid_sz[0], grid_sz[1], i) if i <= len(all_detections): # Confusion matrix classes = [] for k in categories: classes.append(k['name']) cm = confusion_matrix(all_gndtruths[i - 1], all_detections[i - 1], x_axis) g = sns.heatmap(cm, annot=True, fmt="d", xticklabels=classes[:10], yticklabels=classes[:10], linewidths=0.5, linecolor='black', cbar=True, vmin=0, vmax=2000) # t = 0 # for ytick_label, xtick_label in zip(g.axes.get_yticklabels(), g.axes.get_xticklabels()): # if t <= 9: # ytick_label.set_color("r") # xtick_label.set_color("r") # elif t <= 22: # ytick_label.set_color("b") # xtick_label.set_color("b") # else: # ytick_label.set_color("g") # xtick_label.set_color("g") # t += 1 plt.xticks(rotation=-90) plt.title("Pose Confusion Matrix (" + filters[i - 1] + ")") plt.show()