def read_data_and_evaluate(input_config, eval_config): """Reads pre-computed object detections and groundtruth from tf_record. Args: input_config: input config proto of type object_detection.protos.InputReader. eval_config: evaluation config proto of type object_detection.protos.EvalConfig. Returns: Evaluated detections metrics. Raises: ValueError: if input_reader type is not supported or metric type is unknown. """ if input_config.WhichOneof('input_reader') == 'tf_record_input_reader': input_paths = input_config.tf_record_input_reader.input_path categories = label_map_util.create_categories_from_labelmap( input_config.label_map_path) object_detection_evaluators = evaluator.get_evaluators( eval_config, categories) # Support a single evaluator object_detection_evaluator = object_detection_evaluators[0] skipped_images = 0 processed_images = 0 for input_path in _generate_filenames(input_paths): tf.logging.info('Processing file: {0}'.format(input_path)) record_iterator = tf.python_io.tf_record_iterator(path=input_path) data_parser = tf_example_parser.TfExampleDetectionAndGTParser() for string_record in record_iterator: tf.logging.log_every_n(tf.logging.INFO, 'Processed %d images...', 1000, processed_images) processed_images += 1 example = tf.train.Example() example.ParseFromString(string_record) decoded_dict = data_parser.parse(example) if decoded_dict: object_detection_evaluator.add_single_ground_truth_image_info( decoded_dict[ standard_fields.DetectionResultFields.key], decoded_dict) object_detection_evaluator.add_single_detected_image_info( decoded_dict[ standard_fields.DetectionResultFields.key], decoded_dict) else: skipped_images += 1 tf.logging.info( 'Skipped images: {0}'.format(skipped_images)) return object_detection_evaluator.evaluate() raise ValueError('Unsupported input_reader_config.')
def process_detections(detections_record, categories): record_iterator = tf.python_io.tf_record_iterator(path=detections_record) data_parser = tf_example_parser.TfExampleDetectionAndGTParser() confusion_matrix = np.zeros(shape=(len(categories) + 1, len(categories) + 1)) category_count = {} for i in range(1, len(categories) + 1): category_count[i] = 0 image_index = 0 for string_record in record_iterator: example = tf.train.Example() example.ParseFromString(string_record) decoded_dict = data_parser.parse(example) image_index += 1 if decoded_dict: groundtruth_boxes = decoded_dict[ standard_fields.InputDataFields.groundtruth_boxes] groundtruth_classes = decoded_dict[ standard_fields.InputDataFields.groundtruth_classes] for c in groundtruth_classes: category_count[c] = category_count[c] + 1 detection_scores = decoded_dict[ standard_fields.DetectionResultFields.detection_scores] detection_classes = decoded_dict[ standard_fields.DetectionResultFields.detection_classes][ detection_scores >= CONFIDENCE_THRESHOLD] detection_boxes = decoded_dict[ standard_fields.DetectionResultFields.detection_boxes][ detection_scores >= CONFIDENCE_THRESHOLD] matches = [] if image_index % 100 == 0: print("Processed %d images" % (image_index)) for i in range(len(groundtruth_boxes)): for j in range(len(detection_boxes)): iou = compute_iou(groundtruth_boxes[i], detection_boxes[j]) if iou > IOU_THRESHOLD: matches.append([i, j, iou]) matches = np.array(matches) if matches.shape[0] > 0: # Sort list of matches by descending IOU so we can remove duplicate detections # while keeping the highest IOU entry. matches = matches[matches[:, 2].argsort()[::-1][:len(matches)]] # Remove duplicate detections from the list. matches = matches[np.unique(matches[:, 1], return_index=True)[1]] # Sort the list again by descending IOU. Removing duplicates doesn't preserve # our previous sort. matches = matches[matches[:, 2].argsort()[::-1][:len(matches)]] # Remove duplicate ground truths from the list. matches = matches[np.unique(matches[:, 0], return_index=True)[1]] for i in range(len(groundtruth_boxes)): if matches.shape[0] > 0 and matches[matches[:, 0] == i].shape[0] == 1: confusion_matrix[groundtruth_classes[i] - 1][ detection_classes[int(matches[matches[:, 0] == i, 1][0])] - 1] += 1 else: confusion_matrix[groundtruth_classes[i] - 1][confusion_matrix.shape[1] - 1] += 1 for i in range(len(detection_boxes)): if matches.shape[0] > 0 and matches[matches[:, 1] == i].shape[0] == 0: confusion_matrix[confusion_matrix.shape[0] - 1][detection_classes[i] - 1] += 1 else: print("Skipped image %d" % (image_index)) print("Processed %d images" % (image_index)) return (confusion_matrix, category_count)
def testParseDetectionsAndGT(self): source_id = b'abc.jpg' # y_min, x_min, y_max, x_max object_bb = np.array([[0.0, 0.5, 0.3], [0.0, 0.1, 0.6], [1.0, 0.6, 0.8], [1.0, 0.6, 0.7]]).transpose() detection_bb = np.array([[0.1, 0.2], [0.0, 0.8], [1.0, 0.6], [1.0, 0.85]]).transpose() object_class_label = [1, 1, 2] object_difficult = [1, 0, 0] object_group_of = [0, 0, 1] verified_labels = [1, 2, 3, 4] detection_class_label = [2, 1] detection_score = [0.5, 0.3] features = { fields.TfExampleFields.source_id: self._BytesFeature(source_id), fields.TfExampleFields.object_bbox_ymin: self._FloatFeature(object_bb[:, 0].tolist()), fields.TfExampleFields.object_bbox_xmin: self._FloatFeature(object_bb[:, 1].tolist()), fields.TfExampleFields.object_bbox_ymax: self._FloatFeature(object_bb[:, 2].tolist()), fields.TfExampleFields.object_bbox_xmax: self._FloatFeature(object_bb[:, 3].tolist()), fields.TfExampleFields.detection_bbox_ymin: self._FloatFeature(detection_bb[:, 0].tolist()), fields.TfExampleFields.detection_bbox_xmin: self._FloatFeature(detection_bb[:, 1].tolist()), fields.TfExampleFields.detection_bbox_ymax: self._FloatFeature(detection_bb[:, 2].tolist()), fields.TfExampleFields.detection_bbox_xmax: self._FloatFeature(detection_bb[:, 3].tolist()), fields.TfExampleFields.detection_class_label: self._Int64Feature(detection_class_label), fields.TfExampleFields.detection_score: self._FloatFeature(detection_score), } example = tf.train.Example(features=tf.train.Features( feature=features)) parser = tf_example_parser.TfExampleDetectionAndGTParser() results_dict = parser.parse(example) self.assertIsNone(results_dict) features[fields.TfExampleFields.object_class_label] = ( self._Int64Feature(object_class_label)) features[fields.TfExampleFields.object_difficult] = ( self._Int64Feature(object_difficult)) example = tf.train.Example(features=tf.train.Features( feature=features)) results_dict = parser.parse(example) self.assertIsNotNone(results_dict) self.assertEqual(source_id, results_dict[fields.DetectionResultFields.key]) np_testing.assert_almost_equal( object_bb, results_dict[fields.InputDataFields.groundtruth_boxes]) np_testing.assert_almost_equal( detection_bb, results_dict[fields.DetectionResultFields.detection_boxes]) np_testing.assert_almost_equal( detection_score, results_dict[fields.DetectionResultFields.detection_scores]) np_testing.assert_almost_equal( detection_class_label, results_dict[fields.DetectionResultFields.detection_classes]) np_testing.assert_almost_equal( object_difficult, results_dict[fields.InputDataFields.groundtruth_difficult]) np_testing.assert_almost_equal( object_class_label, results_dict[fields.InputDataFields.groundtruth_classes]) parser = tf_example_parser.TfExampleDetectionAndGTParser() features[fields.TfExampleFields.object_group_of] = ( self._Int64Feature(object_group_of)) example = tf.train.Example(features=tf.train.Features( feature=features)) results_dict = parser.parse(example) self.assertIsNotNone(results_dict) np_testing.assert_equal( object_group_of, results_dict[fields.InputDataFields.groundtruth_group_of]) features[fields.TfExampleFields.image_class_label] = ( self._Int64Feature(verified_labels)) example = tf.train.Example(features=tf.train.Features( feature=features)) results_dict = parser.parse(example) self.assertIsNotNone(results_dict) np_testing.assert_equal( verified_labels, results_dict[fields.InputDataFields.groundtruth_image_classes])
def process_detections(detections_record, categories): record_iterator = tf.python_io.tf_record_iterator(path=detections_record) data_parser = tf_example_parser.TfExampleDetectionAndGTParser() confusion_matrix = np.zeros(shape=(len(categories) + 1, len(categories) + 1)) image_index = 0 for string_record in record_iterator: example = tf.train.Example() example.ParseFromString(string_record) decoded_dict = data_parser.parse(example) image_index += 1 if decoded_dict: groundtruth_boxes = decoded_dict[ standard_fields.InputDataFields.groundtruth_boxes] groundtruth_classes = decoded_dict[ standard_fields.InputDataFields.groundtruth_classes] detection_scores = decoded_dict[ standard_fields.DetectionResultFields.detection_scores] detection_classes = decoded_dict[ standard_fields.DetectionResultFields.detection_classes][ detection_scores >= CONFIDENCE_THRESHOLD] detection_boxes = decoded_dict[ standard_fields.DetectionResultFields.detection_boxes][ detection_scores >= CONFIDENCE_THRESHOLD] matches = [] if image_index % 100 == 0: print("Processed %d images" % (image_index)) for i in range(len(groundtruth_boxes)): for j in range(len(detection_boxes)): iou = compute_iou(groundtruth_boxes[i], detection_boxes[j]) if iou > IOU_THRESHOLD: matches.append([i, j, iou]) matches = np.array(matches) if matches.shape[0] > 0: # Sort list of matches by descending IOU so we can remove duplicate detections # while keeping the highest IOU entry. matches = matches[matches[:, 2].argsort()[::-1][:len(matches)]] # Remove duplicate detections from the list. matches = matches[np.unique(matches[:, 1], return_index=True)[1]] # Sort the list again by descending IOU. Removing duplicates doesn't preserve # our previous sort. matches = matches[matches[:, 2].argsort()[::-1][:len(matches)]] # Remove duplicate ground truths from the list. matches = matches[np.unique(matches[:, 0], return_index=True)[1]] errors = { "wrong_class": [], "not_detected": [], "false_detected": [] } has_errors = False for i in range(len(groundtruth_boxes)): if matches.shape[0] > 0 and matches[matches[:, 0] == i].shape[0] == 1: confusion_matrix[groundtruth_classes[i] - 1][ detection_classes[int(matches[matches[:, 0] == i, 1][0])] - 1] += 1 if groundtruth_classes[i] != detection_classes[int( matches[matches[:, 0] == i, 1][0])]: # Clase equivocada errors["wrong_class"].append({ "box": detection_boxes[int(matches[matches[:, 0] == i, 1][0])], "class": detection_classes[int(matches[matches[:, 0] == i, 1][0])] }) has_errors = True else: confusion_matrix[groundtruth_classes[i] - 1][confusion_matrix.shape[1] - 1] += 1 # Objeto no detectado errors["not_detected"].append({ "box": groundtruth_boxes[i], "class": groundtruth_classes[i] }) has_errors = True for i in range(len(detection_boxes)): if matches.shape[0] > 0 and matches[matches[:, 1] == i].shape[0] == 0: confusion_matrix[confusion_matrix.shape[0] - 1][detection_classes[i] - 1] += 1 # Objeto detectado incorrectamente errors["false_detected"].append({ "box": detection_boxes[i], "class": detection_classes[i] }) has_errors = True if has_errors and FLAGS.output_dir is not None: save_incorrect_example(example, groundtruth_boxes, groundtruth_classes, detection_boxes, detection_classes, categories, errors) else: print("Skipped image %d" % (image_index)) print("Processed %d images" % (image_index)) return confusion_matrix
def process_detections(detections_record, categories): record_iterator = tf.python_io.tf_record_iterator(path=detections_record) data_parser = tf_example_parser.TfExampleDetectionAndGTParser() confusion_matrix = np.zeros(shape=(len(categories) + 1, len(categories) + 1)) num_shown = 0 image_index = 0 os.makedirs(detectionDirName, exist_ok=True) for string_record in record_iterator: example = tf.train.Example() example.ParseFromString(string_record) decoded_dict = data_parser.parse(example) image_index += 1 if decoded_dict: groundtruth_boxes = decoded_dict[ standard_fields.InputDataFields.groundtruth_boxes] groundtruth_classes = decoded_dict[ standard_fields.InputDataFields.groundtruth_classes] detection_scores = decoded_dict[ standard_fields.DetectionResultFields.detection_scores] detection_classes = decoded_dict[ standard_fields.DetectionResultFields.detection_classes][ detection_scores >= CONFIDENCE_THRESHOLD] detection_boxes = decoded_dict[ standard_fields.DetectionResultFields.detection_boxes][ detection_scores >= CONFIDENCE_THRESHOLD] if num_shown < NUM_TO_SHOW: # Convert encoded image in example TF Record to image testing = TfDecoder() features = testing.decode(string_record) image = features['image'] with tf.Session() as sess: image = image.eval() im = PIL.Image.fromarray(image) for box in groundtruth_boxes: vis_util.draw_bounding_box_on_image( im, box[0] * IMAGE_HEIGHT, box[1] * IMAGE_WIDTH, box[2] * IMAGE_HEIGHT, box[3] * IMAGE_WIDTH, color='red', thickness=1, use_normalized_coordinates=False) for box in detection_boxes: vis_util.draw_bounding_box_on_image( im, box[0] * IMAGE_HEIGHT, box[1] * IMAGE_WIDTH, box[2] * IMAGE_HEIGHT, box[3] * IMAGE_WIDTH, color='blue', thickness=1, use_normalized_coordinates=False) # UNCOMMENT TO DISPLAY IMAGES W/ BoundingBox #plt.imshow(np.asarray(im)) #plt.show() # Code to create directory & save images w/ bounding boxes filename = decoded_dict['key'] im.save(detectionDirName + "/" + filename) num_shown += 1 matches = [] if image_index % 100 == 0: print("Processed %d images" % (image_index)) for i in range(len(groundtruth_boxes)): for j in range(len(detection_boxes)): iou = compute_iou(groundtruth_boxes[i], detection_boxes[j]) if iou > IOU_THRESHOLD: matches.append([i, j, iou]) matches = np.array(matches) if matches.shape[0] > 0: # Sort list of matches by descending IOU so we can remove duplicate detections # while keeping the highest IOU entry. matches = matches[matches[:, 2].argsort()[::-1][:len(matches)]] # Remove duplicate detections from the list. matches = matches[np.unique(matches[:, 1], return_index=True)[1]] # Sort the list again by descending IOU. Removing duplicates doesn't preserve # our previous sort. matches = matches[matches[:, 2].argsort()[::-1][:len(matches)]] # Remove duplicate ground truths from the list. matches = matches[np.unique(matches[:, 0], return_index=True)[1]] for i in range(len(groundtruth_boxes)): if matches.shape[0] > 0 and matches[matches[:, 0] == i].shape[0] == 1: confusion_matrix[groundtruth_classes[i] - 1][ detection_classes[int(matches[matches[:, 0] == i, 1][0])] - 1] += 1 else: confusion_matrix[groundtruth_classes[i] - 1][confusion_matrix.shape[1] - 1] += 1 for i in range(len(detection_boxes)): if matches.shape[0] > 0 and matches[matches[:, 1] == i].shape[0] == 0: confusion_matrix[confusion_matrix.shape[0] - 1][detection_classes[i] - 1] += 1 else: print("Skipped image %d" % (image_index)) print("Processed %d images" % (image_index)) return confusion_matrix
def read_data_and_evaluate(input_config, eval_config, score_threshold=0): """Reads pre-computed object detections and groundtruth from tf_record. Args: input_config: input config proto of type object_detection.protos.InputReader. eval_config: evaluation config proto of type object_detection.protos.EvalConfig. Returns: Evaluated detections metrics. Format: filename: metrics Raises: ValueError: if input_reader type is not supported or metric type is unknown. """ if input_config.WhichOneof('input_reader') == 'tf_record_input_reader': input_paths = input_config.tf_record_input_reader.input_path label_map = label_map_util.load_labelmap(input_config.label_map_path) max_num_classes = max([item.id for item in label_map.item]) categories = label_map_util.convert_label_map_to_categories( label_map, max_num_classes) ''' filename: metric list ''' metrics_image_map = {} # Setup metrics map metrics_image_map['filename'] = [] metrics_image_map['is_atleast_one_mobile_tower_detected'] = [] metrics_image_map['result_scores_class_1'] = [] metrics_image_map['result_tp_fp_labels_class_1'] = [] metrics_image_map['recall'] = [] metrics_image_map['precision (greater than threshold)'] = [] skipped_images = 0 processed_images = 0 per_image_eval = per_image_evaluation.PerImageEvaluation( 1, matching_iou_threshold=0.2) for input_path in _generate_filenames(input_paths): tf.logging.info('Processing file: {0}'.format(input_path)) record_iterator = tf.python_io.tf_record_iterator(path=input_path) data_parser = tf_example_parser.TfExampleDetectionAndGTParser() for string_record in record_iterator: # Image metric set tf.logging.log_every_n(tf.logging.INFO, 'Processed %d images...', 100, processed_images) processed_images += 1 example = tf.train.Example() example.ParseFromString(string_record) decoded_dict = data_parser.parse(example) if decoded_dict: detected_boxes = decoded_dict[ standard_fields.DetectionResultFields.detection_boxes] detected_scores = decoded_dict[ standard_fields.DetectionResultFields.detection_scores] detected_class_labels = decoded_dict[ standard_fields.DetectionResultFields. detection_classes] - 1 groundtruth_boxes = decoded_dict[ standard_fields.InputDataFields.groundtruth_boxes] groundtruth_class_labels = decoded_dict[ standard_fields.InputDataFields. groundtruth_classes] - 1 groundtruth_is_difficult_list = decoded_dict[ standard_fields.InputDataFields.groundtruth_difficult] groundtruth_is_group_of_list = np.zeros( len(groundtruth_boxes), dtype=np.int64) != 0 scores, tp_fp_labels, is_class_correctly_detected_in_image = \ per_image_eval.compute_object_detection_metrics(detected_boxes=detected_boxes,\ detected_scores=detected_scores,\ detected_class_labels=detected_class_labels,\ groundtruth_boxes=groundtruth_boxes, \ groundtruth_class_labels=groundtruth_class_labels,\ groundtruth_is_difficult_list=groundtruth_is_difficult_list,\ groundtruth_is_group_of_list=groundtruth_is_group_of_list) ''' Returns: scores: A list of C float numpy arrays. Each numpy array is of shape [K, 1], representing K scores detected with object class label c tp_fp_labels: A list of C boolean numpy arrays. Each numpy array is of shape [K, 1], representing K True/False positive label of object instances detected with class label c is_class_correctly_detected_in_image: a numpy integer array of shape [C, 1], indicating whether the correponding class has a least one instance being correctly detected in the image ''' indices = np.where(scores[0] > score_threshold) metrics_image_map['filename'].append(decoded_dict[ standard_fields.DetectionResultFields.key]) metrics_image_map[ 'is_atleast_one_mobile_tower_detected'].append( is_class_correctly_detected_in_image[0]) metrics_image_map['result_scores_class_1'].append( (" - ").join([str(i) for i in scores[0]])) metrics_image_map['result_tp_fp_labels_class_1'].append( (" - ").join([str(i) for i in tp_fp_labels[0]])) metrics_image_map['recall'].append( np.sum(tp_fp_labels[0]) / len(groundtruth_class_labels)) metrics_image_map[ 'precision (greater than threshold)'].append( np.sum(tp_fp_labels[0][indices]) / len(tp_fp_labels[0][indices])) else: skipped_images += 1 tf.logging.info( 'Skipped images: {0}'.format(skipped_images)) return metrics_image_map raise ValueError('Unsupported input_reader_config.')
def read_data_and_evaluate(input_config, eval_config): """Reads pre-computed object detections and groundtruth from tf_record. Args: input_config: input config proto of type object_detection.protos.InputReader. eval_config: evaluation config proto of type object_detection.protos.EvalConfig. Returns: Evaluated detections metrics. Raises: ValueError: if input_reader type is not supported or metric type is unknown. """ if input_config.WhichOneof('input_reader') == 'tf_record_input_reader': input_paths = input_config.tf_record_input_reader.input_path label_map = label_map_util.load_labelmap(input_config.label_map_path) max_num_classes = max([item.id for item in label_map.item]) categories = label_map_util.convert_label_map_to_categories( label_map, max_num_classes) object_detection_evaluators = evaluator.get_evaluators( eval_config, categories) metrics = {} # Now Supports multiple evaluators XD for object_detection_evaluator in object_detection_evaluators: # Support a single evaluator #object_detection_evaluator = object_detection_evaluators[0] print(object_detection_evaluator) skipped_images = 0 processed_images = 0 for input_path in _generate_filenames(input_paths): tf.logging.info('Processing file: {0}'.format(input_path)) record_iterator = tf.python_io.tf_record_iterator( path=input_path) data_parser = tf_example_parser.TfExampleDetectionAndGTParser() for string_record in record_iterator: #print(type(string_record)) #string_record = unicode(string_record, errors='ignore') #string_record=string_record.decode("utf-8","ignore") tf.logging.log_every_n(tf.logging.INFO, 'Processed %d images...', 1000, processed_images) processed_images += 1 example = tf.train.Example() example.ParseFromString(string_record) decoded_dict = data_parser.parse(example) print(decoded_dict) if decoded_dict: object_detection_evaluator.add_single_ground_truth_image_info(\ decoded_dict[standard_fields.DetectionResultFields.key],decoded_dict) object_detection_evaluator.add_single_detected_image_info( decoded_dict[ standard_fields.DetectionResultFields.key], decoded_dict) else: skipped_images += 1 tf.logging.info( 'Skipped images: {0}'.format(skipped_images)) print(type(object_detection_evaluator.evaluate())) metrics.update(object_detection_evaluator.evaluate()) print(metrics) return metrics raise ValueError('Unsupported input_reader_config.')
def get_dataframe(detections_record, categories): record_iterator = tf.python_io.tf_record_iterator(path=detections_record) data_parser = tf_example_parser.TfExampleDetectionAndGTParser() confusion_matrix = np.zeros(shape=(len(categories) + 1, len(categories) + 1)) df = pd.DataFrame(columns=('iou', 'score', 'class', 'filename')) image_index = 0 for string_record in record_iterator: example = tf.train.Example() example.ParseFromString(string_record) decoded_dict = data_parser.parse(example) image_index += 1 if decoded_dict: groundtruth_boxes = decoded_dict[ standard_fields.InputDataFields.groundtruth_boxes] groundtruth_classes = decoded_dict[ standard_fields.InputDataFields.groundtruth_classes] detection_scores = decoded_dict[ standard_fields.DetectionResultFields.detection_scores] detection_classes = decoded_dict[ standard_fields.DetectionResultFields.detection_classes][ detection_scores >= CONFIDENCE_THRESHOLD] detection_boxes = decoded_dict[ standard_fields.DetectionResultFields.detection_boxes][ detection_scores >= CONFIDENCE_THRESHOLD] detection_filename = decoded_dict[ standard_fields.DetectionResultFields.key].decode("utf-8") matches = [] if image_index % 100 == 0: print("Processed %d images" % (image_index)) for i in range(len(groundtruth_boxes)): for j in range(len(detection_boxes)): iou = compute_iou(groundtruth_boxes[i], detection_boxes[j]) if iou > IOU_THRESHOLD: matches.append([i, j, iou]) matches = np.array(matches) if matches.shape[0] > 0: # Sort list of matches by descending IOU so we can remove duplicate detections # while keeping the highest IOU entry. matches = matches[matches[:, 2].argsort()[::-1][:len(matches)]] # Remove duplicate detections from the list. matches = matches[np.unique(matches[:, 1], return_index=True)[1]] # Sort the list again by descending IOU. Removing duplicates doesn't preserve # our previous sort. matches = matches[matches[:, 2].argsort()[::-1][:len(matches)]] # Remove duplicate ground truths from the list. matches = matches[np.unique(matches[:, 0], return_index=True)[1]] #para el primer elemento de matches if len(matches) > 0: iou = matches[0][2] ground_class = groundtruth_classes[int(matches[0][0])] detection_class = detection_classes[int(matches[0][1])] detection_score = detection_scores[int(matches[0][1])] if ground_class == detection_class: df.loc[len(df)] = [ iou, detection_score, detection_class, detection_filename ] return df