def main(parsed_args): dir_exp = os.path.dirname(parsed_args.output_metrics) name = os.path.basename(parsed_args.output_metrics).split('.json')[0] Logger(dir_exp, name=name) # create/load logs Logger()('Begin evaluation') all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes) all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels) all_annotations = pd.concat([all_box_annotations, all_label_annotations], sort=True) class_label_map = _load_labelmap(parsed_args.input_class_labelmap) relationship_label_map = _load_labelmap( parsed_args.input_relationship_labelmap) relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator() phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator() for _, groundtruth in enumerate(all_annotations.groupby('ImageID')): image_id, image_groundtruth = groundtruth groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary( image_groundtruth, class_label_map, relationship_label_map) relation_evaluator.add_single_ground_truth_image_info( image_id, groundtruth_dictionary) phrase_evaluator.add_single_ground_truth_image_info(image_id, groundtruth_dictionary) all_predictions = pd.read_csv(parsed_args.input_predictions) for _, prediction_data in enumerate(all_predictions.groupby('ImageID')): image_id, image_predictions = prediction_data prediction_dictionary = utils.build_predictions_vrd_dictionary( image_predictions, class_label_map, relationship_label_map) relation_evaluator.add_single_detected_image_info(image_id, prediction_dictionary) phrase_evaluator.add_single_detected_image_info(image_id, prediction_dictionary) relation_metrics = relation_evaluator.evaluate( relationships=_swap_labelmap_dict(relationship_label_map)) phrase_metrics = phrase_evaluator.evaluate( relationships=_swap_labelmap_dict(relationship_label_map)) for k,v in relation_metrics.items(): Logger().log_value('eval_epoch.{}'.format(k), v) for k,v in phrase_metrics.items(): Logger().log_value('eval_epoch.{}'.format(k), v) score = relation_metrics['[email protected]'] * 0.4 score += relation_metrics['VRDMetric_Relationships_Recall@[email protected]'] * 0.2 score += phrase_metrics['[email protected]'] * 0.4 Logger().log_value('eval_epoch.score', score) epoch_id = len(Logger().values['eval_epoch.score']) - 1 Logger().log_value('eval_epoch.epoch', epoch_id) Logger().flush() Logger()('End evaluation')
def main(parsed_args): all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes) all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels) all_annotations = pd.concat([all_box_annotations, all_label_annotations]) class_label_map = _load_labelmap(parsed_args.input_class_labelmap) relationship_label_map = _load_labelmap( parsed_args.input_relationship_labelmap) relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator() phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator() for _, groundtruth in enumerate(all_annotations.groupby('ImageID')): image_id, image_groundtruth = groundtruth groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary( image_groundtruth, class_label_map, relationship_label_map) relation_evaluator.add_single_ground_truth_image_info( image_id, groundtruth_dictionary) phrase_evaluator.add_single_ground_truth_image_info( image_id, groundtruth_dictionary) all_predictions = pd.read_csv(parsed_args.input_predictions) for _, prediction_data in enumerate(all_predictions.groupby('ImageID')): image_id, image_predictions = prediction_data prediction_dictionary = utils.build_predictions_vrd_dictionary( image_predictions, class_label_map, relationship_label_map) relation_evaluator.add_single_detected_image_info( image_id, prediction_dictionary) phrase_evaluator.add_single_detected_image_info( image_id, prediction_dictionary) relation_metrics = relation_evaluator.evaluate( relationships=_swap_labelmap_dict(relationship_label_map)) phrase_metrics = phrase_evaluator.evaluate( relationships=_swap_labelmap_dict(relationship_label_map)) with open(parsed_args.output_metrics, 'w') as fid: io_utils.write_csv(fid, relation_metrics) io_utils.write_csv(fid, phrase_metrics)
def test_vrdrelation_evaluator(self): self.vrd_eval = vrd_evaluation.VRDRelationDetectionEvaluator() image_key1 = 'img1' groundtruth_box_tuples1 = np.array( [([0, 0, 1, 1], [1, 1, 2, 2]), ([0, 0, 1, 1], [1, 2, 2, 3])], dtype=vrd_evaluation.vrd_box_data_type) groundtruth_class_tuples1 = np.array( [(1, 2, 3), (1, 4, 3)], dtype=vrd_evaluation.label_data_type) groundtruth_verified_labels1 = np.array([1, 2, 3, 4, 5], dtype=int) self.vrd_eval.add_single_ground_truth_image_info( image_key1, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_box_tuples1, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_tuples1, standard_fields.InputDataFields.groundtruth_image_classes: groundtruth_verified_labels1 }) image_key2 = 'img2' groundtruth_box_tuples2 = np.array( [([0, 0, 1, 1], [1, 1, 2, 2])], dtype=vrd_evaluation.vrd_box_data_type) groundtruth_class_tuples2 = np.array( [(1, 4, 3)], dtype=vrd_evaluation.label_data_type) self.vrd_eval.add_single_ground_truth_image_info( image_key2, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_box_tuples2, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_tuples2, }) image_key3 = 'img3' groundtruth_box_tuples3 = np.array( [([0, 0, 1, 1], [1, 1, 2, 2])], dtype=vrd_evaluation.vrd_box_data_type) groundtruth_class_tuples3 = np.array( [(1, 2, 4)], dtype=vrd_evaluation.label_data_type) self.vrd_eval.add_single_ground_truth_image_info( image_key3, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_box_tuples3, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_tuples3, }) image_key = 'img1' detected_box_tuples = np.array( [([0, 0.3, 1, 1], [1.1, 1, 2, 2]), ([0, 0, 1, 1], [1, 1, 2, 2]), ([0.5, 0, 1, 1], [1, 1, 3, 3])], dtype=vrd_evaluation.vrd_box_data_type) detected_class_tuples = np.array( [(1, 2, 5), (1, 2, 3), (1, 6, 3)], dtype=vrd_evaluation.label_data_type) detected_scores = np.array([0.7, 0.8, 0.9], dtype=float) self.vrd_eval.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: detected_box_tuples, standard_fields.DetectionResultFields.detection_scores: detected_scores, standard_fields.DetectionResultFields.detection_classes: detected_class_tuples }) metrics = self.vrd_eval.evaluate() self.assertAlmostEqual(metrics['[email protected]'], 0.25) self.assertAlmostEqual(metrics['[email protected]'], 0.1666666666666666) self.assertAlmostEqual(metrics['[email protected]/3'], 0.3333333333333333) self.assertAlmostEqual(metrics['[email protected]/4'], 0) self.assertAlmostEqual(metrics['VRDMetric_Relationships_Recall@[email protected]'], 0.25) self.assertAlmostEqual(metrics['VRDMetric_Relationships_Recall@[email protected]'], 0.25) self.vrd_eval.clear() self.assertFalse(self.vrd_eval._image_ids)