def main(parsed_args):
dir_exp = os.path.dirname(parsed_args.output_metrics)
name = os.path.basename(parsed_args.output_metrics).split('.json')[0]
Logger(dir_exp, name=name) # create/load logs
Logger()('Begin evaluation')
all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes)
all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels)
all_annotations = pd.concat([all_box_annotations, all_label_annotations], sort=True)
class_label_map = _load_labelmap(parsed_args.input_class_labelmap)
relationship_label_map = _load_labelmap(
parsed_args.input_relationship_labelmap)
relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator()
phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator()
for _, groundtruth in enumerate(all_annotations.groupby('ImageID')):
image_id, image_groundtruth = groundtruth
groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
image_groundtruth, class_label_map, relationship_label_map)
relation_evaluator.add_single_ground_truth_image_info(
image_id, groundtruth_dictionary)
phrase_evaluator.add_single_ground_truth_image_info(image_id,
groundtruth_dictionary)
all_predictions = pd.read_csv(parsed_args.input_predictions)
for _, prediction_data in enumerate(all_predictions.groupby('ImageID')):
image_id, image_predictions = prediction_data
prediction_dictionary = utils.build_predictions_vrd_dictionary(
image_predictions, class_label_map, relationship_label_map)
relation_evaluator.add_single_detected_image_info(image_id,
prediction_dictionary)
phrase_evaluator.add_single_detected_image_info(image_id,
prediction_dictionary)
relation_metrics = relation_evaluator.evaluate(
relationships=_swap_labelmap_dict(relationship_label_map))
phrase_metrics = phrase_evaluator.evaluate(
relationships=_swap_labelmap_dict(relationship_label_map))
for k,v in relation_metrics.items():
Logger().log_value('eval_epoch.{}'.format(k), v)
for k,v in phrase_metrics.items():
Logger().log_value('eval_epoch.{}'.format(k), v)
score = relation_metrics['[email protected]'] * 0.4
score += relation_metrics['VRDMetric_Relationships_Recall@[email protected]'] * 0.2
score += phrase_metrics['[email protected]'] * 0.4
Logger().log_value('eval_epoch.score', score)
epoch_id = len(Logger().values['eval_epoch.score']) - 1
Logger().log_value('eval_epoch.epoch', epoch_id)
Logger().flush()
Logger()('End evaluation')
def main(parsed_args):
all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes)
all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels)
all_annotations = pd.concat([all_box_annotations, all_label_annotations])
class_label_map = _load_labelmap(parsed_args.input_class_labelmap)
relationship_label_map = _load_labelmap(
parsed_args.input_relationship_labelmap)
relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator()
phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator()
for _, groundtruth in enumerate(all_annotations.groupby('ImageID')):
image_id, image_groundtruth = groundtruth
groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
image_groundtruth, class_label_map, relationship_label_map)
relation_evaluator.add_single_ground_truth_image_info(
image_id, groundtruth_dictionary)
phrase_evaluator.add_single_ground_truth_image_info(
image_id, groundtruth_dictionary)
all_predictions = pd.read_csv(parsed_args.input_predictions)
for _, prediction_data in enumerate(all_predictions.groupby('ImageID')):
image_id, image_predictions = prediction_data
prediction_dictionary = utils.build_predictions_vrd_dictionary(
image_predictions, class_label_map, relationship_label_map)
relation_evaluator.add_single_detected_image_info(
image_id, prediction_dictionary)
phrase_evaluator.add_single_detected_image_info(
image_id, prediction_dictionary)
relation_metrics = relation_evaluator.evaluate(
relationships=_swap_labelmap_dict(relationship_label_map))
phrase_metrics = phrase_evaluator.evaluate(
relationships=_swap_labelmap_dict(relationship_label_map))
with open(parsed_args.output_metrics, 'w') as fid:
io_utils.write_csv(fid, relation_metrics)
io_utils.write_csv(fid, phrase_metrics)
def test_vrdrelation_evaluator(self):
self.vrd_eval = vrd_evaluation.VRDRelationDetectionEvaluator()
image_key1 = 'img1'
groundtruth_box_tuples1 = np.array(
[([0, 0, 1, 1], [1, 1, 2, 2]), ([0, 0, 1, 1], [1, 2, 2, 3])],
dtype=vrd_evaluation.vrd_box_data_type)
groundtruth_class_tuples1 = np.array(
[(1, 2, 3), (1, 4, 3)], dtype=vrd_evaluation.label_data_type)
groundtruth_verified_labels1 = np.array([1, 2, 3, 4, 5], dtype=int)
self.vrd_eval.add_single_ground_truth_image_info(
image_key1, {
standard_fields.InputDataFields.groundtruth_boxes:
groundtruth_box_tuples1,
standard_fields.InputDataFields.groundtruth_classes:
groundtruth_class_tuples1,
standard_fields.InputDataFields.groundtruth_image_classes:
groundtruth_verified_labels1
})
image_key2 = 'img2'
groundtruth_box_tuples2 = np.array(
[([0, 0, 1, 1], [1, 1, 2, 2])], dtype=vrd_evaluation.vrd_box_data_type)
groundtruth_class_tuples2 = np.array(
[(1, 4, 3)], dtype=vrd_evaluation.label_data_type)
self.vrd_eval.add_single_ground_truth_image_info(
image_key2, {
standard_fields.InputDataFields.groundtruth_boxes:
groundtruth_box_tuples2,
standard_fields.InputDataFields.groundtruth_classes:
groundtruth_class_tuples2,
})
image_key3 = 'img3'
groundtruth_box_tuples3 = np.array(
[([0, 0, 1, 1], [1, 1, 2, 2])], dtype=vrd_evaluation.vrd_box_data_type)
groundtruth_class_tuples3 = np.array(
[(1, 2, 4)], dtype=vrd_evaluation.label_data_type)
self.vrd_eval.add_single_ground_truth_image_info(
image_key3, {
standard_fields.InputDataFields.groundtruth_boxes:
groundtruth_box_tuples3,
standard_fields.InputDataFields.groundtruth_classes:
groundtruth_class_tuples3,
})
image_key = 'img1'
detected_box_tuples = np.array(
[([0, 0.3, 1, 1], [1.1, 1, 2, 2]), ([0, 0, 1, 1], [1, 1, 2, 2]),
([0.5, 0, 1, 1], [1, 1, 3, 3])],
dtype=vrd_evaluation.vrd_box_data_type)
detected_class_tuples = np.array(
[(1, 2, 5), (1, 2, 3), (1, 6, 3)], dtype=vrd_evaluation.label_data_type)
detected_scores = np.array([0.7, 0.8, 0.9], dtype=float)
self.vrd_eval.add_single_detected_image_info(
image_key, {
standard_fields.DetectionResultFields.detection_boxes:
detected_box_tuples,
standard_fields.DetectionResultFields.detection_scores:
detected_scores,
standard_fields.DetectionResultFields.detection_classes:
detected_class_tuples
})
metrics = self.vrd_eval.evaluate()
self.assertAlmostEqual(metrics['[email protected]'],
0.25)
self.assertAlmostEqual(metrics['[email protected]'],
0.1666666666666666)
self.assertAlmostEqual(metrics['[email protected]/3'],
0.3333333333333333)
self.assertAlmostEqual(metrics['[email protected]/4'], 0)
self.assertAlmostEqual(metrics['VRDMetric_Relationships_Recall@[email protected]'],
0.25)
self.assertAlmostEqual(metrics['VRDMetric_Relationships_Recall@[email protected]'],
0.25)
self.vrd_eval.clear()
self.assertFalse(self.vrd_eval._image_ids)