def main(parsed_args):
  dir_exp = os.path.dirname(parsed_args.output_metrics)
  name = os.path.basename(parsed_args.output_metrics).split('.json')[0]
  Logger(dir_exp, name=name) # create/load logs

  Logger()('Begin evaluation')

  all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes)
  all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels)
  all_annotations = pd.concat([all_box_annotations, all_label_annotations], sort=True)

  class_label_map = _load_labelmap(parsed_args.input_class_labelmap)
  relationship_label_map = _load_labelmap(
      parsed_args.input_relationship_labelmap)

  relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator()
  phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator()

  for _, groundtruth in enumerate(all_annotations.groupby('ImageID')):
    image_id, image_groundtruth = groundtruth
    groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
        image_groundtruth, class_label_map, relationship_label_map)

    relation_evaluator.add_single_ground_truth_image_info(
        image_id, groundtruth_dictionary)
    phrase_evaluator.add_single_ground_truth_image_info(image_id,
                                                        groundtruth_dictionary)

  all_predictions = pd.read_csv(parsed_args.input_predictions)
  for _, prediction_data in enumerate(all_predictions.groupby('ImageID')):
    image_id, image_predictions = prediction_data
    prediction_dictionary = utils.build_predictions_vrd_dictionary(
        image_predictions, class_label_map, relationship_label_map)

    relation_evaluator.add_single_detected_image_info(image_id,
                                                      prediction_dictionary)
    phrase_evaluator.add_single_detected_image_info(image_id,
                                                    prediction_dictionary)

  relation_metrics = relation_evaluator.evaluate(
      relationships=_swap_labelmap_dict(relationship_label_map))
  phrase_metrics = phrase_evaluator.evaluate(
      relationships=_swap_labelmap_dict(relationship_label_map))

  for k,v in relation_metrics.items():
    Logger().log_value('eval_epoch.{}'.format(k), v)

  for k,v in phrase_metrics.items():
    Logger().log_value('eval_epoch.{}'.format(k), v)

  score = relation_metrics['[email protected]'] * 0.4
  score += relation_metrics['VRDMetric_Relationships_Recall@[email protected]'] * 0.2
  score += phrase_metrics['[email protected]'] * 0.4
  Logger().log_value('eval_epoch.score', score)
  epoch_id = len(Logger().values['eval_epoch.score']) - 1
  Logger().log_value('eval_epoch.epoch', epoch_id)

  Logger().flush()

  Logger()('End evaluation')
def main(parsed_args):
    all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes)
    all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels)
    all_annotations = pd.concat([all_box_annotations, all_label_annotations])

    class_label_map = _load_labelmap(parsed_args.input_class_labelmap)
    relationship_label_map = _load_labelmap(
        parsed_args.input_relationship_labelmap)

    relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator()
    phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator()

    for _, groundtruth in enumerate(all_annotations.groupby('ImageID')):
        image_id, image_groundtruth = groundtruth
        groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
            image_groundtruth, class_label_map, relationship_label_map)

        relation_evaluator.add_single_ground_truth_image_info(
            image_id, groundtruth_dictionary)
        phrase_evaluator.add_single_ground_truth_image_info(
            image_id, groundtruth_dictionary)

    all_predictions = pd.read_csv(parsed_args.input_predictions)
    for _, prediction_data in enumerate(all_predictions.groupby('ImageID')):
        image_id, image_predictions = prediction_data
        prediction_dictionary = utils.build_predictions_vrd_dictionary(
            image_predictions, class_label_map, relationship_label_map)

        relation_evaluator.add_single_detected_image_info(
            image_id, prediction_dictionary)
        phrase_evaluator.add_single_detected_image_info(
            image_id, prediction_dictionary)

    relation_metrics = relation_evaluator.evaluate(
        relationships=_swap_labelmap_dict(relationship_label_map))
    phrase_metrics = phrase_evaluator.evaluate(
        relationships=_swap_labelmap_dict(relationship_label_map))

    with open(parsed_args.output_metrics, 'w') as fid:
        io_utils.write_csv(fid, relation_metrics)
        io_utils.write_csv(fid, phrase_metrics)
  def test_vrdrelation_evaluator(self):
    self.vrd_eval = vrd_evaluation.VRDRelationDetectionEvaluator()

    image_key1 = 'img1'
    groundtruth_box_tuples1 = np.array(
        [([0, 0, 1, 1], [1, 1, 2, 2]), ([0, 0, 1, 1], [1, 2, 2, 3])],
        dtype=vrd_evaluation.vrd_box_data_type)
    groundtruth_class_tuples1 = np.array(
        [(1, 2, 3), (1, 4, 3)], dtype=vrd_evaluation.label_data_type)
    groundtruth_verified_labels1 = np.array([1, 2, 3, 4, 5], dtype=int)
    self.vrd_eval.add_single_ground_truth_image_info(
        image_key1, {
            standard_fields.InputDataFields.groundtruth_boxes:
                groundtruth_box_tuples1,
            standard_fields.InputDataFields.groundtruth_classes:
                groundtruth_class_tuples1,
            standard_fields.InputDataFields.groundtruth_image_classes:
                groundtruth_verified_labels1
        })

    image_key2 = 'img2'
    groundtruth_box_tuples2 = np.array(
        [([0, 0, 1, 1], [1, 1, 2, 2])], dtype=vrd_evaluation.vrd_box_data_type)
    groundtruth_class_tuples2 = np.array(
        [(1, 4, 3)], dtype=vrd_evaluation.label_data_type)
    self.vrd_eval.add_single_ground_truth_image_info(
        image_key2, {
            standard_fields.InputDataFields.groundtruth_boxes:
                groundtruth_box_tuples2,
            standard_fields.InputDataFields.groundtruth_classes:
                groundtruth_class_tuples2,
        })

    image_key3 = 'img3'
    groundtruth_box_tuples3 = np.array(
        [([0, 0, 1, 1], [1, 1, 2, 2])], dtype=vrd_evaluation.vrd_box_data_type)
    groundtruth_class_tuples3 = np.array(
        [(1, 2, 4)], dtype=vrd_evaluation.label_data_type)
    self.vrd_eval.add_single_ground_truth_image_info(
        image_key3, {
            standard_fields.InputDataFields.groundtruth_boxes:
                groundtruth_box_tuples3,
            standard_fields.InputDataFields.groundtruth_classes:
                groundtruth_class_tuples3,
        })

    image_key = 'img1'
    detected_box_tuples = np.array(
        [([0, 0.3, 1, 1], [1.1, 1, 2, 2]), ([0, 0, 1, 1], [1, 1, 2, 2]),
         ([0.5, 0, 1, 1], [1, 1, 3, 3])],
        dtype=vrd_evaluation.vrd_box_data_type)
    detected_class_tuples = np.array(
        [(1, 2, 5), (1, 2, 3), (1, 6, 3)], dtype=vrd_evaluation.label_data_type)
    detected_scores = np.array([0.7, 0.8, 0.9], dtype=float)
    self.vrd_eval.add_single_detected_image_info(
        image_key, {
            standard_fields.DetectionResultFields.detection_boxes:
                detected_box_tuples,
            standard_fields.DetectionResultFields.detection_scores:
                detected_scores,
            standard_fields.DetectionResultFields.detection_classes:
                detected_class_tuples
        })

    metrics = self.vrd_eval.evaluate()

    self.assertAlmostEqual(metrics['[email protected]'],
                           0.25)
    self.assertAlmostEqual(metrics['[email protected]'],
                           0.1666666666666666)
    self.assertAlmostEqual(metrics['[email protected]/3'],
                           0.3333333333333333)
    self.assertAlmostEqual(metrics['[email protected]/4'], 0)
    self.assertAlmostEqual(metrics['VRDMetric_Relationships_Recall@[email protected]'],
                           0.25)
    self.assertAlmostEqual(metrics['VRDMetric_Relationships_Recall@[email protected]'],
                           0.25)
    self.vrd_eval.clear()
    self.assertFalse(self.vrd_eval._image_ids)