def main(parsed_args):
dir_exp = os.path.dirname(parsed_args.output_metrics)
name = os.path.basename(parsed_args.output_metrics).split('.json')[0]
Logger(dir_exp, name=name) # create/load logs
Logger()('Begin evaluation')
all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes)
all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels)
all_annotations = pd.concat([all_box_annotations, all_label_annotations], sort=True)
class_label_map = _load_labelmap(parsed_args.input_class_labelmap)
relationship_label_map = _load_labelmap(
parsed_args.input_relationship_labelmap)
relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator()
phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator()
for _, groundtruth in enumerate(all_annotations.groupby('ImageID')):
image_id, image_groundtruth = groundtruth
groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
image_groundtruth, class_label_map, relationship_label_map)
relation_evaluator.add_single_ground_truth_image_info(
image_id, groundtruth_dictionary)
phrase_evaluator.add_single_ground_truth_image_info(image_id,
groundtruth_dictionary)
all_predictions = pd.read_csv(parsed_args.input_predictions)
for _, prediction_data in enumerate(all_predictions.groupby('ImageID')):
image_id, image_predictions = prediction_data
prediction_dictionary = utils.build_predictions_vrd_dictionary(
image_predictions, class_label_map, relationship_label_map)
relation_evaluator.add_single_detected_image_info(image_id,
prediction_dictionary)
phrase_evaluator.add_single_detected_image_info(image_id,
prediction_dictionary)
relation_metrics = relation_evaluator.evaluate(
relationships=_swap_labelmap_dict(relationship_label_map))
phrase_metrics = phrase_evaluator.evaluate(
relationships=_swap_labelmap_dict(relationship_label_map))
for k,v in relation_metrics.items():
Logger().log_value('eval_epoch.{}'.format(k), v)
for k,v in phrase_metrics.items():
Logger().log_value('eval_epoch.{}'.format(k), v)
score = relation_metrics['[email protected]'] * 0.4
score += relation_metrics['VRDMetric_Relationships_Recall@[email protected]'] * 0.2
score += phrase_metrics['[email protected]'] * 0.4
Logger().log_value('eval_epoch.score', score)
epoch_id = len(Logger().values['eval_epoch.score']) - 1
Logger().log_value('eval_epoch.epoch', epoch_id)
Logger().flush()
Logger()('End evaluation')
def main(parsed_args):
all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes)
all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels)
all_annotations = pd.concat([all_box_annotations, all_label_annotations])
class_label_map = _load_labelmap(parsed_args.input_class_labelmap)
relationship_label_map = _load_labelmap(
parsed_args.input_relationship_labelmap)
relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator()
phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator()
for _, groundtruth in enumerate(all_annotations.groupby('ImageID')):
image_id, image_groundtruth = groundtruth
groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
image_groundtruth, class_label_map, relationship_label_map)
relation_evaluator.add_single_ground_truth_image_info(
image_id, groundtruth_dictionary)
phrase_evaluator.add_single_ground_truth_image_info(
image_id, groundtruth_dictionary)
all_predictions = pd.read_csv(parsed_args.input_predictions)
for _, prediction_data in enumerate(all_predictions.groupby('ImageID')):
image_id, image_predictions = prediction_data
prediction_dictionary = utils.build_predictions_vrd_dictionary(
image_predictions, class_label_map, relationship_label_map)
relation_evaluator.add_single_detected_image_info(
image_id, prediction_dictionary)
phrase_evaluator.add_single_detected_image_info(
image_id, prediction_dictionary)
relation_metrics = relation_evaluator.evaluate(
relationships=_swap_labelmap_dict(relationship_label_map))
phrase_metrics = phrase_evaluator.evaluate(
relationships=_swap_labelmap_dict(relationship_label_map))
with open(parsed_args.output_metrics, 'w') as fid:
io_utils.write_csv(fid, relation_metrics)
io_utils.write_csv(fid, phrase_metrics)
def main(parsed_args):
all_box_annotations = pd.read_csv(parsed_args.input_annotations_boxes)
all_label_annotations = pd.read_csv(parsed_args.input_annotations_labels)
all_annotations = pd.concat([all_box_annotations, all_label_annotations])
class_label_map = _load_labelmap(parsed_args.input_class_labelmap)
relationship_label_map = _load_labelmap(
parsed_args.input_relationship_labelmap)
relation_evaluator = vrd_evaluation.VRDRelationDetectionEvaluator()
phrase_evaluator = vrd_evaluation.VRDPhraseDetectionEvaluator()
for _, groundtruth in enumerate(all_annotations.groupby('ImageID')):
image_id, image_groundtruth = groundtruth
groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
image_groundtruth, class_label_map, relationship_label_map)
relation_evaluator.add_single_ground_truth_image_info(
image_id, groundtruth_dictionary)
phrase_evaluator.add_single_ground_truth_image_info(image_id,
groundtruth_dictionary)
all_predictions = pd.read_csv(parsed_args.input_predictions)
for _, prediction_data in enumerate(all_predictions.groupby('ImageID')):
image_id, image_predictions = prediction_data
prediction_dictionary = utils.build_predictions_vrd_dictionary(
image_predictions, class_label_map, relationship_label_map)
relation_evaluator.add_single_detected_image_info(image_id,
prediction_dictionary)
phrase_evaluator.add_single_detected_image_info(image_id,
prediction_dictionary)
relation_metrics = relation_evaluator.evaluate()
phrase_metrics = phrase_evaluator.evaluate()
with open(parsed_args.output_metrics, 'w') as fid:
utils.write_csv(fid, relation_metrics)
utils.write_csv(fid, phrase_metrics)
def testBuildGroundtruthDictionary(self):
np_data = pd.DataFrame(
[[
'fe58ec1b06db2bb7', '/m/04bcr3', '/m/083vt', 0.0, 0.3, 0.5,
0.6, 0.0, 0.3, 0.5, 0.6, 'is', None, None
],
[
'fe58ec1b06db2bb7', '/m/04bcr3', '/m/02gy9n', 0.0, 0.3, 0.5,
0.6, 0.1, 0.2, 0.3, 0.4, 'under', None, None
],
[
'fe58ec1b06db2bb7', '/m/04bcr3', '/m/083vt', 0.0, 0.1, 0.2,
0.3, 0.0, 0.1, 0.2, 0.3, 'is', None, None
],
[
'fe58ec1b06db2bb7', '/m/083vt', '/m/04bcr3', 0.1, 0.2, 0.3,
0.4, 0.5, 0.6, 0.7, 0.8, 'at', None, None
],
[
'fe58ec1b06db2bb7', None, None, None, None, None, None, None,
None, None, None, None, '/m/04bcr3', 1.0
],
[
'fe58ec1b06db2bb7', None, None, None, None, None, None, None,
None, None, None, None, '/m/083vt', 0.0
],
[
'fe58ec1b06db2bb7', None, None, None, None, None, None, None,
None, None, None, None, '/m/02gy9n', 0.0
]],
columns=[
'ImageID', 'LabelName1', 'LabelName2', 'XMin1', 'XMax1',
'YMin1', 'YMax1', 'XMin2', 'XMax2', 'YMin2', 'YMax2',
'RelationshipLabel', 'LabelName', 'Confidence'
])
class_label_map = {'/m/04bcr3': 1, '/m/083vt': 2, '/m/02gy9n': 3}
relationship_label_map = {'is': 1, 'under': 2, 'at': 3}
groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
np_data, class_label_map, relationship_label_map)
self.assertTrue(standard_fields.InputDataFields.groundtruth_boxes in
groundtruth_dictionary)
self.assertTrue(standard_fields.InputDataFields.groundtruth_classes in
groundtruth_dictionary)
self.assertTrue(standard_fields.InputDataFields.
groundtruth_image_classes in groundtruth_dictionary)
self.assertAllEqual(
np.array([(1, 2, 1), (1, 3, 2), (1, 2, 1), (2, 1, 3)],
dtype=vrd_evaluation.label_data_type),
groundtruth_dictionary[
standard_fields.InputDataFields.groundtruth_classes])
expected_vrd_data = np.array([
([0.5, 0.0, 0.6, 0.3], [0.5, 0.0, 0.6, 0.3]),
([0.5, 0.0, 0.6, 0.3], [0.3, 0.1, 0.4, 0.2]),
([0.2, 0.0, 0.3, 0.1], [0.2, 0.0, 0.3, 0.1]),
([0.3, 0.1, 0.4, 0.2], [0.7, 0.5, 0.8, 0.6]),
],
dtype=vrd_evaluation.vrd_box_data_type)
for field in expected_vrd_data.dtype.fields:
self.assertNDArrayNear(
expected_vrd_data[field], groundtruth_dictionary[
standard_fields.InputDataFields.groundtruth_boxes][field],
1e-5)
self.assertAllEqual(
np.array([1, 2, 3]), groundtruth_dictionary[
standard_fields.InputDataFields.groundtruth_image_classes])
def testBuildGroundtruthDictionary(self):
np_data = pd.DataFrame(
[[
'fe58ec1b06db2bb7', '/m/04bcr3', '/m/083vt', 0.0, 0.3, 0.5, 0.6,
0.0, 0.3, 0.5, 0.6, 'is', None, None
], [
'fe58ec1b06db2bb7', '/m/04bcr3', '/m/02gy9n', 0.0, 0.3, 0.5, 0.6,
0.1, 0.2, 0.3, 0.4, 'under', None, None
], [
'fe58ec1b06db2bb7', '/m/04bcr3', '/m/083vt', 0.0, 0.1, 0.2, 0.3,
0.0, 0.1, 0.2, 0.3, 'is', None, None
], [
'fe58ec1b06db2bb7', '/m/083vt', '/m/04bcr3', 0.1, 0.2, 0.3, 0.4,
0.5, 0.6, 0.7, 0.8, 'at', None, None
], [
'fe58ec1b06db2bb7', None, None, None, None, None, None, None, None,
None, None, None, '/m/04bcr3', 1.0
], [
'fe58ec1b06db2bb7', None, None, None, None, None, None, None, None,
None, None, None, '/m/083vt', 0.0
], [
'fe58ec1b06db2bb7', None, None, None, None, None, None, None, None,
None, None, None, '/m/02gy9n', 0.0
]],
columns=[
'ImageID', 'LabelName1', 'LabelName2', 'XMin1', 'XMax1', 'YMin1',
'YMax1', 'XMin2', 'XMax2', 'YMin2', 'YMax2', 'RelationshipLabel',
'LabelName', 'Confidence'
])
class_label_map = {'/m/04bcr3': 1, '/m/083vt': 2, '/m/02gy9n': 3}
relationship_label_map = {'is': 1, 'under': 2, 'at': 3}
groundtruth_dictionary = utils.build_groundtruth_vrd_dictionary(
np_data, class_label_map, relationship_label_map)
self.assertTrue(standard_fields.InputDataFields.groundtruth_boxes in
groundtruth_dictionary)
self.assertTrue(standard_fields.InputDataFields.groundtruth_classes in
groundtruth_dictionary)
self.assertTrue(standard_fields.InputDataFields.verified_labels in
groundtruth_dictionary)
self.assertAllEqual(
np.array(
[(1, 2, 1), (1, 3, 2), (1, 2, 1), (2, 1, 3)],
dtype=vrd_evaluation.label_data_type), groundtruth_dictionary[
standard_fields.InputDataFields.groundtruth_classes])
expected_vrd_data = np.array(
[
([0.5, 0.0, 0.6, 0.3], [0.5, 0.0, 0.6, 0.3]),
([0.5, 0.0, 0.6, 0.3], [0.3, 0.1, 0.4, 0.2]),
([0.2, 0.0, 0.3, 0.1], [0.2, 0.0, 0.3, 0.1]),
([0.3, 0.1, 0.4, 0.2], [0.7, 0.5, 0.8, 0.6]),
],
dtype=vrd_evaluation.vrd_box_data_type)
for field in expected_vrd_data.dtype.fields:
self.assertNDArrayNear(
expected_vrd_data[field], groundtruth_dictionary[
standard_fields.InputDataFields.groundtruth_boxes][field], 1e-5)
self.assertAllEqual(
np.array([1, 2, 3]),
groundtruth_dictionary[standard_fields.InputDataFields.verified_labels])
