def test_evaluator_options_from_eval_config_with_super_categories(self):
eval_config_text_proto = """
metrics_set: "coco_detection_metrics"
metrics_set: "coco_mask_metrics"
include_metrics_per_category: true
use_moving_averages: false
batch_size: 1;
super_categories {
key: "supercat1"
value: "a,b,c"
}
super_categories {
key: "supercat2"
value: "d,e,f"
}
"""
eval_config = eval_pb2.EvalConfig()
text_format.Merge(eval_config_text_proto, eval_config)
evaluator_options = eval_util.evaluator_options_from_eval_config(
eval_config)
self.assertIn('super_categories', evaluator_options['coco_mask_metrics'])
super_categories = evaluator_options[
'coco_mask_metrics']['super_categories']
self.assertIn('supercat1', super_categories)
self.assertIn('supercat2', super_categories)
self.assertAllEqual(super_categories['supercat1'], ['a', 'b', 'c'])
self.assertAllEqual(super_categories['supercat2'], ['d', 'e', 'f'])
def test_get_evaluator_with_keypoint_metrics(self):
eval_config = eval_pb2.EvalConfig()
person_keypoints_metric = eval_config.parameterized_metric.add()
person_keypoints_metric.coco_keypoint_metrics.class_label = 'person'
person_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
'left_eye'] = 0.1
person_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
'right_eye'] = 0.2
dog_keypoints_metric = eval_config.parameterized_metric.add()
dog_keypoints_metric.coco_keypoint_metrics.class_label = 'dog'
dog_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
'tail_start'] = 0.3
dog_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
'mouth'] = 0.4
categories = self._get_categories_list_with_keypoints()
evaluator = eval_util.get_evaluators(eval_config,
categories,
evaluator_options=None)
# Verify keypoint evaluator class variables.
self.assertLen(evaluator, 3)
self.assertFalse(evaluator[0]._include_metrics_per_category)
self.assertEqual(evaluator[1]._category_name, 'person')
self.assertEqual(evaluator[2]._category_name, 'dog')
self.assertAllEqual(evaluator[1]._keypoint_ids, [0, 3])
self.assertAllEqual(evaluator[2]._keypoint_ids, [1, 2])
self.assertAllClose([0.1, 0.2], evaluator[1]._oks_sigmas)
self.assertAllClose([0.3, 0.4], evaluator[2]._oks_sigmas)
def test_get_eval_metric_ops_for_evaluators(self):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend([
'coco_detection_metrics', 'coco_mask_metrics',
'precision_at_recall_detection_metrics'
])
eval_config.include_metrics_per_category = True
eval_config.recall_lower_bound = 0.2
eval_config.recall_upper_bound = 0.6
evaluator_options = eval_util.evaluator_options_from_eval_config(
eval_config)
self.assertTrue(evaluator_options['coco_detection_metrics']
['include_metrics_per_category'])
self.assertFalse(evaluator_options['coco_detection_metrics']
['skip_predictions_for_unlabeled_class'])
self.assertTrue(evaluator_options['coco_mask_metrics']
['include_metrics_per_category'])
self.assertAlmostEqual(
evaluator_options['precision_at_recall_detection_metrics']
['recall_lower_bound'], eval_config.recall_lower_bound)
self.assertAlmostEqual(
evaluator_options['precision_at_recall_detection_metrics']
['recall_upper_bound'], eval_config.recall_upper_bound)
self.assertFalse(
evaluator_options['precision_at_recall_detection_metrics']
['skip_predictions_for_unlabeled_class'])
def get_configs_from_checkpoint_dir():
"""Reads evaluation configuration from checkpoint directory.
Reads the evaluation config from the following files:
model_config: Read from FLAGS.checkpoint_dir/model.config
eval_config: Read from FLAGS.checkpoint_dir/eval.config
input_config: Read from FLAGS.checkpoint_dir/input.config
Returns:
model_config: a model_pb2.DetectionModel
eval_config: a eval_pb2.EvalConfig
input_config: a input_reader_pb2.InputReader
"""
eval_config = eval_pb2.EvalConfig()
eval_config_path = os.path.join(FLAGS.checkpoint_dir, 'eval.config')
with tf.gfile.GFile(eval_config_path, 'r') as f:
text_format.Merge(f.read(), eval_config)
model_config = model_pb2.DetectionModel()
model_config_path = os.path.join(FLAGS.checkpoint_dir, 'model.config')
with tf.gfile.GFile(model_config_path, 'r') as f:
text_format.Merge(f.read(), model_config)
input_config = input_reader_pb2.InputReader()
if FLAGS.eval_training_data:
input_config_path = os.path.join(FLAGS.checkpoint_dir,
'train_input.config')
else:
input_config_path = os.path.join(FLAGS.checkpoint_dir,
'eval_input.config')
with tf.gfile.GFile(input_config_path, 'r') as f:
text_format.Merge(f.read(), input_config)
return model_config, eval_config, input_config
def test_get_eval_metric_ops_for_coco_detections_and_resized_masks(
self, batch_size=1, max_gt_boxes=None, scale_to_absolute=False):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend(
['coco_detection_metrics', 'coco_mask_metrics'])
categories = self._get_categories_list()
eval_dict = self._make_evaluation_dict(
batch_size=batch_size,
max_gt_boxes=max_gt_boxes,
scale_to_absolute=scale_to_absolute,
resized_groundtruth_masks=True)
metric_ops = eval_util.get_eval_metric_ops_for_evaluators(
eval_config, categories, eval_dict)
_, update_op_boxes = metric_ops['DetectionBoxes_Precision/mAP']
_, update_op_masks = metric_ops['DetectionMasks_Precision/mAP']
with self.test_session() as sess:
metrics = {}
for key, (value_op, _) in six.iteritems(metric_ops):
metrics[key] = value_op
sess.run(update_op_boxes)
sess.run(update_op_masks)
metrics = sess.run(metrics)
self.assertAlmostEqual(1.0,
metrics['DetectionBoxes_Precision/mAP'])
self.assertAlmostEqual(1.0,
metrics['DetectionMasks_Precision/mAP'])
def get_configs_from_multiple_files():
"""Reads evaluation configuration from multiple config files.
Reads the evaluation config from the following files:
model_config: Read from --model_config_path
eval_config: Read from --eval_config_path
input_config: Read from --input_config_path
Returns:
model_config: a model_pb2.DetectionModel
eval_config: a eval_pb2.EvalConfig
input_config: a input_reader_pb2.InputReader
"""
eval_config = eval_pb2.EvalConfig()
with tf.gfile.GFile(FLAGS.eval_config_path, 'r') as f:
text_format.Merge(f.read(), eval_config)
model_config = model_pb2.DetectionModel()
with tf.gfile.GFile(FLAGS.model_config_path, 'r') as f:
text_format.Merge(f.read(), model_config)
input_config = input_reader_pb2.InputReader()
with tf.gfile.GFile(FLAGS.input_config_path, 'r') as f:
text_format.Merge(f.read(), input_config)
return model_config, eval_config, input_config
def test_get_eval_metric_ops_raises_error_with_unsupported_metric(self):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend(['unsupported_metric'])
categories = self._get_categories_list()
eval_dict = self._make_evaluation_dict()
with self.assertRaises(ValueError):
eval_util.get_eval_metric_ops_for_evaluators(
eval_config, categories, eval_dict)
def get_configs_from_multiple_files(model_config_path="",
train_config_path="",
train_input_config_path="",
eval_config_path="",
eval_input_config_path="",
offline_eval_input_config_path=""):
"""Reads training configuration from multiple config files.
Args:
model_config_path: Path to model_pb2.DetectionModel.
train_config_path: Path to train_pb2.TrainConfig.
train_input_config_path: Path to input_reader_pb2.InputReader.
eval_config_path: Path to eval_pb2.EvalConfig.
eval_input_config_path: Path to input_reader_pb2.InputReader.
Returns:
Dictionary of configuration objects. Keys are `model`, `train_config`,
`train_input_config`, `eval_config`, `eval_input_config`. Key/Values are
returned only for valid (non-empty) strings.
"""
configs = {}
if model_config_path:
model_config = model_pb2.DetectionModel()
with tf.gfile.GFile(model_config_path, "r") as f:
text_format.Merge(f.read(), model_config)
configs["model"] = model_config
if train_config_path:
train_config = train_pb2.TrainConfig()
with tf.gfile.GFile(train_config_path, "r") as f:
text_format.Merge(f.read(), train_config)
configs["train_config"] = train_config
if train_input_config_path:
train_input_config = input_reader_pb2.InputReader()
with tf.gfile.GFile(train_input_config_path, "r") as f:
text_format.Merge(f.read(), train_input_config)
configs["train_input_config"] = train_input_config
if eval_config_path:
eval_config = eval_pb2.EvalConfig()
with tf.gfile.GFile(eval_config_path, "r") as f:
text_format.Merge(f.read(), eval_config)
configs["eval_config"] = eval_config
if eval_input_config_path:
eval_input_config = input_reader_pb2.InputReader()
with tf.gfile.GFile(eval_input_config_path, "r") as f:
text_format.Merge(f.read(), eval_input_config)
configs["eval_input_config"] = eval_input_config
if offline_eval_input_config_path:
offline_eval_input_config = input_reader_pb2.InputReader()
with tf.gfile.GFile(offline_eval_input_config_path, "r") as f:
text_format.Merge(f.read(), offline_eval_input_config)
configs["offline_eval_input_config"] = offline_eval_input_config
return configs
def test_get_configs_from_multiple_files(self):
"""Tests that proto configs can be read from multiple files."""
print(
'\n=========================================================================='
)
print('test_get_configs_from_multiple_files')
#temp_dir = self.get_temp_dir()
temp_dir = '/home/zq/tmp/'
# Write model config file.
model_config_path = os.path.join(temp_dir, "model.config")
model = model_pb2.DetectionModel()
model.faster_rcnn.num_classes = 10
_write_config(model, model_config_path)
# Write train config file.
train_config_path = os.path.join(temp_dir, "train.config")
train_config = train_pb2.TrainConfig()
train_config.batch_size = 32
_write_config(train_config, train_config_path)
# Write train input config file.
train_input_config_path = os.path.join(temp_dir, "train_input.config")
train_input_config = input_reader_pb2.InputReader()
train_input_config.label_map_path = "path/to/label_map"
_write_config(train_input_config, train_input_config_path)
# Write eval config file.
eval_config_path = os.path.join(temp_dir, "eval.config")
eval_config = eval_pb2.EvalConfig()
eval_config.num_examples = 20
_write_config(eval_config, eval_config_path)
# Write eval input config file.
eval_input_config_path = os.path.join(temp_dir, "eval_input.config")
eval_input_config = input_reader_pb2.InputReader()
eval_input_config.label_map_path = "path/to/another/label_map"
_write_config(eval_input_config, eval_input_config_path)
configs = config_util.get_configs_from_multiple_files(
model_config_path=model_config_path,
train_config_path=train_config_path,
train_input_config_path=train_input_config_path,
eval_config_path=eval_config_path,
eval_input_config_path=eval_input_config_path)
self.assertProtoEquals(model, configs["model"])
self.assertProtoEquals(train_config, configs["train_config"])
self.assertProtoEquals(train_input_config,
configs["train_input_config"])
self.assertProtoEquals(eval_config, configs["eval_config"])
self.assertProtoEquals(eval_input_config, configs["eval_input_config"])
def test_get_evaluator_with_evaluator_options(self):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend(['coco_detection_metrics'])
eval_config.include_metrics_per_category = True
categories = self._get_categories_list()
evaluator_options = eval_util.evaluator_options_from_eval_config(
eval_config)
evaluator = eval_util.get_evaluators(eval_config, categories,
evaluator_options)
self.assertTrue(evaluator[0]._include_metrics_per_category)
def test_get_eval_metric_ops_for_evaluators(self):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend(
['coco_detection_metrics', 'coco_mask_metrics'])
eval_config.include_metrics_per_category = True
evaluator_options = eval_util.evaluator_options_from_eval_config(
eval_config)
self.assertTrue(evaluator_options['coco_detection_metrics']
['include_metrics_per_category'])
self.assertTrue(evaluator_options['coco_mask_metrics']
['include_metrics_per_category'])
def test_get_evaluator_with_unmatched_label(self):
eval_config = eval_pb2.EvalConfig()
person_keypoints_metric = eval_config.parameterized_metric.add()
person_keypoints_metric.coco_keypoint_metrics.class_label = 'unmatched'
person_keypoints_metric.coco_keypoint_metrics.keypoint_label_to_sigmas[
'kpt'] = 0.1
categories = self._get_categories_list_with_keypoints()
evaluator = eval_util.get_evaluators(
eval_config, categories, evaluator_options=None)
self.assertLen(evaluator, 1)
self.assertNotIsInstance(
evaluator[0], coco_evaluation.CocoKeypointEvaluator)
def test_evaluator_options_from_eval_config_no_super_categories(self):
eval_config_text_proto = """
metrics_set: "coco_detection_metrics"
metrics_set: "coco_mask_metrics"
include_metrics_per_category: true
use_moving_averages: false
batch_size: 1;
"""
eval_config = eval_pb2.EvalConfig()
text_format.Merge(eval_config_text_proto, eval_config)
evaluator_options = eval_util.evaluator_options_from_eval_config(
eval_config)
self.assertNotIn('super_categories', evaluator_options['coco_mask_metrics'])
def test_get_evaluator_with_no_evaluator_options(self):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend(['coco_detection_metrics'])
eval_config.include_metrics_per_category = True
categories = self._get_categories_list()
evaluator = eval_util.get_evaluators(eval_config,
categories,
evaluator_options=None)
# Even though we are setting eval_config.include_metrics_per_category = True
# this option is never passed into the DetectionEvaluator constructor (via
# `evaluator_options`).
self.assertFalse(evaluator[0]._include_metrics_per_category)
def test_get_evaluator_with_no_evaluator_options(self):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend(
['coco_detection_metrics', 'precision_at_recall_detection_metrics'])
eval_config.include_metrics_per_category = True
eval_config.recall_lower_bound = 0.2
eval_config.recall_upper_bound = 0.6
categories = self._get_categories_list()
evaluator = eval_util.get_evaluators(
eval_config, categories, evaluator_options=None)
# Even though we are setting eval_config.include_metrics_per_category = True
# and bounds on recall, these options are never passed into the
# DetectionEvaluator constructor (via `evaluator_options`).
self.assertFalse(evaluator[0]._include_metrics_per_category)
self.assertAlmostEqual(evaluator[1]._recall_lower_bound, 0.0)
self.assertAlmostEqual(evaluator[1]._recall_upper_bound, 1.0)
def test_get_eval_metric_ops_for_coco_detections(self):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend(['coco_detection_metrics'])
categories = self._get_categories_list()
eval_dict = self._make_evaluation_dict()
metric_ops = eval_util.get_eval_metric_ops_for_evaluators(
eval_config, categories, eval_dict)
_, update_op = metric_ops['DetectionBoxes_Precision/mAP']
with self.test_session() as sess:
metrics = {}
for key, (value_op, _) in metric_ops.iteritems():
metrics[key] = value_op
sess.run(update_op)
metrics = sess.run(metrics)
print(metrics)
self.assertAlmostEqual(1.0,
metrics['DetectionBoxes_Precision/mAP'])
self.assertNotIn('DetectionMasks_Precision/mAP', metrics)
def test_get_evaluator_with_evaluator_options(self):
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend([
'coco_detection_metrics', 'precision_at_recall_detection_metrics'
])
eval_config.include_metrics_per_category = True
eval_config.recall_lower_bound = 0.2
eval_config.recall_upper_bound = 0.6
categories = self._get_categories_list()
evaluator_options = eval_util.evaluator_options_from_eval_config(
eval_config)
evaluator = eval_util.get_evaluators(eval_config, categories,
evaluator_options)
self.assertTrue(evaluator[0]._include_metrics_per_category)
self.assertAlmostEqual(evaluator[1]._recall_lower_bound,
eval_config.recall_lower_bound)
self.assertAlmostEqual(evaluator[1]._recall_upper_bound,
eval_config.recall_upper_bound)
with open(os.path.join(output_dir, 'metrics.csv'), 'w') as csvfile:
metrics_writer = csv.writer(csvfile, delimiter=',')
for metric_name, metric_value in metrics.items():
metrics_writer.writerow([metric_name, str(metric_value)])
# FILENAME = "camera_data/training/training.record-00000-of-00075"
FILENAME = "../../old/predictions_v0/initial_crop_size_28/validation_detections.tfrecord-00000-of-00001"
data_parser = tf_example_parser.TfExampleDetectionAndGTParser()
dataset = tf.data.TFRecordDataset(FILENAME, compression_type='')
# serialized_example = next(iter(dataset))
categories = label_map_util.create_categories_from_labelmap(
"../../data/camera_data/label_map.pbtxt")
eval_config = eval_pb2.EvalConfig()
eval_config.metrics_set.extend(['coco_detection_metrics'])
# Per category metrics not working
# eval_config.include_metrics_per_category = True
evaluator_options = evaluator_options_from_eval_config(eval_config)
object_detection_evaluators = get_evaluators(eval_config, categories,
evaluator_options)
object_detection_evaluator = object_detection_evaluators[0]
def scale_boxes_to_absolute_coordinates(decoded_dict):
def _scale_box_to_absolute(args):
boxes, height, width = args
return box_list_ops.to_absolute_coordinates(box_list.BoxList(boxes),
height, width).get()
