def evaluate_tflite(self,
tflite_filepath: str,
dataset: tf.data.Dataset,
steps: int,
json_file: Optional[str] = None) -> Dict[str, float]:
"""Evaluate the EfficientDet TFLite model.
Args:
tflite_filepath: File path to the TFLite model.
dataset: tf.data.Dataset used for evaluation.
steps: Number of steps to evaluate the model.
json_file: JSON with COCO data format containing golden bounding boxes.
Used for validation. If None, use the ground truth from the dataloader.
Refer to
https://towardsdatascience.com/coco-data-format-for-object-detection-a4c5eaf518c5
for the description of COCO data format.
Returns:
A dict contains AP metrics.
"""
# TODO(b/182441458): Use the task library for evaluation instead once it
# supports python interface.
evaluator, label_map = self._get_evaluator_and_label_map(json_file)
dataset = dataset.take(steps)
lite_runner = eval_tflite.LiteRunner(tflite_filepath,
only_network=False)
progbar = tf.keras.utils.Progbar(steps)
for i, (images, labels) in enumerate(dataset):
# Get the output result after post-processing NMS op.
nms_boxes, nms_classes, nms_scores, _ = lite_runner.run(images)
# CLASS_OFFSET is used since label_id for `background` is 0 in label_map
# while it's not actually included the model. We don't need to add the
# offset in the Android application.
nms_classes += postprocess.CLASS_OFFSET
height, width = utils.parse_image_size(self.config.image_size)
normalize_factor = tf.constant([height, width, height, width],
dtype=tf.float32)
nms_boxes *= normalize_factor
if labels['image_scales'] is not None:
scales = tf.expand_dims(
tf.expand_dims(labels['image_scales'], -1), -1)
nms_boxes = nms_boxes * tf.cast(scales, nms_boxes.dtype)
detections = postprocess.generate_detections_from_nms_output(
nms_boxes, nms_classes, nms_scores, labels['source_ids'])
detections = postprocess.transform_detections(detections)
evaluator.update_state(labels['groundtruth_data'].numpy(),
detections.numpy())
progbar.update(i + 1)
print()
metric_dict = self._get_metric_dict(evaluator, label_map)
return metric_dict
def main(_):
config = hparams_config.get_efficientdet_config(FLAGS.model_name)
config.override(FLAGS.hparams)
config.val_json_file = FLAGS.val_json_file
config.nms_configs.max_nms_inputs = anchors.MAX_DETECTION_POINTS
config.drop_remainder = False # eval all examples w/o drop.
config.image_size = utils.parse_image_size(config['image_size'])
# Evaluator for AP calculation.
label_map = label_util.get_label_map(config.label_map)
evaluator = coco_metric.EvaluationMetric(filename=config.val_json_file,
label_map=label_map)
# dataset
batch_size = 1
ds = dataloader.InputReader(
FLAGS.val_file_pattern,
is_training=False,
max_instances_per_image=config.max_instances_per_image)(
config, batch_size=batch_size)
eval_samples = FLAGS.eval_samples
if eval_samples:
ds = ds.take((eval_samples + batch_size - 1) // batch_size)
# Network
lite_runner = LiteRunner(FLAGS.tflite_path, FLAGS.only_network)
eval_samples = FLAGS.eval_samples or 5000
pbar = tf.keras.utils.Progbar(
(eval_samples + batch_size - 1) // batch_size)
for i, (images, labels) in enumerate(ds):
if not FLAGS.only_network:
nms_boxes_bs, nms_classes_bs, nms_scores_bs, _ = lite_runner.run(
images)
nms_classes_bs += postprocess.CLASS_OFFSET
height, width = utils.parse_image_size(config.image_size)
normalize_factor = tf.constant([height, width, height, width],
dtype=tf.float32)
nms_boxes_bs *= normalize_factor
if labels['image_scales'] is not None:
scales = tf.expand_dims(
tf.expand_dims(labels['image_scales'], -1), -1)
nms_boxes_bs = nms_boxes_bs * tf.cast(scales,
nms_boxes_bs.dtype)
detections = postprocess.generate_detections_from_nms_output(
nms_boxes_bs, nms_classes_bs, nms_scores_bs,
labels['source_ids'])
else:
cls_outputs, box_outputs = lite_runner.run(images)
detections = postprocess.generate_detections(
config,
cls_outputs,
box_outputs,
labels['image_scales'],
labels['source_ids'],
pre_class_nms=FLAGS.pre_class_nms)
detections = postprocess.transform_detections(detections)
evaluator.update_state(labels['groundtruth_data'].numpy(),
detections.numpy())
pbar.update(i)
# compute the final eval results.
metrics = evaluator.result()
metric_dict = {}
for i, name in enumerate(evaluator.metric_names):
metric_dict[name] = metrics[i]
if label_map:
for i, cid in enumerate(sorted(label_map.keys())):
name = 'AP_/%s' % label_map[cid]
metric_dict[name] = metrics[i + len(evaluator.metric_names)]
print(FLAGS.model_name, metric_dict)
