def main(_): config = hparams_config.get_efficientdet_config(FLAGS.model_name) config.override(FLAGS.hparams) config.batch_size = FLAGS.batch_size config.val_json_file = FLAGS.val_json_file # dataset ds = dataloader.InputReader( FLAGS.val_file_pattern, is_training=False, use_fake_data=False, max_instances_per_image=config.max_instances_per_image)(config) # Network model = efficientdet_keras.EfficientDetNet(config=config) model.build((config.batch_size, None, None, 3)) model.load_weights(tf.train.latest_checkpoint(FLAGS.model_dir)) evaluator = coco_metric.EvaluationMetric(filename=config.val_json_file) # compute stats for all batches. for images, labels in ds: config.nms_configs.max_nms_inputs = anchors.MAX_DETECTION_POINTS cls_outputs, box_outputs = model(images, training=False) detections = postprocess.generate_detections(config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids'], False) if FLAGS.enable_tta: images_flipped = tf.image.flip_left_right(images) cls_outputs_flipped, box_outputs_flipped = model(images_flipped, training=False) detections_flipped = postprocess.generate_detections( config, cls_outputs_flipped, box_outputs_flipped, labels['image_scales'], labels['source_ids'], True) for d, df in zip(detections, detections_flipped): combined_detections = wbf.ensemble_detections( config, tf.concat([d, df], 0)) combined_detections = tf.stack([combined_detections]) evaluator.update_state( labels['groundtruth_data'].numpy(), postprocess.transform_detections( combined_detections).numpy()) else: evaluator.update_state( labels['groundtruth_data'].numpy(), postprocess.transform_detections(detections).numpy()) # compute the final eval results. metric_values = evaluator.result() metric_dict = {} for i, metric_value in enumerate(metric_values): metric_dict[evaluator.metric_names[i]] = metric_value print(metric_dict)
def _get_detections(self, images, labels): cls_outputs, box_outputs = self.model(images, training=False) detections = postprocess.generate_detections(self.config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids']) return postprocess.transform_detections(detections)
def model_fn(images, labels): cls_outputs, box_outputs = model(images, training=False) detections = postprocess.generate_detections( config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids']) tf.numpy_function(evaluator.update_state, [ labels['groundtruth_data'], postprocess.transform_detections(detections) ], [])
def _get_detections(self, images, labels): cls_outputs, box_outputs = util_keras.fp16_to_fp32_nested( self.model(images, training=False)) detections = postprocess.generate_detections(self.config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids']) tf.numpy_function(self.evaluator.update_state, [ labels['groundtruth_data'], postprocess.transform_detections(detections) ], [])
def main(_): config = hparams_config.get_efficientdet_config(FLAGS.model_name) config.override(FLAGS.hparams) config.val_json_file = FLAGS.val_json_file config.nms_configs.max_nms_inputs = anchors.MAX_DETECTION_POINTS config.drop_remainder = False # eval all examples w/o drop. config.image_size = utils.parse_image_size(config['image_size']) # Evaluator for AP calculation. label_map = label_util.get_label_map(config.label_map) evaluator = coco_metric.EvaluationMetric( filename=config.val_json_file, label_map=label_map) # dataset batch_size = 1 ds = dataloader.InputReader( FLAGS.val_file_pattern, is_training=False, max_instances_per_image=config.max_instances_per_image)( config, batch_size=batch_size) eval_samples = FLAGS.eval_samples if eval_samples: ds = ds.take((eval_samples + batch_size - 1) // batch_size) # Network lite_runner = LiteRunner(FLAGS.tflite_path) eval_samples = FLAGS.eval_samples or 5000 pbar = tf.keras.utils.Progbar((eval_samples + batch_size - 1) // batch_size) for i, (images, labels) in enumerate(ds): cls_outputs, box_outputs = lite_runner.run(images) detections = postprocess.generate_detections(config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids']) detections = postprocess.transform_detections(detections) evaluator.update_state(labels['groundtruth_data'].numpy(), detections.numpy()) pbar.update(i) # compute the final eval results. metrics = evaluator.result() metric_dict = {} for i, name in enumerate(evaluator.metric_names): metric_dict[name] = metrics[i] if label_map: for i, cid in enumerate(sorted(label_map.keys())): name = 'AP_/%s' % label_map[cid] metric_dict[name] = metrics[i + len(evaluator.metric_names)] print(FLAGS.model_name, metric_dict)
def test_transform_detections(self): corners = tf.constant( [[[0., -1.177383, 1.793507, 8.340945, 4.418388, 0.901576, 2.], [0., 5.676410, 6.102146, 7.785691, 8.537168, 0.888125, 1.]], [[1., 5.885427, 13.529362, 11.410081, 14.154047, 0.884544, 1.], [1., 8.145872, -9.660868, 14.173973, 10.41237, 0.815883, 2.]]]) corner_plus_area = postprocess.transform_detections(corners) self.assertAllClose( corner_plus_area.numpy(), [[[0., -1.177383, 1.793507, 9.518328, 2.624881, 0.901576, 2.], [0., 5.676410, 6.102146, 2.109282, 2.435021, 0.888125, 1.]], [[1., 5.885427, 13.529362, 5.524654, 0.624685, 0.884544, 1.], [1., 8.145872, -9.660868, 6.028101, 20.073238, 0.815883, 2.]]])
def metric_fn(**kwargs): """Returns a dictionary that has the evaluation metrics.""" if params['nms_configs'].get('pyfunc', True): detections_bs = [] nms_configs = params['nms_configs'] for index in range(kwargs['boxes'].shape[0]): detections = tf.numpy_function( functools.partial(nms_np.per_class_nms, nms_configs=nms_configs), [ kwargs['boxes'][index], kwargs['scores'][index], kwargs['classes'][index], tf.slice(kwargs['image_ids'], [index], [1]), tf.slice(kwargs['image_scales'], [index], [1]), params['num_classes'], nms_configs['max_output_size'], ], tf.float32) detections_bs.append(detections) detections_bs = postprocess.transform_detections( tf.stack(detections_bs)) else: # These two branches should be equivalent, but currently they are not. # TODO(tanmingxing): enable the non_pyfun path after bug fix. nms_boxes, nms_scores, nms_classes, _ = postprocess.per_class_nms( params, kwargs['boxes'], kwargs['scores'], kwargs['classes'], kwargs['image_scales']) img_ids = tf.cast(tf.expand_dims(kwargs['image_ids'], -1), nms_scores.dtype) detections_bs = [ img_ids * tf.ones_like(nms_scores), nms_boxes[:, :, 1], nms_boxes[:, :, 0], nms_boxes[:, :, 3] - nms_boxes[:, :, 1], nms_boxes[:, :, 2] - nms_boxes[:, :, 0], nms_scores, nms_classes, ] detections_bs = tf.stack(detections_bs, axis=-1, name='detnections') if params.get('testdev_dir', None): logging.info('Eval testdev_dir %s', params['testdev_dir']) eval_metric = coco_metric.EvaluationMetric( testdev_dir=params['testdev_dir']) coco_metrics = eval_metric.estimator_metric_fn( detections_bs, tf.zeros([1])) else: logging.info('Eval val with groudtruths %s.', params['val_json_file']) eval_metric = coco_metric.EvaluationMetric( filename=params['val_json_file'], label_map=params['label_map']) coco_metrics = eval_metric.estimator_metric_fn( detections_bs, kwargs['groundtruth_data']) # Add metrics to output. cls_loss = tf.metrics.mean(kwargs['cls_loss_repeat']) box_loss = tf.metrics.mean(kwargs['box_loss_repeat']) output_metrics = { 'cls_loss': cls_loss, 'box_loss': box_loss, } output_metrics.update(coco_metrics) return output_metrics
def main(_): config = hparams_config.get_efficientdet_config(FLAGS.model_name) config.override(FLAGS.hparams) config.batch_size = FLAGS.batch_size config.val_json_file = FLAGS.val_json_file config.nms_configs.max_nms_inputs = anchors.MAX_DETECTION_POINTS base_height, base_width = utils.parse_image_size(config['image_size']) if FLAGS.strategy == 'tpu': tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) tf.config.experimental_connect_to_cluster(tpu_cluster_resolver) tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver) ds_strategy = tf.distribute.TPUStrategy(tpu_cluster_resolver) logging.info('All devices: %s', tf.config.list_logical_devices('TPU')) elif FLAGS.strategy == 'gpus': ds_strategy = tf.distribute.MirroredStrategy() logging.info('All devices: %s', tf.config.list_physical_devices('GPU')) else: if tf.config.list_physical_devices('GPU'): ds_strategy = tf.distribute.OneDeviceStrategy('device:GPU:0') else: ds_strategy = tf.distribute.OneDeviceStrategy('device:CPU:0') # in format (height, width, flip) augmentations = [] if FLAGS.enable_tta: for size_offset in (0, 128, 256): for flip in (False, True): augmentations.append((base_height + size_offset, base_width + size_offset, flip)) else: augmentations.append((base_height, base_width, False)) all_detections = [] all_labels = [] with ds_strategy.scope(): # Network model = efficientdet_keras.EfficientDetNet(config=config) model.build((config.batch_size, base_height, base_width, 3)) model.load_weights(tf.train.latest_checkpoint(FLAGS.model_dir)) first_loop = True for height, width, flip in augmentations: config.image_size = (height, width) # dataset ds = dataloader.InputReader( FLAGS.val_file_pattern, is_training=False, use_fake_data=False, max_instances_per_image=config.max_instances_per_image)(config) # create the function once per augmentation, since it closes over the # value of config, which gets updated with the new image size @tf.function def f(images, labels): cls_outputs, box_outputs = model(images, training=False) return postprocess.generate_detections(config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids'], flip) # inference for images, labels in ds: if flip: images = tf.image.flip_left_right(images) detections = f(images, labels) all_detections.append(detections) if first_loop: all_labels.append(labels) first_loop = False # collect the giant list of detections into a map from image id to # detections detections_per_source = dict() for batch in all_detections: for d in batch: img_id = d[0][0] if img_id.numpy() in detections_per_source: detections_per_source[img_id.numpy()] = tf.concat( [d, detections_per_source[img_id.numpy()]], 0) else: detections_per_source[img_id.numpy()] = d # collect the groundtruth per image id groundtruth_per_source = dict() for batch in all_labels: for img_id, groundtruth in zip(batch['source_ids'], batch['groundtruth_data']): groundtruth_per_source[img_id.numpy()] = groundtruth # calucate the AP scores for all the images evaluator = coco_metric.EvaluationMetric(filename=config.val_json_file) for img_id, d in detections_per_source.items(): if FLAGS.enable_tta: d = wbf.ensemble_detections(config, d, len(augmentations)) evaluator.update_state( tf.stack([groundtruth_per_source[img_id]]).numpy(), postprocess.transform_detections(tf.stack([d])).numpy()) # compute the final eval results. if evaluator: metrics = evaluator.result() metric_dict = {} for i, name in enumerate(evaluator.metric_names): metric_dict[name] = metrics[i] label_map = label_util.get_label_map(config.label_map) if label_map: for i, cid in enumerate(sorted(label_map.keys())): name = 'AP_/%s' % label_map[cid] metric_dict[name] = metrics[i - len(evaluator.metric_names)] print(metric_dict)
def main(_): config = hparams_config.get_efficientdet_config(FLAGS.model_name) config.override(FLAGS.hparams) config.val_json_file = FLAGS.val_json_file config.nms_configs.max_nms_inputs = anchors.MAX_DETECTION_POINTS config.drop_remainder = False # eval all examples w/o drop. config.image_size = utils.parse_image_size(config['image_size']) # Evaluator for AP calculation. label_map = label_util.get_label_map(config.label_map) evaluator = coco_metric.EvaluationMetric(filename=config.val_json_file, label_map=label_map) # dataset batch_size = 1 ds = dataloader.InputReader( FLAGS.val_file_pattern, is_training=False, max_instances_per_image=config.max_instances_per_image)( config, batch_size=batch_size) eval_samples = FLAGS.eval_samples if eval_samples: ds = ds.take((eval_samples + batch_size - 1) // batch_size) # Network lite_runner = LiteRunner(FLAGS.tflite_path, FLAGS.only_network) eval_samples = FLAGS.eval_samples or 5000 pbar = tf.keras.utils.Progbar( (eval_samples + batch_size - 1) // batch_size) for i, (images, labels) in enumerate(ds): if not FLAGS.only_network: nms_boxes_bs, nms_classes_bs, nms_scores_bs, _ = lite_runner.run( images) nms_classes_bs += postprocess.CLASS_OFFSET height, width = utils.parse_image_size(config.image_size) normalize_factor = tf.constant([height, width, height, width], dtype=tf.float32) nms_boxes_bs *= normalize_factor if labels['image_scales'] is not None: scales = tf.expand_dims( tf.expand_dims(labels['image_scales'], -1), -1) nms_boxes_bs = nms_boxes_bs * tf.cast(scales, nms_boxes_bs.dtype) detections = postprocess.generate_detections_from_nms_output( nms_boxes_bs, nms_classes_bs, nms_scores_bs, labels['source_ids']) else: cls_outputs, box_outputs = lite_runner.run(images) detections = postprocess.generate_detections( config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids'], pre_class_nms=FLAGS.pre_class_nms) detections = postprocess.transform_detections(detections) evaluator.update_state(labels['groundtruth_data'].numpy(), detections.numpy()) pbar.update(i) # compute the final eval results. metrics = evaluator.result() metric_dict = {} for i, name in enumerate(evaluator.metric_names): metric_dict[name] = metrics[i] if label_map: for i, cid in enumerate(sorted(label_map.keys())): name = 'AP_/%s' % label_map[cid] metric_dict[name] = metrics[i + len(evaluator.metric_names)] print(FLAGS.model_name, metric_dict)
def main(_): config = hparams_config.get_efficientdet_config(FLAGS.model_name) config.override(FLAGS.hparams) config.batch_size = FLAGS.batch_size config.val_json_file = FLAGS.val_json_file config.nms_configs.max_nms_inputs = anchors.MAX_DETECTION_POINTS base_height, base_width = utils.parse_image_size(config['image_size']) # Network model = efficientdet_keras.EfficientDetNet(config=config) model.build((config.batch_size, base_height, base_width, 3)) model.load_weights(tf.train.latest_checkpoint(FLAGS.model_dir)) @tf.function def f(imgs, labels, flip): cls_outputs, box_outputs = model(imgs, training=False) return postprocess.generate_detections(config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids'], flip) # in format (height, width, flip) augmentations = [] if FLAGS.enable_tta: for size_offset in (0, 128, 256): for flip in (False, True): augmentations.append((base_height + size_offset, base_width + size_offset, flip)) else: augmentations.append((base_height, base_width, False)) evaluator = None detections_per_source = dict() for height, width, flip in augmentations: config.image_size = (height, width) # dataset ds = dataloader.InputReader( FLAGS.val_file_pattern, is_training=False, use_fake_data=False, max_instances_per_image=config.max_instances_per_image)(config) # compute stats for all batches. total_steps = FLAGS.eval_samples // FLAGS.batch_size progress = tf.keras.utils.Progbar(total_steps) for i, (images, labels) in enumerate(ds): progress.update(i, values=None) if i > total_steps: break if flip: images = tf.image.flip_left_right(images) detections = f(images, labels, flip) for img_id, d in zip(labels['source_ids'], detections): if img_id.numpy() in detections_per_source: detections_per_source[img_id.numpy()] = tf.concat( [d, detections_per_source[img_id.numpy()]], 0) else: detections_per_source[img_id.numpy()] = d evaluator = coco_metric.EvaluationMetric( filename=config.val_json_file) for d in detections_per_source.values(): if FLAGS.enable_tta: d = wbf.ensemble_detections(config, d, len(augmentations)) evaluator.update_state( labels['groundtruth_data'].numpy(), postprocess.transform_detections(tf.stack([d])).numpy()) # compute the final eval results. if evaluator: metrics = evaluator.result() metric_dict = {} for i, name in enumerate(evaluator.metric_names): metric_dict[name] = metrics[i] label_map = label_util.get_label_map(config.label_map) if label_map: for i, cid in enumerate(sorted(label_map.keys())): name = 'AP_/%s' % label_map[cid] metric_dict[name] = metrics[i - len(evaluator.metric_names)] print(metric_dict)
def main(_): config = hparams_config.get_efficientdet_config(FLAGS.model_name) config.override(FLAGS.hparams) config.batch_size = FLAGS.batch_size config.val_json_file = FLAGS.val_json_file config.nms_configs.max_nms_inputs = anchors.MAX_DETECTION_POINTS base_height, base_width = utils.parse_image_size(config['image_size']) # Network model = efficientdet_keras.EfficientDetNet(config=config) model.build((config.batch_size, base_height, base_width, 3)) model.load_weights(tf.train.latest_checkpoint(FLAGS.model_dir)) # in format (height, width, flip) augmentations = [] if FLAGS.enable_tta: for size_offset in (0, 128, 256): for flip in (False, True): augmentations.append((base_height + size_offset, base_width + size_offset, flip)) else: augmentations.append((base_height, base_width, False)) detections_per_source = dict() for height, width, flip in augmentations: config.image_size = (height, width) # dataset ds = dataloader.InputReader( FLAGS.val_file_pattern, is_training=False, use_fake_data=False, max_instances_per_image=config.max_instances_per_image)( config) # compute stats for all batches. for images, labels in ds: if flip: images = tf.image.flip_left_right(images) cls_outputs, box_outputs = model(images, training=False) detections = postprocess.generate_detections(config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids'], flip) for id, d in zip(labels['source_ids'], detections): if id.numpy() in detections_per_source: detections_per_source[id.numpy()] = tf.concat([d, detections_per_source[id.numpy()]], 0) else: detections_per_source[id.numpy()] = d evaluator = coco_metric.EvaluationMetric(filename=config.val_json_file) for d in detections_per_source.values(): if FLAGS.enable_tta: d = wbf.ensemble_detections(config, d, len(augmentations)) evaluator.update_state( labels['groundtruth_data'].numpy(), postprocess.transform_detections(tf.stack([d])).numpy()) # compute the final eval results. metric_values = evaluator.result() metric_dict = {} for i, metric_value in enumerate(metric_values): metric_dict[evaluator.metric_names[i]] = metric_value print(metric_dict)
def eval_update(gt, pred): tf.numpy_function(evaluator.update_state, [gt, postprocess.transform_detections(pred)], [])
def main(_): config = hparams_config.get_efficientdet_config(FLAGS.model_name) config.override(FLAGS.hparams) config.batch_size = FLAGS.batch_size config.val_json_file = FLAGS.val_json_file config.nms_configs.max_nms_inputs = anchors.MAX_DETECTION_POINTS config.drop_remainder = False # eval all examples w/o drop. config.image_size = utils.parse_image_size(config['image_size']) if FLAGS.strategy == 'tpu': tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) tf.config.experimental_connect_to_cluster(tpu_cluster_resolver) tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver) ds_strategy = tf.distribute.TPUStrategy(tpu_cluster_resolver) logging.info('All devices: %s', tf.config.list_logical_devices('TPU')) elif FLAGS.strategy == 'gpus': ds_strategy = tf.distribute.MirroredStrategy() logging.info('All devices: %s', tf.config.list_physical_devices('GPU')) else: if tf.config.list_physical_devices('GPU'): ds_strategy = tf.distribute.OneDeviceStrategy('device:GPU:0') else: ds_strategy = tf.distribute.OneDeviceStrategy('device:CPU:0') with ds_strategy.scope(): # Network model = efficientdet_keras.EfficientDetNet(config=config) model.build((config.batch_size, *config.image_size, 3)) model.load_weights(tf.train.latest_checkpoint(FLAGS.model_dir)) @tf.function def f(images, labels): cls_outputs, box_outputs = model(images, training=False) return postprocess.generate_detections(config, cls_outputs, box_outputs, labels['image_scales'], labels['source_ids']) # dataset ds = dataloader.InputReader( FLAGS.val_file_pattern, is_training=False, max_instances_per_image=config.max_instances_per_image)(config) if FLAGS.eval_samples: ds = ds.take(FLAGS.eval_samples // config.batch_size) # create a evaluator for AP calculation. label_map = label_util.get_label_map(config.label_map) evaluator = coco_metric.EvaluationMetric(filename=config.val_json_file, label_map=label_map) # evaluate all images. eval_samples = FLAGS.eval_samples or 5000 pbar = tf.keras.utils.Progbar(eval_samples // config.batch_size) for i, (images, labels) in enumerate(ds): detections = f(images, labels) evaluator.update_state( labels['groundtruth_data'].numpy(), postprocess.transform_detections(detections).numpy()) pbar.update(i) # compute the final eval results. metrics = evaluator.result() metric_dict = {} for i, name in enumerate(evaluator.metric_names): metric_dict[name] = metrics[i] if label_map: for i, cid in enumerate(sorted(label_map.keys())): name = 'AP_/%s' % label_map[cid] metric_dict[name] = metrics[i + len(evaluator.metric_names)] print(FLAGS.model_name, metric_dict)