def test_from_pascal_voc(self): images_dir, annotations_dir, label_map = test_util.create_pascal_voc( self.get_temp_dir()) model_spec = MockDetectorModelSpec('efficientdet-lite0') data = object_detector_dataloader.DataLoader.from_pascal_voc( images_dir, annotations_dir, label_map) self.assertIsInstance(data, object_detector_dataloader.DataLoader) self.assertLen(data, 1) self.assertEqual(data.label_map, label_map) self.assertTrue(os.path.isfile(data.annotations_json_file)) self.assertGreater(os.path.getsize(data.annotations_json_file), 0) expected_json_file = test_util.get_test_data_path('annotations.json') self.assertTrue( filecmp.cmp(data.annotations_json_file, expected_json_file)) ds = data.gen_dataset(model_spec, batch_size=1, is_training=False) for i, (images, labels) in enumerate(ds): self.assertEqual(i, 0) images_shape = tf.shape(images).numpy() expected_shape = np.array([1, *model_spec.config.image_size, 3]) self.assertTrue((images_shape == expected_shape).all()) self.assertLen(labels, 15) ds1 = data.gen_dataset(model_spec, batch_size=1, is_training=True) # Comments out this assert since it fails externally. # self.assertEqual(ds1.cardinality(), tf.data.INFINITE_CARDINALITY) for images, labels in ds1.take(10): images_shape = tf.shape(images).numpy() expected_shape = np.array([1, *model_spec.config.image_size, 3]) self.assertTrue((images_shape == expected_shape).all()) self.assertLen(labels, 15)
def testEfficientDetLite0(self): # Gets model specification. spec = model_spec.get('efficientdet_lite0') # Prepare data. images_dir, annotations_dir, label_map = test_util.create_pascal_voc( self.get_temp_dir()) data = object_detector_dataloader.DataLoader.from_pascal_voc( images_dir, annotations_dir, label_map) # Train the model. task = object_detector.create(data, spec, batch_size=1, epochs=1) self.assertEqual(spec.config.num_classes, 2) # Evaluate trained model metrics = task.evaluate(data) self.assertIsInstance(metrics, dict) self.assertGreaterEqual(metrics['AP'], 0) # Export the model to saved model. output_path = os.path.join(self.get_temp_dir(), 'saved_model') task.export(self.get_temp_dir(), export_format=ExportFormat.SAVED_MODEL) self.assertTrue(os.path.isdir(output_path)) self.assertNotEqual(len(os.listdir(output_path)), 0) # Export the model to the float TFLite model. output_path = os.path.join(self.get_temp_dir(), 'float.tflite') task.export(self.get_temp_dir(), tflite_filename='float.tflite', quantization_config=None, export_format=ExportFormat.TFLITE, with_metadata=True, export_metadata_json_file=True) # Checks the sizes of the float32 TFLite model files in bytes. model_size = 13476379 self.assertNear(os.path.getsize(output_path), model_size, 50000) json_output_file = os.path.join(self.get_temp_dir(), 'float.json') self.assertTrue(os.path.isfile(json_output_file)) self.assertGreater(os.path.getsize(json_output_file), 0) expected_json_file = test_util.get_test_data_path( 'efficientdet_lite0_metadata.json') self.assertTrue(filecmp.cmp(json_output_file, expected_json_file)) # Evaluate the TFLite model. task.evaluate_tflite(output_path, data) self.assertIsInstance(metrics, dict) self.assertGreaterEqual(metrics['AP'], 0) # Tests the default quantized model. filename = 'model_quant.tflite' output_path = os.path.join(self.get_temp_dir(), filename) task.export(self.get_temp_dir(), tflite_filename=filename, export_format=ExportFormat.TFLITE) model_size = 4312187 err = model_size * 0.05 self.assertTrue(os.path.isfile(output_path)) self.assertNear(os.path.getsize(output_path), model_size, err)
def test_pascal_voc_cache_writer(self): images_dir, annotations_dir, label_map = test_util.create_pascal_voc( self.get_temp_dir()) cache_writer = dataloader_util.PascalVocCacheFilesWriter(label_map, images_dir, num_shards=1) cache_files = dataloader_util.get_cache_files( cache_dir=self.get_temp_dir(), cache_prefix_filename='pascal') cache_writer.write_files(cache_files, annotations_dir) # Checks the TFRecord file. tfrecord_files = cache_files.tfrecord_files self.assertTrue(os.path.isfile(tfrecord_files[0])) self.assertGreater(os.path.getsize(tfrecord_files[0]), 0) # Checks the annotation json file. annotations_json_file = cache_files.annotations_json_file self.assertTrue(os.path.isfile(annotations_json_file)) self.assertGreater(os.path.getsize(annotations_json_file), 0) expected_json_file = test_util.get_test_data_path('annotations.json') self.assertTrue(filecmp.cmp(annotations_json_file, expected_json_file)) # Checks the meta_data file. meta_data_file = cache_files.meta_data_file self.assertTrue(os.path.isfile(meta_data_file)) self.assertGreater(os.path.getsize(meta_data_file), 0) with tf.io.gfile.GFile(meta_data_file, 'r') as f: meta_data_dict = yaml.load(f, Loader=yaml.FullLoader) self.assertEqual(meta_data_dict['size'], 1) self.assertEqual(meta_data_dict['label_map'], label_map) # Checks xml_dict from `_get_xml_dict` function. xml_dict = next(cache_writer._get_xml_dict(annotations_dir)) expected_xml_dict = { 'filename': '2012_12.jpg', 'folder': '', 'object': [{ 'difficult': '1', 'bndbox': { 'xmin': '64', 'ymin': '64', 'xmax': '192', 'ymax': '192', }, 'name': 'person', 'truncated': '0', 'pose': '', }], 'size': { 'width': '256', 'height': '256', } } self.assertEqual(xml_dict, expected_xml_dict)
def testEfficientDetLite0(self): # Gets model specification. spec = model_spec.get('efficientdet_lite0') # Prepare data. images_dir, annotations_dir, label_map = test_util.create_pascal_voc( self.get_temp_dir()) data = object_detector_dataloader.DataLoader.from_pascal_voc( images_dir, annotations_dir, label_map) # Train the model. task = object_detector.create(data, spec, batch_size=1, epochs=1) self.assertEqual(spec.config.num_classes, 2) # Evaluate trained model metrics = task.evaluate(data, batch_size=1) self.assertIsInstance(metrics, dict) self.assertGreaterEqual(metrics['AP'], 0) # Export the model to saved model. output_path = os.path.join(self.get_temp_dir(), 'saved_model') task.export(self.get_temp_dir(), export_format=ExportFormat.SAVED_MODEL) self.assertTrue(os.path.isdir(output_path)) self.assertNotEqual(len(os.listdir(output_path)), 0) # Export the model to TFLite model. output_path = os.path.join(self.get_temp_dir(), 'float.tflite') task.export(self.get_temp_dir(), tflite_filename='float.tflite', export_format=ExportFormat.TFLITE, with_metadata=True, export_metadata_json_file=True) self.assertTrue(tf.io.gfile.exists(output_path)) self.assertGreater(os.path.getsize(output_path), 0) json_output_file = os.path.join(self.get_temp_dir(), 'float.json') self.assertTrue(os.path.isfile(json_output_file)) self.assertGreater(os.path.getsize(json_output_file), 0) expected_json_file = test_util.get_test_data_path( 'efficientdet_lite0_metadata.json') self.assertTrue(filecmp.cmp(json_output_file, expected_json_file)) # Export the model to quantized TFLite model. # TODO(b/175173304): Skips the test for stable tensorflow 2.4 for now since # it fails. Will revert this change after TF upgrade. if tf.__version__.startswith('2.4'): return output_path = os.path.join(self.get_temp_dir(), 'model_quantized.tflite') config = configs.QuantizationConfig.create_full_integer_quantization( data, is_integer_only=True) task.export(self.get_temp_dir(), tflite_filename='model_quantized.tflite', quantization_config=config, export_format=ExportFormat.TFLITE) self.assertTrue(os.path.isfile(output_path)) self.assertGreater(os.path.getsize(output_path), 0)
def testEfficientDetLite0(self): # Gets model specification. hub_path = test_util.get_test_data_path('fake_effdet_lite0_hub') spec = object_detector_spec.EfficientDetModelSpec( model_name='efficientdet-lite0', uri=hub_path) # Prepare data. images_dir, annotations_dir, label_map = test_util.create_pascal_voc( self.get_temp_dir()) data = object_detector_dataloader.DataLoader.from_pascal_voc( images_dir, annotations_dir, label_map) # Train the model. task = object_detector.create(data, spec, batch_size=1, epochs=1) # Evaluate trained model metrics = task.evaluate(data, batch_size=1) self.assertIsInstance(metrics, dict) self.assertGreaterEqual(metrics['AP'], 0)
def testEfficientDetLite0(self): # Gets model specification. spec = model_spec.get('efficientdet_lite0') # Prepare data. images_dir, annotations_dir, label_map = test_util.create_pascal_voc( self.get_temp_dir()) data = object_detector_dataloader.DataLoader.from_pascal_voc( images_dir, annotations_dir, label_map) # Train the model. task = object_detector.create(data, spec, batch_size=1, epochs=1) self.assertEqual(spec.config.num_classes, 2) # Evaluate trained model metrics = task.evaluate(data) self.assertIsInstance(metrics, dict) self.assertGreaterEqual(metrics['AP'], 0) # Export the model to saved model. output_path = os.path.join(self.get_temp_dir(), 'saved_model') task.export(self.get_temp_dir(), export_format=ExportFormat.SAVED_MODEL) self.assertTrue(os.path.isdir(output_path)) self.assertNotEqual(len(os.listdir(output_path)), 0) # Export the model to TFLite model. output_path = os.path.join(self.get_temp_dir(), 'float.tflite') task.export(self.get_temp_dir(), tflite_filename='float.tflite', quantization_type=QuantizationType.FP32, export_format=ExportFormat.TFLITE, with_metadata=True, export_metadata_json_file=True) # Checks the sizes of the float32 TFLite model files in bytes. model_size = 13476379 self.assertNear(os.path.getsize(output_path), model_size, 50000) json_output_file = os.path.join(self.get_temp_dir(), 'float.json') self.assertTrue(os.path.isfile(json_output_file)) self.assertGreater(os.path.getsize(json_output_file), 0) expected_json_file = test_util.get_test_data_path( 'efficientdet_lite0_metadata.json') self.assertTrue(filecmp.cmp(json_output_file, expected_json_file)) # Evaluate the TFLite model. task.evaluate_tflite(output_path, data) self.assertIsInstance(metrics, dict) self.assertGreaterEqual(metrics['AP'], 0) # Export the model to quantized TFLite model. # TODO(b/175173304): Skips the test for stable tensorflow 2.4 for now since # it fails. Will revert this change after TF upgrade. if tf.__version__.startswith('2.4'): return # Not include QuantizationType.FP32 here since we have already tested it # above together with metadata file test. types = (QuantizationType.INT8, QuantizationType.FP16, QuantizationType.DYNAMIC) # The sizes of the TFLite model files in bytes. model_sizes = (4439987, 6840331, 4289875) for quantization_type, model_size in zip(types, model_sizes): filename = quantization_type.name.lower() + '.tflite' output_path = os.path.join(self.get_temp_dir(), filename) task.export(self.get_temp_dir(), quantization_type=quantization_type, tflite_filename=filename, export_format=ExportFormat.TFLITE) self.assertNear(os.path.getsize(output_path), model_size, 50000)