def create_for_inference(cls,
model_buffer: bytearray,
input_norm_mean: List[float],
input_norm_std: List[float],
label_file_paths: List[str],
score_calibration_md: Optional[
metadata_info.ScoreCalibrationMd] = None):
"""Creates mandatory metadata for TFLite Support inference.
The parameters required in this method are mandatory when using TFLite
Support features, such as Task library and Codegen tool (Android Studio ML
Binding). Other metadata fields will be set to default. If other fields need
to be filled, use the method `create_from_metadata_info` to edit them.
Args:
model_buffer: valid buffer of the model file.
input_norm_mean: the mean value used in the input tensor normalization
[1].
input_norm_std: the std value used in the input tensor normalizarion [1].
label_file_paths: paths to the label files [2] in the category tensor.
Pass in an empty list, If the model does not have any label file.
score_calibration_md: information of the score calibration operation [3]
in the classification tensor. Optional if the model does not use score
calibration.
[1]:
https://www.tensorflow.org/lite/convert/metadata#normalization_and_quantization_parameters
[2]:
https://github.com/tensorflow/tflite-support/blob/b80289c4cd1224d0e1836c7654e82f070f9eefaa/tensorflow_lite_support/metadata/metadata_schema.fbs#L108
[3]:
https://github.com/tensorflow/tflite-support/blob/5e0cdf5460788c481f5cd18aab8728ec36cf9733/tensorflow_lite_support/metadata/metadata_schema.fbs#L434
Returns:
A MetadataWriter object.
"""
input_md = metadata_info.InputImageTensorMd(
name=_INPUT_NAME,
description=_INPUT_DESCRIPTION,
norm_mean=input_norm_mean,
norm_std=input_norm_std,
color_space_type=_metadata_fb.ColorSpaceType.RGB,
tensor_type=writer_utils.get_input_tensor_types(model_buffer)[0])
output_category_md = metadata_info.CategoryTensorMd(
name=_OUTPUT_CATRGORY_NAME,
description=_OUTPUT_CATEGORY_DESCRIPTION,
label_files=[
metadata_info.LabelFileMd(file_path=file_path)
for file_path in label_file_paths
])
output_score_md = metadata_info.ClassificationTensorMd(
name=_OUTPUT_SCORE_NAME,
description=_OUTPUT_SCORE_DESCRIPTION,
score_calibration_md=score_calibration_md)
return cls.create_from_metadata_info(
model_buffer,
input_md=input_md,
output_category_md=output_category_md,
output_score_md=output_score_md)
def test_init_should_throw_exception_with_incompatible_mean_and_std(self):
norm_mean = [0]
norm_std = [1, 2]
with self.assertRaises(ValueError) as error:
metadata_info.InputImageTensorMd(norm_mean=norm_mean,
norm_std=norm_std)
self.assertEqual(
f"norm_mean and norm_std are expected to be the same dim. But got "
f"{len(norm_mean)} and {len(norm_std)}", str(error.exception))
def test_init_should_throw_exception_with_incompatible_mean_and_std(self):
norm_mean = [0]
norm_std = [1, 2]
with self.assertRaises(ValueError) as error:
metadata_info.InputImageTensorMd(norm_mean=norm_mean,
norm_std=norm_std)
# TODO(b/175843689): Python version cannot be specified in Kokoro bazel test
self.assertEqual(
"norm_mean and norm_std are expected to be the same dim. But got "
+ "{} and {}".format(len(norm_mean), len(norm_std)),
str(error.exception))
def test_create_metadata_should_succeed(self, tensor_type, golden_json):
tesnor_md = metadata_info.InputImageTensorMd(
name=self._NAME,
description=self._DESCRIPTION,
norm_mean=list(self._NORM_MEAN),
norm_std=list(self._NORM_STD),
color_space_type=self._COLOR_SPACE_TYPE,
tensor_type=tensor_type)
tensor_metadata = tesnor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata(tensor_metadata))
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)
def create_from_metadata_info(
cls,
model_buffer: bytearray,
general_md: Optional[metadata_info.GeneralMd] = None,
input_md: Optional[metadata_info.InputImageTensorMd] = None,
output_md: Optional[metadata_info.TensorMd] = None):
"""Creates MetadataWriter based on general/input/outputs information.
Args:
model_buffer: valid buffer of the model file.
general_md: general infromation about the model.
input_md: input image tensor informaton.
output_md: output segmentation mask tensor informaton. This tensor is a
multidimensional array of [1 x mask_height x mask_width x num_classes],
where mask_width and mask_height are the dimensions of the segmentation
masks produced by the model, and num_classes is the number of classes
supported by the model.
Returns:
A MetadataWriter object.
"""
if general_md is None:
general_md = metadata_info.GeneralMd(
name=_MODEL_NAME, description=_MODEL_DESCRIPTION)
if input_md is None:
input_md = metadata_info.InputImageTensorMd(
name=_INPUT_NAME,
description=_INPUT_DESCRIPTION,
color_space_type=_metadata_fb.ColorSpaceType.RGB)
if output_md is None:
output_md = metadata_info.TensorMd(name=_OUTPUT_NAME,
description=_OUTPUT_DESCRIPTION)
if output_md.associated_files is None:
output_md.associated_files = []
return super().create_from_metadata(
model_buffer,
model_metadata=general_md.create_metadata(),
input_metadata=[input_md.create_metadata()],
output_metadata=[_create_segmentation_masks_metadata(output_md)],
associated_files=[
file.file_path for file in output_md.associated_files
])
def create_from_metadata_info(
cls,
model_buffer: bytearray,
general_md: Optional[metadata_info.GeneralMd] = None,
input_md: Optional[metadata_info.InputImageTensorMd] = None,
output_md: Optional[metadata_info.ClassificationTensorMd] = None):
"""Creates MetadataWriter based on general/input/output information.
Args:
model_buffer: valid buffer of the model file.
general_md: general infromation about the model. If not specified, default
general metadata will be generated.
input_md: input image tensor informaton, if not specified, default input
metadata will be generated.
output_md: output classification tensor informaton, if not specified,
default output metadata will be generated.
Returns:
A MetadataWriter object.
"""
if general_md is None:
general_md = metadata_info.GeneralMd(
name=_MODEL_NAME, description=_MODEL_DESCRIPTION)
if input_md is None:
input_md = metadata_info.InputImageTensorMd(
name=_INPUT_NAME,
description=_INPUT_DESCRIPTION,
color_space_type=_metadata_fb.ColorSpaceType.RGB)
if output_md is None:
output_md = metadata_info.ClassificationTensorMd(
name=_OUTPUT_NAME, description=_OUTPUT_DESCRIPTION)
if output_md.associated_files is None:
output_md.associated_files = []
return super().create_from_metadata_info(
model_buffer=model_buffer,
general_md=general_md,
input_md=[input_md],
output_md=[output_md],
associated_files=[
file.file_path for file in output_md.associated_files
])
def add_image_input(
self,
norm_mean: List[float],
norm_std: List[float],
color_space_type: Optional[
_metadata_fb.ColorSpaceType] = _metadata_fb.ColorSpaceType.RGB,
name: str = _INPUT_IMAGE_NAME,
description: str = _INPUT_IMAGE_DESCRIPTION):
"""Marks the next input tensor as an image input.
Args:
norm_mean: The mean value used to normalize each input channel. If there
is only one element in the list, its value will be broadcasted to all
channels. Also note that norm_mean and norm_std should have the same
number of elements. [1]
norm_std: The std value used to normalize each input channel. If there is
only one element in the list, its value will be broadcasted to all
channels. [1]
color_space_type: The color space type of the input image. [2]
name: Name of the input tensor.
description: Description of the input tensor.
Returns:
The Writer instance, can be used for chained operation.
[1]:
https://www.tensorflow.org/lite/convert/metadata#normalization_and_quantization_parameters
[2]:
https://github.com/tensorflow/tflite-support/blob/b80289c4cd1224d0e1836c7654e82f070f9eefaa/tensorflow_lite_support/metadata/metadata_schema.fbs#L172
"""
input_md = metadata_info.InputImageTensorMd(
name=name,
description=description,
norm_mean=norm_mean,
norm_std=norm_std,
color_space_type=color_space_type,
tensor_type=self._input_tensor_type(len(self._input_mds)))
self._input_mds.append(input_md)
return self
def create_from_metadata_info(
cls,
model_buffer: bytearray,
general_md: Optional[metadata_info.GeneralMd] = None,
input_md: Optional[metadata_info.InputImageTensorMd] = None,
output_location_md: Optional[metadata_info.TensorMd] = None,
output_category_md: Optional[
metadata_info.CategoryTensorMd] = None,
output_score_md: Union[
None, metadata_info.TensorMd,
metadata_info.ClassificationTensorMd] = None,
output_number_md: Optional[metadata_info.TensorMd] = None):
"""Creates MetadataWriter based on general/input/outputs information.
Args:
model_buffer: valid buffer of the model file.
general_md: general infromation about the model.
input_md: input image tensor informaton.
output_location_md: output location tensor informaton. The location tensor
is a multidimensional array of [N][4] floating point values between 0
and 1, the inner arrays representing bounding boxes in the form [top,
left, bottom, right].
output_category_md: output category tensor information. The category
tensor is an array of N integers (output as floating point values) each
indicating the index of a class label from the labels file.
output_score_md: output score tensor information. The score tensor is an
array of N floating point values between 0 and 1 representing
probability that a class was detected. Use ClassificationTensorMd to
calibrate score.
output_number_md: output number of detections tensor information. This
tensor is an integer value of N.
Returns:
A MetadataWriter object.
"""
if general_md is None:
general_md = metadata_info.GeneralMd(
name=_MODEL_NAME, description=_MODEL_DESCRIPTION)
if input_md is None:
input_md = metadata_info.InputImageTensorMd(
name=_INPUT_NAME,
description=_INPUT_DESCRIPTION,
color_space_type=_metadata_fb.ColorSpaceType.RGB)
warn_message_format = (
"The output name isn't the default string \"%s\". This may cause the "
"model not work in the TFLite Task Library since the tensor name will "
"be used to handle the output order in the TFLite Task Library.")
if output_location_md is None:
output_location_md = metadata_info.TensorMd(
name=_OUTPUT_LOCATION_NAME,
description=_OUTPUT_LOCATION_DESCRIPTION)
elif output_location_md.name != _OUTPUT_LOCATION_NAME:
logging.warning(warn_message_format, _OUTPUT_LOCATION_NAME)
if output_category_md is None:
output_category_md = metadata_info.CategoryTensorMd(
name=_OUTPUT_CATRGORY_NAME,
description=_OUTPUT_CATEGORY_DESCRIPTION)
elif output_category_md.name != _OUTPUT_CATRGORY_NAME:
logging.warning(warn_message_format, _OUTPUT_CATRGORY_NAME)
if output_score_md is None:
output_score_md = metadata_info.ClassificationTensorMd(
name=_OUTPUT_SCORE_NAME,
description=_OUTPUT_SCORE_DESCRIPTION,
)
elif output_score_md.name != _OUTPUT_SCORE_NAME:
logging.warning(warn_message_format, _OUTPUT_SCORE_NAME)
if output_number_md is None:
output_number_md = metadata_info.TensorMd(
name=_OUTPUT_NUMBER_NAME,
description=_OUTPUT_NUMBER_DESCRIPTION)
elif output_number_md.name != _OUTPUT_NUMBER_NAME:
logging.warning(warn_message_format, _OUTPUT_NUMBER_NAME)
# Create output tensor group info.
group = _metadata_fb.TensorGroupT()
group.name = _GROUP_NAME
group.tensorNames = [
output_location_md.name, output_category_md.name,
output_score_md.name
]
# Gets the tensor inidces of tflite outputs and then gets the order of the
# output metadata by the value of tensor indices. For instance, if the
# output indices are [601, 599, 598, 600], tensor names and indices aligned
# are:
# - location: 598
# - category: 599
# - score: 600
# - number of detections: 601
# because of the op's ports of TFLITE_DETECTION_POST_PROCESS
# (https://github.com/tensorflow/tensorflow/blob/a4fe268ea084e7d323133ed7b986e0ae259a2bc7/tensorflow/lite/kernels/detection_postprocess.cc#L47-L50).
# Thus, the metadata of tensors are sorted in this way, according to
# output_tensor_indicies correctly.
output_tensor_indices = _get_tflite_outputs(model_buffer)
metadata_list = [
_create_location_metadata(output_location_md),
_create_metadata_with_value_range(output_category_md),
_create_metadata_with_value_range(output_score_md),
output_number_md.create_metadata()
]
# Align indices with tensors.
sorted_indices = sorted(output_tensor_indices)
indices_to_tensors = dict(zip(sorted_indices, metadata_list))
# Output metadata according to output_tensor_indices.
output_metadata = [
indices_to_tensors[i] for i in output_tensor_indices
]
# Create subgraph info.
subgraph_metadata = _metadata_fb.SubGraphMetadataT()
subgraph_metadata.inputTensorMetadata = [input_md.create_metadata()]
subgraph_metadata.outputTensorMetadata = output_metadata
subgraph_metadata.outputTensorGroups = [group]
# Create model metadata
model_metadata = general_md.create_metadata()
model_metadata.subgraphMetadata = [subgraph_metadata]
b = flatbuffers.Builder(0)
b.Finish(model_metadata.Pack(b),
_metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)
associated_files = []
_extend_new_files(associated_files,
output_category_md.associated_files)
_extend_new_files(associated_files, output_score_md.associated_files)
return cls(model_buffer, b.Output(), associated_files=associated_files)
def create_from_metadata_info(
cls,
model_buffer: bytearray,
general_md: Optional[metadata_info.GeneralMd] = None,
input_md: Optional[metadata_info.InputImageTensorMd] = None,
output_location_md: Optional[metadata_info.TensorMd] = None,
output_category_md: Optional[metadata_info.CategoryTensorMd] = None,
output_score_md: Optional[metadata_info.TensorMd] = None,
output_number_md: Optional[metadata_info.TensorMd] = None):
"""Creates MetadataWriter based on general/input/outputs information.
Args:
model_buffer: valid buffer of the model file.
general_md: general infromation about the model.
input_md: input image tensor informaton.
output_location_md: output location tensor informaton. The location tensor
is a multidimensional array of [N][4] floating point values between 0
and 1, the inner arrays representing bounding boxes in the form [top,
left, bottom, right].
output_category_md: output category tensor information. The category
tensor is an array of N integers (output as floating point values) each
indicating the index of a class label from the labels file.
output_score_md: output score tensor information. The score tensor is an
array of N floating point values between 0 and 1 representing
probability that a class was detected.
output_number_md: output number of dections tensor information. This
tensor is an integer value of N.
Returns:
A MetadataWriter object.
"""
if general_md is None:
general_md = metadata_info.GeneralMd(
name=_MODEL_NAME, description=_MODEL_DESCRIPTION)
if input_md is None:
input_md = metadata_info.InputImageTensorMd(
name=_INPUT_NAME,
description=_INPUT_DESCRIPTION,
color_space_type=_metadata_fb.ColorSpaceType.RGB)
if output_location_md is None:
output_location_md = metadata_info.TensorMd(
name=_OUTPUT_LOCATION_NAME, description=_OUTPUT_LOCATION_DESCRIPTION)
if output_category_md is None:
output_category_md = metadata_info.CategoryTensorMd(
name=_OUTPUT_CATRGORY_NAME, description=_OUTPUT_CATEGORY_DESCRIPTION)
if output_score_md is None:
output_score_md = metadata_info.TensorMd(
name=_OUTPUT_SCORE_NAME, description=_OUTPUT_SCORE_DESCRIPTION)
if output_number_md is None:
output_number_md = metadata_info.TensorMd(
name=_OUTPUT_NUMBER_NAME, description=_OUTPUT_NUMBER_DESCRIPTION)
if output_category_md.associated_files is None:
output_category_md.associated_files = []
return super().create_from_metadata(
model_buffer,
model_metadata=general_md.create_metadata(),
input_metadata=[input_md.create_metadata()],
output_metadata=[
_create_location_metadata(output_location_md),
_create_metadata_with_value_range(output_category_md),
_create_metadata_with_value_range(output_score_md),
output_number_md.create_metadata()
],
associated_files=[
file.file_path for file in output_category_md.associated_files
])
def create_from_metadata_info(
cls,
model_buffer: bytearray,
general_md: Optional[metadata_info.GeneralMd] = None,
input_md: Optional[metadata_info.InputImageTensorMd] = None,
output_location_md: Optional[metadata_info.TensorMd] = None,
output_category_md: Optional[metadata_info.CategoryTensorMd] = None,
output_score_md: Optional[metadata_info.TensorMd] = None,
output_number_md: Optional[metadata_info.TensorMd] = None):
"""Creates MetadataWriter based on general/input/outputs information.
Args:
model_buffer: valid buffer of the model file.
general_md: general infromation about the model.
input_md: input image tensor informaton.
output_location_md: output location tensor informaton. The location tensor
is a multidimensional array of [N][4] floating point values between 0
and 1, the inner arrays representing bounding boxes in the form [top,
left, bottom, right].
output_category_md: output category tensor information. The category
tensor is an array of N integers (output as floating point values) each
indicating the index of a class label from the labels file.
output_score_md: output score tensor information. The score tensor is an
array of N floating point values between 0 and 1 representing
probability that a class was detected.
output_number_md: output number of dections tensor information. This
tensor is an integer value of N.
Returns:
A MetadataWriter object.
"""
if general_md is None:
general_md = metadata_info.GeneralMd(
name=_MODEL_NAME, description=_MODEL_DESCRIPTION)
if input_md is None:
input_md = metadata_info.InputImageTensorMd(
name=_INPUT_NAME,
description=_INPUT_DESCRIPTION,
color_space_type=_metadata_fb.ColorSpaceType.RGB)
if output_location_md is None:
output_location_md = metadata_info.TensorMd(
name=_OUTPUT_LOCATION_NAME, description=_OUTPUT_LOCATION_DESCRIPTION)
if output_category_md is None:
output_category_md = metadata_info.CategoryTensorMd(
name=_OUTPUT_CATRGORY_NAME, description=_OUTPUT_CATEGORY_DESCRIPTION)
if output_score_md is None:
output_score_md = metadata_info.TensorMd(
name=_OUTPUT_SCORE_NAME, description=_OUTPUT_SCORE_DESCRIPTION)
if output_number_md is None:
output_number_md = metadata_info.TensorMd(
name=_OUTPUT_NUMBER_NAME, description=_OUTPUT_NUMBER_DESCRIPTION)
if output_category_md.associated_files is None:
output_category_md.associated_files = []
# Create output tensor group info.
group = _metadata_fb.TensorGroupT()
group.name = _GROUP_NAME
group.tensorNames = [
output_location_md.name, output_category_md.name, output_score_md.name
]
# Create subgraph info.
subgraph_metadata = _metadata_fb.SubGraphMetadataT()
subgraph_metadata.inputTensorMetadata = [input_md.create_metadata()]
subgraph_metadata.outputTensorMetadata = [
_create_location_metadata(output_location_md),
_create_metadata_with_value_range(output_category_md),
_create_metadata_with_value_range(output_score_md),
output_number_md.create_metadata()
]
subgraph_metadata.outputTensorGroups = [group]
# Create model metadata
model_metadata = general_md.create_metadata()
model_metadata.subgraphMetadata = [subgraph_metadata]
b = flatbuffers.Builder(0)
b.Finish(
model_metadata.Pack(b),
_metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)
return cls(
model_buffer,
b.Output(),
associated_files=[
file.file_path for file in output_category_md.associated_files
])
