def test_create_from_metadata_info_by_default_should_succeed(self):
writer = bert_nl_classifier.MetadataWriter.create_from_metadata_info(
test_utils.load_file(_MODEL))
metadata_json = writer.get_metadata_json()
expected_json = test_utils.load_file(_JSON_DEFAULT, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_from_metadata_info_by_default_succeeds_for_multihead(self):
writer = (audio_classifier.MetadataWriter.
create_from_metadata_info_for_multihead(
test_utils.load_file(_MULTIHEAD_MODEL)))
metadata_json = writer.get_metadata_json()
expected_json = test_utils.load_file(_JSON_DEFAULT_MULTIHEAD, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed(self):
writer = image_segmenter.MetadataWriter.create_for_inference(
test_utils.load_file(_MODEL), [_NORM_MEAN], [_NORM_STD],
[_LABEL_FILE])
metadata_json = writer.get_metadata_json()
expected_json = test_utils.load_file(_JSON_FOR_INFERENCE, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed(self, model_file,
golden_json):
writer = image_classifier.MetadataWriter.create_for_inference(
test_utils.load_file(model_file), [_NORM_MEAN], [_NORM_STD],
[_LABEL_FILE])
metadata_json = writer.get_metadata_json()
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_with_bert_should_succeed(self):
writer = bert_nl_classifier.MetadataWriter.create_for_inference(
test_utils.load_file(_MODEL),
metadata_info.BertTokenizerMd(_VOCAB_FILE), [_LABEL_FILE])
displayer = _metadata.MetadataDisplayer.with_model_buffer(writer.populate())
metadata_json = displayer.get_metadata_json()
expected_json = test_utils.load_file(_JSON_FOR_INFERENCE_WITH_BERT, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed_dynaamic_input_shape_model(self):
writer = audio_classifier.MetadataWriter.create_for_inference(
test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), _SAMPLE_RATE,
_CHANNELS, [_LABEL_FILE],
metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG,
_DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE))
metadata_json = writer.get_metadata_json()
expected_json = test_utils.load_file(_JSON_FOR_INFERENCE_DYNAMIC, "r")
self.assertEqual(metadata_json, expected_json)
def test_get_metadata_json_should_succeed(self):
model_buffer = test_utils.load_file(_MODEL)
model_metadata, input_metadata, output_metadata = (
self._create_dummy_metadata())
writer = metadata_writer.MetadataWriter.create_from_metadata(
model_buffer, model_metadata, [input_metadata], [output_metadata],
[_LABEL_FILE])
metadata_json = writer.get_metadata_json()
expected_json = test_utils.load_file(_EXPECTED_DUMMY_NO_VERSION_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed(self, model_file,
golden_json):
writer = image_classifier.MetadataWriter.create_for_inference(
test_utils.load_file(model_file), [_NORM_MEAN], [_NORM_STD],
[_LABEL_FILE],
metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG,
_DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE))
metadata_json = writer.get_metadata_json()
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)
def test_initialize_and_populate(self):
with mt.Writer(test_utils.load_file(_AUDIO_CLASSIFICATION_MODEL),
model_name='my_audio_model',
model_description='my_description') as writer:
out_dir = self.create_tempdir()
_, metadata_json = writer.populate(
os.path.join(out_dir, 'model.tflite'),
os.path.join(out_dir, 'metadata.json'))
self.assertJsonEqual(
metadata_json, """{
"name": "my_audio_model",
"description": "my_description",
"subgraph_metadata": [
{
"input_tensor_metadata": [
{
"name": "waveform_binary"
}
],
"output_tensor_metadata": [
{
"name": "tower0/network/layer32/final_output"
}
]
}
],
"min_parser_version": "1.0.0"
}
""")
def test_create_from_metadata_with_default_value_should_succeed(self):
model_buffer = test_utils.load_file(_MODEL)
writer = metadata_writer.MetadataWriter.create_from_metadata(model_buffer)
model_with_metadata = writer.populate()
self._assert_correct_metadata(model_with_metadata, _EXPECTED_DEFAULT_JSON)
def test_create_metadata_should_succeed(self):
tokenizer_md = metadata_info.BertTokenizerMd(self._VOCAB_FILE)
tokenizer_metadata = tokenizer_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_process_uint(tokenizer_metadata))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_from_metadata_info_by_default_should_succeed(
self, model_name: str):
model_path = os.path.join(_PATH, model_name + ".tflite")
writer = object_detector.MetadataWriter.create_from_metadata_info(
test_utils.load_file(model_path))
json_path = os.path.join(_PATH, model_name + "_default.json")
self._validate_metadata(writer, json_path)
self._validate_populated_model(writer)
def test_create_for_inference_should_succeed(self, model_name):
model_path = os.path.join(_PATH, model_name + ".tflite")
writer = object_detector.MetadataWriter.create_for_inference(
test_utils.load_file(model_path), [_NORM_MEAN], [_NORM_STD],
[self._label_file])
json_path = os.path.join(_PATH, model_name + ".json")
self._validate_metadata(writer, json_path)
self._validate_populated_model(writer)
def test_create_metadata_by_default_should_succeed(self):
audio_tensor_md = metadata_info.InputAudioTensorMd()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_tensor(
audio_tensor_md.create_metadata()))
expected_json = test_utils.load_file(
self._EXPECTED_TENSOR_DEFAULT_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self):
text_tensor_md = metadata_info.InputAudioTensorMd(
self._NAME, self._DESCRIPTION, self._SAMPLE_RATE, self._CHANNELS)
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_tensor(
text_tensor_md.create_metadata()))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_dummy_score_calibration_should_succeed(self):
score_calibration_md = metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.INVERSE_LOGISTIC,
_DUMMY_SCORE_CALIBRATION_DEFAULT_SCORE,
self._dummy_score_file,
)
writer = object_detector.MetadataWriter.create_for_inference(
test_utils.load_file(_MODEL_COCO), [_NORM_MEAN], [_NORM_STD],
[self._label_file], score_calibration_md)
self._validate_metadata(writer, _JSON_FOR_DUMMY_SCORE_CALIBRATION)
self._validate_populated_model(writer)
# Test if populated model is equivalent to the expected model.
metadata_dict = json.loads(writer.get_metadata_json())
displayer = metadata.MetadataDisplayer.with_model_buffer(
test_utils.load_file(_EXPECTED_DUMMY_MODEL))
expected_metadata_dict = json.loads(displayer.get_metadata_json())
self.assertDictContainsSubset(metadata_dict, expected_metadata_dict)
def test_create_from_metadata_info_fails_with_wrong_output_tesnor_name(self):
model_buffer = test_utils.load_file(_MODEL)
output_md = metadata_info.TensorMd(tensor_name="wrong_tensor_name")
with self.assertRaises(ValueError) as error:
metadata_writer.MetadataWriter.create_from_metadata_info(
model_buffer, output_md=[output_md])
self.assertEqual(
"The tensor names from arguments (['wrong_tensor_name']) do not match"
" the tensor names read from the model (['output']).",
str(error.exception))
def test_create_metadata_should_succeed(self):
score_calibration_md = metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE,
self._SCORE_CALIBRATION_FILE)
score_calibration_metadata = score_calibration_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_process_uint(
score_calibration_metadata))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_score_calibration_should_succeed(self):
score_calibration_md = metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.INVERSE_LOGISTIC,
_SCORE_CALIBRATION_DEFAULT_SCORE,
self._score_file,
)
writer = object_detector.MetadataWriter.create_for_inference(
test_utils.load_file(_MODEL_COCO), [_NORM_MEAN], [_NORM_STD],
[self._label_file], score_calibration_md)
self._validate_metadata(writer, _JSON_FOR_SCORE_CALIBRATION)
self._validate_populated_model(writer)
def test_create_from_metadata_info_succeeds_for_multihead(self):
calibration_file1 = test_utils.create_calibration_file(
self.get_temp_dir(), "score_cali_1.txt")
calibration_file2 = test_utils.create_calibration_file(
self.get_temp_dir(), "score_cali_2.txt")
general_md = metadata_info.GeneralMd(name="AudioClassifier")
input_md = metadata_info.InputAudioTensorMd(
name="audio_clip", sample_rate=_SAMPLE_RATE, channels=_CHANNELS)
# The output tensors in the model are: Identity, Identity_1
# Create metadata in a different order to test if MetadataWriter can correct
# it.
output_head_md_1 = metadata_info.ClassificationTensorMd(
name="head1",
label_files=[
metadata_info.LabelFileMd("labels_en_1.txt"),
metadata_info.LabelFileMd("labels_cn_1.txt")
],
score_calibration_md=metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG,
_DEFAULT_SCORE_CALIBRATION_VALUE, calibration_file1),
tensor_name="Identity_1")
output_head_md_2 = metadata_info.ClassificationTensorMd(
name="head2",
label_files=[
metadata_info.LabelFileMd("labels_en_2.txt"),
metadata_info.LabelFileMd("labels_cn_2.txt")
],
score_calibration_md=metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG,
_DEFAULT_SCORE_CALIBRATION_VALUE, calibration_file2),
tensor_name="Identity")
writer = (
audio_classifier.MetadataWriter.create_from_metadata_info_for_multihead(
test_utils.load_file(_MULTIHEAD_MODEL), general_md, input_md,
[output_head_md_1, output_head_md_2]))
metadata_json = writer.get_metadata_json()
expected_json = test_utils.load_file(_JSON_MULTIHEAD, "r")
self.assertEqual(metadata_json, expected_json)
def test_populate_from_metadata_should_succeed(self):
model_buffer = test_utils.load_file(_MODEL)
model_metadata, input_metadata, output_metadata = (
self._create_dummy_metadata())
writer = metadata_writer.MetadataWriter.create_from_metadata(
model_buffer, model_metadata, [input_metadata], [output_metadata],
[_LABEL_FILE])
model_with_metadata = writer.populate()
self._assert_correct_metadata(model_with_metadata,
_EXPECTED_DUMMY_JSON)
def test_create_metadata_should_succeed(self):
regex_tokenizer_md = metadata_info.RegexTokenizerMd(
self._DELIM_REGEX_PATTERN, self._VOCAB_FILE)
text_tensor_md = metadata_info.InputTextTensorMd(
self._NAME, self._DESCRIPTION, regex_tokenizer_md)
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_tensor(
text_tensor_md.create_metadata()))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_populate_create_from_metadata_info_should_succeed(self):
model_buffer = test_utils.load_file(_MODEL)
general_md = metadata_info.GeneralMd(name=_MODEL_NAME)
input_md = metadata_info.TensorMd(name=_INPUT_NAME)
output_md = metadata_info.TensorMd(name=_OUTPUT_NAME)
writer = metadata_writer.MetadataWriter.create_from_metadata_info(
model_buffer, general_md, [input_md], [output_md], [_LABEL_FILE])
model_with_metadata = writer.populate()
self._assert_correct_metadata(model_with_metadata,
_EXPECTED_META_INFO_JSON)
def _assert_correct_metadata(self, model_with_metadata,
expected_json_file):
# Verify if the metadata populated is correct.
displayer = _metadata.MetadataDisplayer.with_model_buffer(
model_with_metadata)
metadata_json = displayer.get_metadata_json()
expected_json = test_utils.load_file(expected_json_file, "r")
self.assertEqual(metadata_json, expected_json)
# Verify if the associated file is packed as expected.
packed_files = displayer.get_packed_associated_file_list()
expected_packed_files = [os.path.basename(_LABEL_FILE)]
self.assertEqual(set(packed_files), set(expected_packed_files))
def test_create_for_inference_fails_with_wrong_channels(self, wrong_channels):
with self.assertRaises(ValueError) as error:
audio_classifier.MetadataWriter.create_for_inference(
test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), _SAMPLE_RATE,
wrong_channels, [_LABEL_FILE],
metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG,
_DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE))
self.assertEqual(
"channels should be positive, but got {}.".format(wrong_channels),
str(error.exception))
def test_audio_embedder(self):
with mt.Writer(
test_utils.load_file(_AUDIO_EMBEDDING_MODEL),
model_name='audio_embedder',
model_description='Generate embedding for the input audio clip'
) as writer:
out_dir = self.create_tempdir()
writer.add_audio_input(sample_rate=16000, channels=1)
writer.add_embedding_output()
_, metadata_json = writer.populate(
os.path.join(out_dir, 'model.tflite'),
os.path.join(out_dir, 'metadata.json'))
self.assertEqual(
metadata_json, """{
"name": "audio_embedder",
"description": "Generate embedding for the input audio clip",
"subgraph_metadata": [
{
"input_tensor_metadata": [
{
"name": "audio",
"description": "Input audio clip to be processed.",
"content": {
"content_properties_type": "AudioProperties",
"content_properties": {
"sample_rate": 16000,
"channels": 1
}
},
"stats": {
}
}
],
"output_tensor_metadata": [
{
"name": "embedding",
"description": "Embedding vector of the input.",
"content": {
"content_properties_type": "FeatureProperties",
"content_properties": {
}
},
"stats": {
}
}
]
}
],
"min_parser_version": "1.3.0"
}
""")
def test_create_metadata_should_succeed(self, tensor_type, golden_json):
tesnor_md = metadata_info.InputImageTensorMd(
name=self._NAME,
description=self._DESCRIPTION,
norm_mean=list(self._NORM_MEAN),
norm_std=list(self._NORM_STD),
color_space_type=self._COLOR_SPACE_TYPE,
tensor_type=tensor_type)
tensor_metadata = tesnor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata(tensor_metadata))
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self):
label_file_en = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_EN, locale="en")
label_file_cn = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_CN, locale="cn")
tesnor_md = metadata_info.CategoryTensorMd(
name=self._NAME,
description=self._DESCRIPTION,
label_files=[label_file_en, label_file_cn])
tensor_metadata = tesnor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata(tensor_metadata))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_fails_with_wrong_sample_rate(
self, wrong_sample_rate):
with self.assertRaises(ValueError) as error:
audio_classifier.MetadataWriter.create_for_inference(
test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), wrong_sample_rate,
_CHANNELS, [_LABEL_FILE],
metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG,
_DEFAULT_SCORE_CALIBRATION_VALUE,
test_utils.create_calibration_file(self.get_temp_dir())))
self.assertEqual(
"sample_rate should be positive, but got {}.".format(wrong_sample_rate),
str(error.exception))
def test_create_metadata_should_succeed(self, tensor_type, golden_json):
label_file_en = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_EN, locale="en")
label_file_cn = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_CN, locale="cn")
tesnor_md = metadata_info.ClassificationTensorMd(
name=self._NAME,
description=self._DESCRIPTION,
label_files=[label_file_en, label_file_cn],
tensor_type=tensor_type)
tensor_metadata = tesnor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata(tensor_metadata))
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)