def test_create_metadata_should_succeed(self):
tokenizer_md = metadata_info.BertTokenizerMd(self._VOCAB_FILE)
tokenizer_metadata = tokenizer_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_process_uint(tokenizer_metadata))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_convert_to_json_should_succeed(self):
metadata_buf = _read_file(self._metadata_file_with_version)
metadata_json = _metadata.convert_to_json(metadata_buf)
# Verifies the generated json file.
golden_json_file_path = resource_loader.get_path_to_datafile(
"testdata/golden_json.json")
expected = _read_file(golden_json_file_path, "r")
self.assertEqual(metadata_json, expected)
def test_create_metadata_by_default_should_succeed(self):
audio_tensor_md = metadata_info.InputAudioTensorMd()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_tensor(
audio_tensor_md.create_metadata()))
expected_json = test_utils.load_file(
self._EXPECTED_TENSOR_DEFAULT_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self):
text_tensor_md = metadata_info.InputAudioTensorMd(
self._NAME, self._DESCRIPTION, self._SAMPLE_RATE, self._CHANNELS)
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_tensor(
text_tensor_md.create_metadata()))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self):
score_calibration_md = metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE,
self._SCORE_CALIBRATION_FILE)
score_calibration_metadata = score_calibration_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_process_uint(
score_calibration_metadata))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self):
regex_tokenizer_md = metadata_info.RegexTokenizerMd(
self._DELIM_REGEX_PATTERN, self._VOCAB_FILE)
text_tensor_md = metadata_info.InputTextTensorMd(
self._NAME, self._DESCRIPTION, regex_tokenizer_md)
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_tensor(
text_tensor_md.create_metadata()))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def get_metadata_json(self) -> str:
"""Gets the generated JSON metadata string before populated into model.
This method returns the metadata buffer before populated into the model.
More fields could be filled by MetadataPopulator, such as
min_parser_version. Use get_populated_metadata_json() if you want to get the
final metadata string.
Returns:
The generated JSON metadata string before populated into model.
"""
return _metadata.convert_to_json(bytes(self._metadata_buffer))
def testGetMetadataBufferShouldSucceed(self):
displayer = _metadata.MetadataDisplayer.with_model_file(
self._model_with_meta_file)
actual_buffer = displayer.get_metadata_buffer()
actual_json = _metadata.convert_to_json(actual_buffer)
# Verifies the generated json file.
golden_json_file_path = resource_loader.get_path_to_datafile(
"testdata/golden_json.json")
with open(golden_json_file_path, "r") as f:
expected = f.read()
self.assertEqual(actual_json, expected)
def test_create_metadata_should_succeed(self, tensor_type, golden_json):
tesnor_md = metadata_info.InputImageTensorMd(
name=self._NAME,
description=self._DESCRIPTION,
norm_mean=list(self._NORM_MEAN),
norm_std=list(self._NORM_STD),
color_space_type=self._COLOR_SPACE_TYPE,
tensor_type=tensor_type)
tensor_metadata = tesnor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata(tensor_metadata))
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self):
label_file_en = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_EN, locale="en")
label_file_cn = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_CN, locale="cn")
tesnor_md = metadata_info.CategoryTensorMd(
name=self._NAME,
description=self._DESCRIPTION,
label_files=[label_file_en, label_file_cn])
tensor_metadata = tesnor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata(tensor_metadata))
expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self, tensor_type, golden_json):
label_file_en = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_EN, locale="en")
label_file_cn = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_CN, locale="cn")
tesnor_md = metadata_info.ClassificationTensorMd(
name=self._NAME,
description=self._DESCRIPTION,
label_files=[label_file_en, label_file_cn],
tensor_type=tensor_type)
tensor_metadata = tesnor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata(tensor_metadata))
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self):
general_md = metadata_info.GeneralMd(name="model",
version="v1",
description="A ML model.",
author="TensorFlow",
licenses="Apache")
general_metadata = general_md.create_metadata()
# Create the Flatbuffers object and convert it to the json format.
builder = flatbuffers.Builder(0)
builder.Finish(general_metadata.Pack(builder),
_metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)
metadata_json = _metadata.convert_to_json(bytes(builder.Output()))
expected_json = test_utils.load_file(self._EXPECTED_GENERAL_META_JSON,
"r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self):
file_md = metadata_info.AssociatedFileMd(
file_path="label.txt",
description="The label file.",
file_type=_metadata_fb.AssociatedFileType.TENSOR_AXIS_LABELS,
locale="en")
file_metadata = file_md.create_metadata()
# Create the Flatbuffers object and convert it to the json format.
model_metadata = _metadata_fb.ModelMetadataT()
model_metadata.associatedFiles = [file_metadata]
builder = flatbuffers.Builder(0)
builder.Finish(model_metadata.Pack(builder),
_metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)
metadata_json = _metadata.convert_to_json(bytes(builder.Output()))
expected_json = test_utils.load_file(self._EXPECTED_META_JSON, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self, content_type, golden_json):
associated_file1 = metadata_info.AssociatedFileMd(
file_path=self._LABEL_FILE_EN, locale="en")
associated_file2 = metadata_info.AssociatedFileMd(
file_path=self._LABEL_FILE_CN, locale="cn")
tensor_md = metadata_info.TensorMd(
name=self._TENSOR_NAME,
description=self._TENSOR_DESCRIPTION,
min_values=[self._TENSOR_MIN],
max_values=[self._TENSOR_MAX],
content_type=content_type,
associated_files=[associated_file1, associated_file2])
tensor_metadata = tensor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata(tensor_metadata))
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)
def test_create_score_calibration_file_md_should_succeed(self):
score_calibration_md = metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE,
self._SCORE_CALIBRATION_FILE)
score_calibration_file_md = (
score_calibration_md.create_score_calibration_file_md())
file_metadata = score_calibration_file_md.create_metadata()
# Create the Flatbuffers object and convert it to the json format.
model_metadata = _metadata_fb.ModelMetadataT()
model_metadata.associatedFiles = [file_metadata]
builder = flatbuffers.Builder(0)
builder.Finish(model_metadata.Pack(builder),
_metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)
metadata_json = _metadata.convert_to_json(bytes(builder.Output()))
expected_json = test_utils.load_file(self._EXPECTED_MODEL_META_JSON,
"r")
self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self, tensor_type, golden_json):
label_file_en = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_EN, locale="en")
label_file_cn = metadata_info.LabelFileMd(
file_path=self._LABEL_FILE_CN, locale="cn")
score_calibration_md = metadata_info.ScoreCalibrationMd(
_metadata_fb.ScoreTransformationType.IDENTITY,
self._CALIBRATION_DEFAULT_SCORE, self._SCORE_CALIBRATION_FILE)
tesnor_md = metadata_info.ClassificationTensorMd(
name=self._NAME,
description=self._DESCRIPTION,
label_files=[label_file_en, label_file_cn],
tensor_type=tensor_type,
score_calibration_md=score_calibration_md)
tensor_metadata = tesnor_md.create_metadata()
metadata_json = _metadata.convert_to_json(
_create_dummy_model_metadata_with_tensor(tensor_metadata))
expected_json = test_utils.load_file(golden_json, "r")
self.assertEqual(metadata_json, expected_json)
def get_metadata_json(self) -> str:
"""Gets the generated metadata string in JSON format."""
return _metadata.convert_to_json(bytes(self._metadata_buffer))
