def test_create_from_metadata_info_by_default_should_succeed(self): writer = bert_nl_classifier.MetadataWriter.create_from_metadata_info( test_utils.load_file(_MODEL)) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(_JSON_DEFAULT, "r") self.assertEqual(metadata_json, expected_json)
def test_create_from_metadata_info_by_default_succeeds_for_multihead(self): writer = (audio_classifier.MetadataWriter. create_from_metadata_info_for_multihead( test_utils.load_file(_MULTIHEAD_MODEL))) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(_JSON_DEFAULT_MULTIHEAD, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed(self): writer = image_segmenter.MetadataWriter.create_for_inference( test_utils.load_file(_MODEL), [_NORM_MEAN], [_NORM_STD], [_LABEL_FILE]) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(_JSON_FOR_INFERENCE, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed(self, model_file, golden_json): writer = image_classifier.MetadataWriter.create_for_inference( test_utils.load_file(model_file), [_NORM_MEAN], [_NORM_STD], [_LABEL_FILE]) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(golden_json, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_with_bert_should_succeed(self): writer = bert_nl_classifier.MetadataWriter.create_for_inference( test_utils.load_file(_MODEL), metadata_info.BertTokenizerMd(_VOCAB_FILE), [_LABEL_FILE]) displayer = _metadata.MetadataDisplayer.with_model_buffer(writer.populate()) metadata_json = displayer.get_metadata_json() expected_json = test_utils.load_file(_JSON_FOR_INFERENCE_WITH_BERT, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed_dynaamic_input_shape_model(self): writer = audio_classifier.MetadataWriter.create_for_inference( test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), _SAMPLE_RATE, _CHANNELS, [_LABEL_FILE], metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE)) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(_JSON_FOR_INFERENCE_DYNAMIC, "r") self.assertEqual(metadata_json, expected_json)
def test_get_metadata_json_should_succeed(self): model_buffer = test_utils.load_file(_MODEL) model_metadata, input_metadata, output_metadata = ( self._create_dummy_metadata()) writer = metadata_writer.MetadataWriter.create_from_metadata( model_buffer, model_metadata, [input_metadata], [output_metadata], [_LABEL_FILE]) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(_EXPECTED_DUMMY_NO_VERSION_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed(self, model_file, golden_json): writer = image_classifier.MetadataWriter.create_for_inference( test_utils.load_file(model_file), [_NORM_MEAN], [_NORM_STD], [_LABEL_FILE], metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE)) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(golden_json, "r") self.assertEqual(metadata_json, expected_json)
def test_initialize_and_populate(self): with mt.Writer(test_utils.load_file(_AUDIO_CLASSIFICATION_MODEL), model_name='my_audio_model', model_description='my_description') as writer: out_dir = self.create_tempdir() _, metadata_json = writer.populate( os.path.join(out_dir, 'model.tflite'), os.path.join(out_dir, 'metadata.json')) self.assertJsonEqual( metadata_json, """{ "name": "my_audio_model", "description": "my_description", "subgraph_metadata": [ { "input_tensor_metadata": [ { "name": "waveform_binary" } ], "output_tensor_metadata": [ { "name": "tower0/network/layer32/final_output" } ] } ], "min_parser_version": "1.0.0" } """)
def test_create_from_metadata_with_default_value_should_succeed(self): model_buffer = test_utils.load_file(_MODEL) writer = metadata_writer.MetadataWriter.create_from_metadata(model_buffer) model_with_metadata = writer.populate() self._assert_correct_metadata(model_with_metadata, _EXPECTED_DEFAULT_JSON)
def test_create_metadata_should_succeed(self): tokenizer_md = metadata_info.BertTokenizerMd(self._VOCAB_FILE) tokenizer_metadata = tokenizer_md.create_metadata() metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata_with_process_uint(tokenizer_metadata)) expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_create_from_metadata_info_by_default_should_succeed( self, model_name: str): model_path = os.path.join(_PATH, model_name + ".tflite") writer = object_detector.MetadataWriter.create_from_metadata_info( test_utils.load_file(model_path)) json_path = os.path.join(_PATH, model_name + "_default.json") self._validate_metadata(writer, json_path) self._validate_populated_model(writer)
def test_create_for_inference_should_succeed(self, model_name): model_path = os.path.join(_PATH, model_name + ".tflite") writer = object_detector.MetadataWriter.create_for_inference( test_utils.load_file(model_path), [_NORM_MEAN], [_NORM_STD], [self._label_file]) json_path = os.path.join(_PATH, model_name + ".json") self._validate_metadata(writer, json_path) self._validate_populated_model(writer)
def test_create_metadata_by_default_should_succeed(self): audio_tensor_md = metadata_info.InputAudioTensorMd() metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata_with_tensor( audio_tensor_md.create_metadata())) expected_json = test_utils.load_file( self._EXPECTED_TENSOR_DEFAULT_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self): text_tensor_md = metadata_info.InputAudioTensorMd( self._NAME, self._DESCRIPTION, self._SAMPLE_RATE, self._CHANNELS) metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata_with_tensor( text_tensor_md.create_metadata())) expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_dummy_score_calibration_should_succeed(self): score_calibration_md = metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.INVERSE_LOGISTIC, _DUMMY_SCORE_CALIBRATION_DEFAULT_SCORE, self._dummy_score_file, ) writer = object_detector.MetadataWriter.create_for_inference( test_utils.load_file(_MODEL_COCO), [_NORM_MEAN], [_NORM_STD], [self._label_file], score_calibration_md) self._validate_metadata(writer, _JSON_FOR_DUMMY_SCORE_CALIBRATION) self._validate_populated_model(writer) # Test if populated model is equivalent to the expected model. metadata_dict = json.loads(writer.get_metadata_json()) displayer = metadata.MetadataDisplayer.with_model_buffer( test_utils.load_file(_EXPECTED_DUMMY_MODEL)) expected_metadata_dict = json.loads(displayer.get_metadata_json()) self.assertDictContainsSubset(metadata_dict, expected_metadata_dict)
def test_create_from_metadata_info_fails_with_wrong_output_tesnor_name(self): model_buffer = test_utils.load_file(_MODEL) output_md = metadata_info.TensorMd(tensor_name="wrong_tensor_name") with self.assertRaises(ValueError) as error: metadata_writer.MetadataWriter.create_from_metadata_info( model_buffer, output_md=[output_md]) self.assertEqual( "The tensor names from arguments (['wrong_tensor_name']) do not match" " the tensor names read from the model (['output']).", str(error.exception))
def test_create_metadata_should_succeed(self): score_calibration_md = metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE, self._SCORE_CALIBRATION_FILE) score_calibration_metadata = score_calibration_md.create_metadata() metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata_with_process_uint( score_calibration_metadata)) expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_score_calibration_should_succeed(self): score_calibration_md = metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.INVERSE_LOGISTIC, _SCORE_CALIBRATION_DEFAULT_SCORE, self._score_file, ) writer = object_detector.MetadataWriter.create_for_inference( test_utils.load_file(_MODEL_COCO), [_NORM_MEAN], [_NORM_STD], [self._label_file], score_calibration_md) self._validate_metadata(writer, _JSON_FOR_SCORE_CALIBRATION) self._validate_populated_model(writer)
def test_create_from_metadata_info_succeeds_for_multihead(self): calibration_file1 = test_utils.create_calibration_file( self.get_temp_dir(), "score_cali_1.txt") calibration_file2 = test_utils.create_calibration_file( self.get_temp_dir(), "score_cali_2.txt") general_md = metadata_info.GeneralMd(name="AudioClassifier") input_md = metadata_info.InputAudioTensorMd( name="audio_clip", sample_rate=_SAMPLE_RATE, channels=_CHANNELS) # The output tensors in the model are: Identity, Identity_1 # Create metadata in a different order to test if MetadataWriter can correct # it. output_head_md_1 = metadata_info.ClassificationTensorMd( name="head1", label_files=[ metadata_info.LabelFileMd("labels_en_1.txt"), metadata_info.LabelFileMd("labels_cn_1.txt") ], score_calibration_md=metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, calibration_file1), tensor_name="Identity_1") output_head_md_2 = metadata_info.ClassificationTensorMd( name="head2", label_files=[ metadata_info.LabelFileMd("labels_en_2.txt"), metadata_info.LabelFileMd("labels_cn_2.txt") ], score_calibration_md=metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, calibration_file2), tensor_name="Identity") writer = ( audio_classifier.MetadataWriter.create_from_metadata_info_for_multihead( test_utils.load_file(_MULTIHEAD_MODEL), general_md, input_md, [output_head_md_1, output_head_md_2])) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(_JSON_MULTIHEAD, "r") self.assertEqual(metadata_json, expected_json)
def test_populate_from_metadata_should_succeed(self): model_buffer = test_utils.load_file(_MODEL) model_metadata, input_metadata, output_metadata = ( self._create_dummy_metadata()) writer = metadata_writer.MetadataWriter.create_from_metadata( model_buffer, model_metadata, [input_metadata], [output_metadata], [_LABEL_FILE]) model_with_metadata = writer.populate() self._assert_correct_metadata(model_with_metadata, _EXPECTED_DUMMY_JSON)
def test_create_metadata_should_succeed(self): regex_tokenizer_md = metadata_info.RegexTokenizerMd( self._DELIM_REGEX_PATTERN, self._VOCAB_FILE) text_tensor_md = metadata_info.InputTextTensorMd( self._NAME, self._DESCRIPTION, regex_tokenizer_md) metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata_with_tensor( text_tensor_md.create_metadata())) expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_populate_create_from_metadata_info_should_succeed(self): model_buffer = test_utils.load_file(_MODEL) general_md = metadata_info.GeneralMd(name=_MODEL_NAME) input_md = metadata_info.TensorMd(name=_INPUT_NAME) output_md = metadata_info.TensorMd(name=_OUTPUT_NAME) writer = metadata_writer.MetadataWriter.create_from_metadata_info( model_buffer, general_md, [input_md], [output_md], [_LABEL_FILE]) model_with_metadata = writer.populate() self._assert_correct_metadata(model_with_metadata, _EXPECTED_META_INFO_JSON)
def _assert_correct_metadata(self, model_with_metadata, expected_json_file): # Verify if the metadata populated is correct. displayer = _metadata.MetadataDisplayer.with_model_buffer( model_with_metadata) metadata_json = displayer.get_metadata_json() expected_json = test_utils.load_file(expected_json_file, "r") self.assertEqual(metadata_json, expected_json) # Verify if the associated file is packed as expected. packed_files = displayer.get_packed_associated_file_list() expected_packed_files = [os.path.basename(_LABEL_FILE)] self.assertEqual(set(packed_files), set(expected_packed_files))
def test_create_for_inference_fails_with_wrong_channels(self, wrong_channels): with self.assertRaises(ValueError) as error: audio_classifier.MetadataWriter.create_for_inference( test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), _SAMPLE_RATE, wrong_channels, [_LABEL_FILE], metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE)) self.assertEqual( "channels should be positive, but got {}.".format(wrong_channels), str(error.exception))
def test_audio_embedder(self): with mt.Writer( test_utils.load_file(_AUDIO_EMBEDDING_MODEL), model_name='audio_embedder', model_description='Generate embedding for the input audio clip' ) as writer: out_dir = self.create_tempdir() writer.add_audio_input(sample_rate=16000, channels=1) writer.add_embedding_output() _, metadata_json = writer.populate( os.path.join(out_dir, 'model.tflite'), os.path.join(out_dir, 'metadata.json')) self.assertEqual( metadata_json, """{ "name": "audio_embedder", "description": "Generate embedding for the input audio clip", "subgraph_metadata": [ { "input_tensor_metadata": [ { "name": "audio", "description": "Input audio clip to be processed.", "content": { "content_properties_type": "AudioProperties", "content_properties": { "sample_rate": 16000, "channels": 1 } }, "stats": { } } ], "output_tensor_metadata": [ { "name": "embedding", "description": "Embedding vector of the input.", "content": { "content_properties_type": "FeatureProperties", "content_properties": { } }, "stats": { } } ] } ], "min_parser_version": "1.3.0" } """)
def test_create_metadata_should_succeed(self, tensor_type, golden_json): tesnor_md = metadata_info.InputImageTensorMd( name=self._NAME, description=self._DESCRIPTION, norm_mean=list(self._NORM_MEAN), norm_std=list(self._NORM_STD), color_space_type=self._COLOR_SPACE_TYPE, tensor_type=tensor_type) tensor_metadata = tesnor_md.create_metadata() metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata(tensor_metadata)) expected_json = test_utils.load_file(golden_json, "r") self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self): label_file_en = metadata_info.LabelFileMd( file_path=self._LABEL_FILE_EN, locale="en") label_file_cn = metadata_info.LabelFileMd( file_path=self._LABEL_FILE_CN, locale="cn") tesnor_md = metadata_info.CategoryTensorMd( name=self._NAME, description=self._DESCRIPTION, label_files=[label_file_en, label_file_cn]) tensor_metadata = tesnor_md.create_metadata() metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata(tensor_metadata)) expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_fails_with_wrong_sample_rate( self, wrong_sample_rate): with self.assertRaises(ValueError) as error: audio_classifier.MetadataWriter.create_for_inference( test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), wrong_sample_rate, _CHANNELS, [_LABEL_FILE], metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, test_utils.create_calibration_file(self.get_temp_dir()))) self.assertEqual( "sample_rate should be positive, but got {}.".format(wrong_sample_rate), str(error.exception))
def test_create_metadata_should_succeed(self, tensor_type, golden_json): label_file_en = metadata_info.LabelFileMd( file_path=self._LABEL_FILE_EN, locale="en") label_file_cn = metadata_info.LabelFileMd( file_path=self._LABEL_FILE_CN, locale="cn") tesnor_md = metadata_info.ClassificationTensorMd( name=self._NAME, description=self._DESCRIPTION, label_files=[label_file_en, label_file_cn], tensor_type=tensor_type) tensor_metadata = tesnor_md.create_metadata() metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata(tensor_metadata)) expected_json = test_utils.load_file(golden_json, "r") self.assertEqual(metadata_json, expected_json)