def test_create_score_calibration_file_fails_with_negative_scale(self): malformed_calibration_file = test_utils.create_calibration_file( self.get_temp_dir(), content="-1.0,0.2,0.1") with self.assertRaisesRegex( ValueError, "Expected scale to be a non-negative value, but got -1.0."): metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE, malformed_calibration_file)
def test_create_for_inference_score_calibration_should_succeed(self): score_calibration_md = metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.INVERSE_LOGISTIC, _SCORE_CALIBRATION_DEFAULT_SCORE, self._score_file, ) writer = object_detector.MetadataWriter.create_for_inference( test_utils.load_file(_MODEL_COCO), [_NORM_MEAN], [_NORM_STD], [self._label_file], score_calibration_md) self._validate_metadata(writer, _JSON_FOR_SCORE_CALIBRATION) self._validate_populated_model(writer)
def test_create_metadata_should_succeed(self): score_calibration_md = metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE, self._SCORE_CALIBRATION_FILE) score_calibration_metadata = score_calibration_md.create_metadata() metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata_with_process_uint( score_calibration_metadata)) expected_json = test_utils.load_file(self._EXPECTED_TENSOR_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed_dynaamic_input_shape_model(self): writer = audio_classifier.MetadataWriter.create_for_inference( test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), _SAMPLE_RATE, _CHANNELS, [_LABEL_FILE], metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE)) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(_JSON_FOR_INFERENCE_DYNAMIC, "r") self.assertEqual(metadata_json, expected_json)
def test_create_score_calibration_file_fails_with_less_colunms(self): malformed_calibration_file = test_utils.create_calibration_file( self.get_temp_dir(), content="1.0,0.2") with self.assertRaisesRegex( ValueError, "Expected empty lines or 3 or 4 parameters per line in score" + " calibration file, but got 2."): metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE, malformed_calibration_file)
def test_create_from_metadata_info_succeeds_for_multihead(self): calibration_file1 = test_utils.create_calibration_file( self.get_temp_dir(), "score_cali_1.txt") calibration_file2 = test_utils.create_calibration_file( self.get_temp_dir(), "score_cali_2.txt") general_md = metadata_info.GeneralMd(name="AudioClassifier") input_md = metadata_info.InputAudioTensorMd( name="audio_clip", sample_rate=_SAMPLE_RATE, channels=_CHANNELS) # The output tensors in the model are: Identity, Identity_1 # Create metadata in a different order to test if MetadataWriter can correct # it. output_head_md_1 = metadata_info.ClassificationTensorMd( name="head1", label_files=[ metadata_info.LabelFileMd("labels_en_1.txt"), metadata_info.LabelFileMd("labels_cn_1.txt") ], score_calibration_md=metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, calibration_file1), tensor_name="Identity_1") output_head_md_2 = metadata_info.ClassificationTensorMd( name="head2", label_files=[ metadata_info.LabelFileMd("labels_en_2.txt"), metadata_info.LabelFileMd("labels_cn_2.txt") ], score_calibration_md=metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, calibration_file2), tensor_name="Identity") writer = ( audio_classifier.MetadataWriter.create_from_metadata_info_for_multihead( test_utils.load_file(_MULTIHEAD_MODEL), general_md, input_md, [output_head_md_1, output_head_md_2])) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(_JSON_MULTIHEAD, "r") self.assertEqual(metadata_json, expected_json)
def test_create_for_inference_should_succeed(self, model_file, golden_json): writer = image_classifier.MetadataWriter.create_for_inference( test_utils.load_file(model_file), [_NORM_MEAN], [_NORM_STD], [_LABEL_FILE], metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE)) metadata_json = writer.get_metadata_json() expected_json = test_utils.load_file(golden_json, "r") self.assertEqual(metadata_json, expected_json)
def add_classification_output( self, labels: Labels, score_calibration: Optional[ScoreCalibration] = None, name=_OUTPUT_CLASSIFICATION_NAME, description=_OUTPUT_CLASSIFICATION_DESCRIPTION): """Marks model's next output tensor as a classification head. Example usage: writer.add_classification_output( Labels() .add(['cat', 'dog], 'en') .add(['chat', 'chien], 'fr') .add(['/m/011l78', '/m/031d23'], use_as_category_name=True)) Args: labels: an instance of Labels helper class. score_calibration: an instance of ScoreCalibration helper class. name: Metadata name of the tensor. Note that this is different from tensor name in the flatbuffer. description: human readable description of what the tensor does. Returns: The current Writer instance to allow chained operation. """ calibration_md = None if score_calibration: calibration_md = metadata_info.ScoreCalibrationMd( score_transformation_type=score_calibration. transformation_type, default_score=score_calibration.default_score, file_path=self._export_calibration_file( 'score_calibration.txt', score_calibration.parameters)) idx = len(self._output_mds) label_files = [] for item in labels._labels: # pylint: disable=protected-access label_files.append( metadata_info.LabelFileMd(self._export_labels( item.filename, item.names), locale=item.locale)) output_md = metadata_info.ClassificationTensorMd( name=name, description=description, label_files=label_files, tensor_type=self._output_tensor_type(idx), score_calibration_md=calibration_md, ) self._output_mds.append(output_md) return self
def test_create_for_inference_fails_with_wrong_channels(self, wrong_channels): with self.assertRaises(ValueError) as error: audio_classifier.MetadataWriter.create_for_inference( test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), _SAMPLE_RATE, wrong_channels, [_LABEL_FILE], metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE)) self.assertEqual( "channels should be positive, but got {}.".format(wrong_channels), str(error.exception))
def test_create_for_inference_fails_with_wrong_sample_rate( self, wrong_sample_rate): with self.assertRaises(ValueError) as error: audio_classifier.MetadataWriter.create_for_inference( test_utils.load_file(_DYNAMIC_INPUT_SIZE_MODEL), wrong_sample_rate, _CHANNELS, [_LABEL_FILE], metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, _DEFAULT_SCORE_CALIBRATION_VALUE, test_utils.create_calibration_file(self.get_temp_dir()))) self.assertEqual( "sample_rate should be positive, but got {}.".format(wrong_sample_rate), str(error.exception))
def test_create_for_inference_dummy_score_calibration_should_succeed(self): score_calibration_md = metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.INVERSE_LOGISTIC, _DUMMY_SCORE_CALIBRATION_DEFAULT_SCORE, self._dummy_score_file, ) writer = object_detector.MetadataWriter.create_for_inference( test_utils.load_file(_MODEL_COCO), [_NORM_MEAN], [_NORM_STD], [self._label_file], score_calibration_md) self._validate_metadata(writer, _JSON_FOR_DUMMY_SCORE_CALIBRATION) self._validate_populated_model(writer) # Test if populated model is equivalent to the expected model. metadata_dict = json.loads(writer.get_metadata_json()) displayer = metadata.MetadataDisplayer.with_model_buffer( test_utils.load_file(_EXPECTED_DUMMY_MODEL)) expected_metadata_dict = json.loads(displayer.get_metadata_json()) self.assertDictContainsSubset(metadata_dict, expected_metadata_dict)
def test_create_score_calibration_file_md_should_succeed(self): score_calibration_md = metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE, self._SCORE_CALIBRATION_FILE) score_calibration_file_md = ( score_calibration_md.create_score_calibration_file_md()) file_metadata = score_calibration_file_md.create_metadata() # Create the Flatbuffers object and convert it to the json format. model_metadata = _metadata_fb.ModelMetadataT() model_metadata.associatedFiles = [file_metadata] builder = flatbuffers.Builder(0) builder.Finish(model_metadata.Pack(builder), _metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER) metadata_json = _metadata.convert_to_json(bytes(builder.Output())) expected_json = test_utils.load_file(self._EXPECTED_MODEL_META_JSON, "r") self.assertEqual(metadata_json, expected_json)
def test_create_metadata_should_succeed(self, tensor_type, golden_json): label_file_en = metadata_info.LabelFileMd( file_path=self._LABEL_FILE_EN, locale="en") label_file_cn = metadata_info.LabelFileMd( file_path=self._LABEL_FILE_CN, locale="cn") score_calibration_md = metadata_info.ScoreCalibrationMd( _metadata_fb.ScoreTransformationType.IDENTITY, self._CALIBRATION_DEFAULT_SCORE, self._SCORE_CALIBRATION_FILE) tesnor_md = metadata_info.ClassificationTensorMd( name=self._NAME, description=self._DESCRIPTION, label_files=[label_file_en, label_file_cn], tensor_type=tensor_type, score_calibration_md=score_calibration_md) tensor_metadata = tesnor_md.create_metadata() metadata_json = _metadata.convert_to_json( _create_dummy_model_metadata_with_tensor(tensor_metadata)) expected_json = test_utils.load_file(golden_json, "r") self.assertEqual(metadata_json, expected_json)