def evaluate_tflite(self, tflite_filepath, data): """Evaluates the tflite model. Args: tflite_filepath: File path to the TFLite model. data: Data to be evaluated. Returns: The evaluation result of TFLite model - accuracy. """ ds = data.gen_dataset( batch_size=1, is_training=False, preprocess=self.preprocess) predictions, labels = [], [] lite_runner = model_util.get_lite_runner(tflite_filepath, self.model_spec) for i, (feature, label) in enumerate(data_util.generate_elements(ds)): log_steps = 1000 tf.compat.v1.logging.log_every_n(tf.compat.v1.logging.INFO, 'Processing example: #%d\n%s', log_steps, i, feature) probabilities = lite_runner.run(feature) predictions.append(np.argmax(probabilities)) # Gets the ground-truth labels. label = label[0] if label.size > 1: # one-hot tensor. label = np.argmax(label) labels.append(label) predictions, labels = np.array(predictions), np.array(labels) result = {'accuracy': (predictions == labels).mean()} return result
def test_yamnet_single_head(self): tflite_path = self._train_and_export( audio_spec.YAMNetSpec(keep_yamnet_and_custom_heads=False), num_classes=2, filename='single_head.tflite', expected_model_size=13 * 1000 * 1000) self.assertEqual( 1, len(model_util.get_lite_runner(tflite_path).output_details))
def test_no_metadata(self): audio_spec.ENABLE_METADATA = False tflite_path = self._train_and_export( audio_spec.YAMNetSpec(keep_yamnet_and_custom_heads=True), num_classes=2, filename='two_heads.tflite', expected_model_size=15 * 1000 * 1000) self.assertEqual( 2, len(model_util.get_lite_runner(tflite_path).output_details)) with self.assertRaisesRegex(ValueError, 'The model does not have metadata'): model_util.extract_tflite_metadata_json(tflite_path) audio_spec.ENABLE_METADATA = True
def evaluate_tflite(self, tflite_filepath, data): """Evaluates the tflite model. The data is padded to required length, and multiple metrics are evaluated. Args: tflite_filepath: File path to the TFLite model. data: Data to be evaluated. Returns: Dict of (metric, value), evaluation result of TFLite model. """ lite_runner = model_util.get_lite_runner(tflite_filepath, self.model_spec) ds = data.gen_dataset(batch_size=1, is_training=False) max_output_size = data.max_vocab_id + 1 # +1 because 0 is reserved for OOV. eval_top_k = self.model_spec.params['eval_top_k'] metrics = [ _metrics.GlobalRecall(name=f'Global_Recall/Recall_{k}', top_k=k) for k in eval_top_k ] for feature, label in data_util.generate_elements(ds): context = feature['context'][0].astype(np.int32) # x: shape [max_length], padded with PAD_ID if len(context) == self._max_history_length: x = context elif len(context) > self._max_history_length: x = context[:self._max_history_length] else: # Padding. x = np.empty([self._max_history_length], dtype=np.int32) x.fill(self.OOV_ID) x[:len(context)] = context # ids, scores: shape [top_k] ids, scores = lite_runner.run(x) # y_true: shape [1, 1] y_true = label # y_pred: shape [1, max_output_size]; fill only scores with top-k ids. y_pred = np.zeros([1, max_output_size]) for i, score in zip(ids, scores): if i in data.vocab: # Only set if id is in vocab. y_pred[0, i] = score # Update metrics. for m in metrics: m.update_state(y_true, y_pred) result = collections.OrderedDict([(m.name, m.result()) for m in metrics]) return result
def is_same_output(tflite_file, keras_model, input_tensors, model_spec=None, atol=1e-04): """Whether the output of TFLite model is the same as keras model.""" # Gets output from lite model. lite_runner = model_util.get_lite_runner(tflite_file, model_spec) lite_output = lite_runner.run(input_tensors) # Gets output from keras model. keras_output = keras_model.predict_on_batch(input_tensors) return np.allclose(lite_output, keras_output, atol=atol)
def evaluate_tflite(self, tflite_filepath, data): """Evaluates the tflite model. The data is padded to required length, and multiple metrics are evaluated. Args: tflite_filepath: File path to the TFLite model. data: Data to be evaluated. Returns: Dict of (metric, value), evaluation result of TFLite model. """ label_name = self.input_spec.label_feature.feature_name lite_runner = model_util.get_lite_runner(tflite_filepath, self.model_spec) ds = data.gen_dataset(batch_size=1, is_training=False) max_output_size = data.max_vocab_id + 1 # +1 because 0 is reserved for OOV. eval_top_k = self.model_hparams.eval_top_k metrics = [ _metrics.GlobalRecall(top_k=k, name=f'Global_Recall/Recall_{k}') for k in eval_top_k ] for feature, y_true in data_util.generate_elements(ds): feature.pop(label_name) x = feature ids, scores = lite_runner.run(x) # y_true: shape [1, 1] # y_pred: shape [1, max_output_size]; fill only scores with top-k ids. y_pred = np.zeros([1, max_output_size]) for i, score in zip(ids, scores): if i in data.vocab: # Only set if id is in vocab. y_pred[0, i] = score # Update metrics. for m in metrics: m.update_state(y_true, y_pred) result = collections.OrderedDict([(m.name, m.result()) for m in metrics]) return result
def evaluate_tflite(self, tflite_filepath, data): ds = data.gen_dataset(batch_size=1, is_training=False) predictions, labels = [], [] lite_runner = model_util.get_lite_runner(tflite_filepath, self.model_spec) for i, (feature, label) in enumerate(data_util.generate_elements(ds)): log_steps = 1000 tf.compat.v1.logging.log_every_n(tf.compat.v1.logging.INFO, 'Processing example: #%d\n%s', log_steps, i, feature) probability = lite_runner.run(feature) # Shape: (batch=1, 1) probability = probability.flatten()[0] # Get sclar value predictions.append(probability > 0.5) label = label[0] labels.append(label) predictions = np.array(predictions).astype(int) labels = np.array(labels).astype(int) return {'accuracy': (predictions == labels).mean()}
def test_yamnet_single_head(self): tflite_path = self._train_and_export( audio_spec.YAMNetSpec(keep_yamnet_and_custom_heads=False), num_classes=2, filename='single_head.tflite', expected_model_size=13 * 1000 * 1000) self.assertEqual( 1, len(model_util.get_lite_runner(tflite_path).output_details)) self.assertAllEqual( [1, 2], model_util.get_lite_runner(tflite_path).output_details[0]['shape']) self.assertEqual( model_util.extract_tflite_metadata_json(tflite_path), """{ "name": "yamnet/classification", "description": "Recognizes sound events", "version": "v1", "subgraph_metadata": [ { "input_tensor_metadata": [ { "name": "audio_clip", "description": "Input audio clip to be classified.", "content": { "content_properties_type": "AudioProperties", "content_properties": { "sample_rate": 16000, "channels": 1 } }, "stats": { } } ], "output_tensor_metadata": [ { "name": "custom", "description": "Scores in range 0..1.0 for each output classes.", "content": { "content_properties_type": "FeatureProperties", "content_properties": { } }, "stats": { "max": [ 1.0 ], "min": [ 0.0 ] }, "associated_files": [ { "name": "custom_labels.txt", "description": "Labels for categories that the model can recognize.", "type": "TENSOR_AXIS_LABELS" } ] } ] } ], "author": "TensorFlow Lite Model Maker", "license": "Apache License. Version 2.0 http://www.apache.org/licenses/LICENSE-2.0.", "min_parser_version": "1.3.0" } """)