def _test_spec(self, train_spec, tflite_eval_spec): temp_folder = self.get_temp_dir() cat1 = write_sample(temp_folder, 'cat', '1.wav', 44100, duration_sec=1) cat2 = write_sample(temp_folder, 'cat', '2.wav', 44100, duration_sec=2) dog1 = write_sample(temp_folder, 'dog', '1.wav', 44100, duration_sec=3) dog2 = write_sample(temp_folder, 'dog', '2.wav', 44100, duration_sec=4) index_to_labels = ['cat', 'dog'] np.random.seed(123) tf.random.set_seed(123) # Prepare data. ds = tf.data.Dataset.from_tensor_slices(([cat1, cat2, dog1, dog2], [0, 0, 1, 1])) data_loader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels, train_spec) # Train a floating point model. task = audio_classifier.create(data_loader, train_spec, batch_size=1, epochs=15) # Evaluate trained model _, acc = task.evaluate(data_loader) # Better than random guessing. self.assertGreater(acc, .5) # Export the model to saved model. output_path = os.path.join(train_spec.model_dir, 'saved_model') task.export(train_spec.model_dir, export_format=ExportFormat.SAVED_MODEL) self.assertTrue(os.path.isdir(output_path)) self.assertNotEqual(len(os.listdir(output_path)), 0) # Export the model to TFLite. output_path = os.path.join(train_spec.model_dir, 'float.tflite') task.export(train_spec.model_dir, tflite_filename='float.tflite', export_format=ExportFormat.TFLITE) self.assertTrue(tf.io.gfile.exists(output_path)) self.assertGreater(os.path.getsize(output_path), 0) # Evaluate accurarcy on TFLite model. # Create a new dataset without preprocessing since preprocessing has been # packaged inside TFLite model. tflite_dataloader = audio_dataloader.DataLoader( ds, len(ds), index_to_labels, tflite_eval_spec) # Evaluate accurarcy on float model. result = task.evaluate_tflite( output_path, tflite_dataloader, # Skip yamnet output during TFLite evaluation. postprocess_fn=lambda x: x[-1]) self.assertGreaterEqual(result['accuracy'], .5)
def testBrowserFFT(self): def pcm(shape): # Convert random number between (0, 1] to int16 return np.random.rand(*shape) * (1 << 15) np.random.seed(123) tf.random.set_seed(123) # Prepare data. spec = audio_spec.BrowserFFTSpec() dataset_shape = (1, spec.expected_waveform_len) sounds = [pcm(dataset_shape) for category in range(2)] labels = list(range(2)) index_to_labels = ['sound1', 'sound2'] ds = tf.data.Dataset.zip((tf.data.Dataset.from_tensor_slices(sounds), tf.data.Dataset.from_tensor_slices(labels))) preprocessed_ds = ds.map(spec.preprocess) data_loader = audio_dataloader.DataLoader(preprocessed_ds, len(preprocessed_ds), index_to_labels) # Train a floating point model. task = audio_classifier.create(data_loader, spec, batch_size=1, epochs=50) # Evaluate trained model _, acc = task.evaluate(data_loader) # Better than random guessing. self.assertGreater(acc, .5) # Export the model to saved model. output_path = os.path.join(spec.model_dir, 'saved_model') task.export(spec.model_dir, export_format=ExportFormat.SAVED_MODEL) self.assertTrue(os.path.isdir(output_path)) self.assertNotEqual(len(os.listdir(output_path)), 0) # Export the model to TFLite. output_path = os.path.join(spec.model_dir, 'float.tflite') task.export( spec.model_dir, tflite_filename='float.tflite', export_format=ExportFormat.TFLITE) self.assertTrue(tf.io.gfile.exists(output_path)) self.assertGreater(os.path.getsize(output_path), 0) # Evaluate accurarcy on TFLite model. # Create a new dataset without preprocessing since preprocessing has been # packaged inside TFLite model. squeezed_ds = ds.map(lambda x, y: (tf.squeeze(tf.cast(x, tf.float32)), y)) tflite_dataloader = audio_dataloader.DataLoader(squeezed_ds, len(squeezed_ds), index_to_labels) # Evaluate accurarcy on float model. result = task.evaluate_tflite(output_path, tflite_dataloader) self.assertGreater(result['accuracy'], .5)
def testConfusionMatrix(self): spec = audio_spec.BrowserFFTSpec() temp_folder = self.get_temp_dir() cat1 = write_sample(temp_folder, 'cat', '1.wav', 44100, duration_sec=1) cat2 = write_sample(temp_folder, 'cat', '2.wav', 44100, duration_sec=2) dog1 = write_sample(temp_folder, 'dog', '1.wav', 44100, duration_sec=3) dog2 = write_sample(temp_folder, 'dog', '2.wav', 44100, duration_sec=4) index_to_labels = ['cat', 'dog'] # Prepare data. ds = tf.data.Dataset.from_tensor_slices(([cat1, cat2, dog1, dog2], [0, 0, 1, 1])) data_loader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels, spec) # Train a floating point model. task = audio_classifier.create(data_loader, spec, batch_size=1, epochs=15) confusion_matrx = task.confusion_matrix(data_loader) # BrowserFFTSpec generates 1 sample for 1 second audio so there are # 10 samples in total. self.assertEqual(tf.math.reduce_sum(confusion_matrx), 10) # confusion_matrix is of shape (truth, predication) # We have 2 classes, 3 cat samples and 7 dog samples. self.assertEqual(confusion_matrx.shape, (2, 2)) self.assertAllEqual( tf.math.reduce_sum(confusion_matrx, axis=-1).numpy(), np.array([3, 7]))
def testBrowserFFT(self): def pcm(shape): # Convert random number between (0, 1] to int16 return np.random.rand(*shape) * (1 << 15) np.random.seed(123) spec = audio_spec.BrowserFFTSpec() dataset_shape = (1, spec.expected_waveform_len) sounds = [pcm(dataset_shape) for category in range(2)] labels = list(range(2)) index_to_labels = ['sound1', 'sound2'] ds = tf.data.Dataset.zip((tf.data.Dataset.from_tensor_slices(sounds), tf.data.Dataset.from_tensor_slices(labels))) ds = ds.map(spec.preprocess) data_loader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels) task = audio_classifier.create(data_loader, spec, batch_size=1, epochs=100) _, acc = task.evaluate(data_loader) # Better than random guessing. self.assertGreater(acc, .5) # Export the model to saved model. saved_model_output_path = os.path.join(spec.model_dir, 'saved_model') task.export(spec.model_dir, export_format=ExportFormat.SAVED_MODEL) self.assertTrue(os.path.isdir(saved_model_output_path)) self.assertNotEqual(len(os.listdir(saved_model_output_path)), 0)
def _test_spec(self, train_spec, tflite_eval_spec): temp_folder = self.get_temp_dir() cat1 = write_sample(temp_folder, 'cat', '1.wav', 44100, duration_sec=1) cat2 = write_sample(temp_folder, 'cat', '2.wav', 44100, duration_sec=2) dog1 = write_sample(temp_folder, 'dog', '1.wav', 44100, duration_sec=3) dog2 = write_sample(temp_folder, 'dog', '2.wav', 44100, duration_sec=4) index_to_labels = ['cat', 'dog'] np.random.seed(123) tf.random.set_seed(123) # Prepare data. ds = tf.data.Dataset.from_tensor_slices(([cat1, cat2, dog1, dog2], [0, 0, 1, 1])) data_loader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels, train_spec) # Train a floating point model. task = audio_classifier.create( data_loader, train_spec, batch_size=1, epochs=15) # Evaluate trained model _, acc = task.evaluate(data_loader) # Better than random guessing. self.assertGreater(acc, .5) # Create a new dataset without preprocessing since preprocessing has been # packaged inside TFLite model. tflite_dataloader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels, tflite_eval_spec) # Export the floating point model to TFLite. output_path = os.path.join(train_spec.model_dir, 'float.tflite') task.export( train_spec.model_dir, tflite_filename='float.tflite', export_format=ExportFormat.TFLITE, quantization_config=None) self.assertTrue(tf.io.gfile.exists(output_path)) self.assertGreater(os.path.getsize(output_path), 0) result = task.evaluate_tflite( output_path, tflite_dataloader, # Skip yamnet output during TFLite evaluation. postprocess_fn=lambda x: x[-1]) self.assertGreaterEqual(result['accuracy'], .5) # Export the model to TFLite with dynamic range quantization. dynamic_range_output_path = os.path.join(train_spec.model_dir, 'dynamic_range.tflite') task.export( train_spec.model_dir, tflite_filename='dynamic_range.tflite', export_format=ExportFormat.TFLITE, quantization_config=configs.QuantizationConfig.for_dynamic()) self.assertTrue(tf.io.gfile.exists(dynamic_range_output_path)) self.assertGreater(os.path.getsize(dynamic_range_output_path), 0) result = task.evaluate_tflite( dynamic_range_output_path, tflite_dataloader, # Skip yamnet output during TFLite evaluation. postprocess_fn=lambda x: x[-1]) self.assertGreaterEqual(result['accuracy'], .5) # Float model should be bigger than the dynamic range quantized model by # a margin. self.assertGreater( os.path.getsize(output_path), os.path.getsize(dynamic_range_output_path) + 1 * 1000 * 1000) # Test serving model. keras_model = task.create_serving_model() self.assertAllEqual(keras_model.input_shape, [None, train_spec.EXPECTED_WAVEFORM_LENGTH]) # Test exporting to the saved model. task.export(train_spec.model_dir, export_format=ExportFormat.SAVED_MODEL) new_model = tf.keras.models.load_model( os.path.join(train_spec.model_dir, 'saved_model')) self.assertAllEqual(new_model.input_shape, [None, train_spec.EXPECTED_WAVEFORM_LENGTH]) output_path = os.path.join(train_spec.model_dir, 'saved_model') self.assertTrue(os.path.isdir(output_path)) self.assertNotEqual(len(os.listdir(output_path)), 0)