def testConfusionMatrix(self):
spec = audio_spec.BrowserFFTSpec()
temp_folder = self.get_temp_dir()
cat1 = write_sample(temp_folder, 'cat', '1.wav', 44100, duration_sec=1)
cat2 = write_sample(temp_folder, 'cat', '2.wav', 44100, duration_sec=2)
dog1 = write_sample(temp_folder, 'dog', '1.wav', 44100, duration_sec=3)
dog2 = write_sample(temp_folder, 'dog', '2.wav', 44100, duration_sec=4)
index_to_labels = ['cat', 'dog']
# Prepare data.
ds = tf.data.Dataset.from_tensor_slices(([cat1, cat2, dog1,
dog2], [0, 0, 1, 1]))
data_loader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels,
spec)
# Train a floating point model.
task = audio_classifier.create(data_loader,
spec,
batch_size=1,
epochs=15)
confusion_matrx = task.confusion_matrix(data_loader)
# BrowserFFTSpec generates 1 sample for 1 second audio so there are
# 10 samples in total.
self.assertEqual(tf.math.reduce_sum(confusion_matrx), 10)
# confusion_matrix is of shape (truth, predication)
# We have 2 classes, 3 cat samples and 7 dog samples.
self.assertEqual(confusion_matrx.shape, (2, 2))
self.assertAllEqual(
tf.math.reduce_sum(confusion_matrx, axis=-1).numpy(),
np.array([3, 7]))
def run(spec, data_dir, dataset_type, export_dir, **kwargs):
"""Runs demo."""
spec = model_spec.get(spec)
if dataset_type == 'esc50':
# Limit to 2 categories to speed up the demo
categories = ['dog', 'cat']
train_data = audio_dataloader.DataLoader.from_esc50(
spec, data_dir, folds=[0, 1, 2, 3], categories=categories)
validation_data = audio_dataloader.DataLoader.from_esc50(
spec, data_dir, folds=[
4,
], categories=categories)
test_data = audio_dataloader.DataLoader.from_esc50(
spec, data_dir, folds=[
5,
], categories=categories)
else:
data = audio_dataloader.DataLoader.from_folder(spec, data_dir)
train_data, rest_data = data.split(0.8)
validation_data, test_data = rest_data.split(0.5)
print('Training the model')
model = audio_classifier.create(train_data, spec, validation_data,
**kwargs)
print('Evaluating the model')
_, acc = model.evaluate(test_data)
print('Test accuracy: %f' % acc)
model.export(export_dir)
def testBrowserFFT(self):
def pcm(shape):
# Convert random number between (0, 1] to int16
return np.random.rand(*shape) * (1 << 15)
np.random.seed(123)
spec = audio_spec.BrowserFFTSpec()
dataset_shape = (1, spec.expected_waveform_len)
sounds = [pcm(dataset_shape) for category in range(2)]
labels = list(range(2))
index_to_labels = ['sound1', 'sound2']
ds = tf.data.Dataset.zip((tf.data.Dataset.from_tensor_slices(sounds),
tf.data.Dataset.from_tensor_slices(labels)))
ds = ds.map(spec.preprocess)
data_loader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels)
task = audio_classifier.create(data_loader,
spec,
batch_size=1,
epochs=100)
_, acc = task.evaluate(data_loader)
# Better than random guessing.
self.assertGreater(acc, .5)
# Export the model to saved model.
saved_model_output_path = os.path.join(spec.model_dir, 'saved_model')
task.export(spec.model_dir, export_format=ExportFormat.SAVED_MODEL)
self.assertTrue(os.path.isdir(saved_model_output_path))
self.assertNotEqual(len(os.listdir(saved_model_output_path)), 0)
def _test_spec(self, train_spec, tflite_eval_spec):
temp_folder = self.get_temp_dir()
cat1 = write_sample(temp_folder, 'cat', '1.wav', 44100, duration_sec=1)
cat2 = write_sample(temp_folder, 'cat', '2.wav', 44100, duration_sec=2)
dog1 = write_sample(temp_folder, 'dog', '1.wav', 44100, duration_sec=3)
dog2 = write_sample(temp_folder, 'dog', '2.wav', 44100, duration_sec=4)
index_to_labels = ['cat', 'dog']
np.random.seed(123)
tf.random.set_seed(123)
# Prepare data.
ds = tf.data.Dataset.from_tensor_slices(([cat1, cat2, dog1,
dog2], [0, 0, 1, 1]))
data_loader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels,
train_spec)
# Train a floating point model.
task = audio_classifier.create(data_loader,
train_spec,
batch_size=1,
epochs=15)
# Evaluate trained model
_, acc = task.evaluate(data_loader)
# Better than random guessing.
self.assertGreater(acc, .5)
# Export the model to saved model.
output_path = os.path.join(train_spec.model_dir, 'saved_model')
task.export(train_spec.model_dir,
export_format=ExportFormat.SAVED_MODEL)
self.assertTrue(os.path.isdir(output_path))
self.assertNotEqual(len(os.listdir(output_path)), 0)
# Export the model to TFLite.
output_path = os.path.join(train_spec.model_dir, 'float.tflite')
task.export(train_spec.model_dir,
tflite_filename='float.tflite',
export_format=ExportFormat.TFLITE)
self.assertTrue(tf.io.gfile.exists(output_path))
self.assertGreater(os.path.getsize(output_path), 0)
# Evaluate accurarcy on TFLite model.
# Create a new dataset without preprocessing since preprocessing has been
# packaged inside TFLite model.
tflite_dataloader = audio_dataloader.DataLoader(
ds, len(ds), index_to_labels, tflite_eval_spec)
# Evaluate accurarcy on float model.
result = task.evaluate_tflite(
output_path,
tflite_dataloader,
# Skip yamnet output during TFLite evaluation.
postprocess_fn=lambda x: x[-1])
self.assertGreaterEqual(result['accuracy'], .5)
def testBrowserFFT(self):
def pcm(shape):
# Convert random number between (0, 1] to int16
return np.random.rand(*shape) * (1 << 15)
np.random.seed(123)
tf.random.set_seed(123)
# Prepare data.
spec = audio_spec.BrowserFFTSpec()
dataset_shape = (1, spec.expected_waveform_len)
sounds = [pcm(dataset_shape) for category in range(2)]
labels = list(range(2))
index_to_labels = ['sound1', 'sound2']
ds = tf.data.Dataset.zip((tf.data.Dataset.from_tensor_slices(sounds),
tf.data.Dataset.from_tensor_slices(labels)))
preprocessed_ds = ds.map(spec.preprocess)
data_loader = audio_dataloader.DataLoader(preprocessed_ds,
len(preprocessed_ds),
index_to_labels)
# Train a floating point model.
task = audio_classifier.create(data_loader, spec, batch_size=1, epochs=50)
# Evaluate trained model
_, acc = task.evaluate(data_loader)
# Better than random guessing.
self.assertGreater(acc, .5)
# Export the model to saved model.
output_path = os.path.join(spec.model_dir, 'saved_model')
task.export(spec.model_dir, export_format=ExportFormat.SAVED_MODEL)
self.assertTrue(os.path.isdir(output_path))
self.assertNotEqual(len(os.listdir(output_path)), 0)
# Export the model to TFLite.
output_path = os.path.join(spec.model_dir, 'float.tflite')
task.export(
spec.model_dir,
tflite_filename='float.tflite',
export_format=ExportFormat.TFLITE)
self.assertTrue(tf.io.gfile.exists(output_path))
self.assertGreater(os.path.getsize(output_path), 0)
# Evaluate accurarcy on TFLite model.
# Create a new dataset without preprocessing since preprocessing has been
# packaged inside TFLite model.
squeezed_ds = ds.map(lambda x, y: (tf.squeeze(tf.cast(x, tf.float32)), y))
tflite_dataloader = audio_dataloader.DataLoader(squeezed_ds,
len(squeezed_ds),
index_to_labels)
# Evaluate accurarcy on float model.
result = task.evaluate_tflite(output_path, tflite_dataloader)
self.assertGreater(result['accuracy'], .5)
def run(data_dir, export_dir, spec='audio_browser_fft', **kwargs):
"""Runs demo."""
spec = model_spec.get(spec)
data = audio_dataloader.DataLoader.from_folder(spec, data_dir)
train_data, rest_data = data.split(0.8)
validation_data, test_data = rest_data.split(0.5)
print('Training the model')
model = audio_classifier.create(train_data, spec, validation_data, **kwargs)
print('Evaluating the model')
_, acc = model.evaluate(test_data)
print('Test accuracy: %f' % acc)
model.export(export_dir)
def _test_spec(self, train_spec, tflite_eval_spec):
temp_folder = self.get_temp_dir()
cat1 = write_sample(temp_folder, 'cat', '1.wav', 44100, duration_sec=1)
cat2 = write_sample(temp_folder, 'cat', '2.wav', 44100, duration_sec=2)
dog1 = write_sample(temp_folder, 'dog', '1.wav', 44100, duration_sec=3)
dog2 = write_sample(temp_folder, 'dog', '2.wav', 44100, duration_sec=4)
index_to_labels = ['cat', 'dog']
np.random.seed(123)
tf.random.set_seed(123)
# Prepare data.
ds = tf.data.Dataset.from_tensor_slices(([cat1, cat2, dog1,
dog2], [0, 0, 1, 1]))
data_loader = audio_dataloader.DataLoader(ds, len(ds), index_to_labels,
train_spec)
# Train a floating point model.
task = audio_classifier.create(
data_loader, train_spec, batch_size=1, epochs=15)
# Evaluate trained model
_, acc = task.evaluate(data_loader)
# Better than random guessing.
self.assertGreater(acc, .5)
# Create a new dataset without preprocessing since preprocessing has been
# packaged inside TFLite model.
tflite_dataloader = audio_dataloader.DataLoader(ds, len(ds),
index_to_labels,
tflite_eval_spec)
# Export the floating point model to TFLite.
output_path = os.path.join(train_spec.model_dir, 'float.tflite')
task.export(
train_spec.model_dir,
tflite_filename='float.tflite',
export_format=ExportFormat.TFLITE,
quantization_config=None)
self.assertTrue(tf.io.gfile.exists(output_path))
self.assertGreater(os.path.getsize(output_path), 0)
result = task.evaluate_tflite(
output_path,
tflite_dataloader,
# Skip yamnet output during TFLite evaluation.
postprocess_fn=lambda x: x[-1])
self.assertGreaterEqual(result['accuracy'], .5)
# Export the model to TFLite with dynamic range quantization.
dynamic_range_output_path = os.path.join(train_spec.model_dir,
'dynamic_range.tflite')
task.export(
train_spec.model_dir,
tflite_filename='dynamic_range.tflite',
export_format=ExportFormat.TFLITE,
quantization_config=configs.QuantizationConfig.for_dynamic())
self.assertTrue(tf.io.gfile.exists(dynamic_range_output_path))
self.assertGreater(os.path.getsize(dynamic_range_output_path), 0)
result = task.evaluate_tflite(
dynamic_range_output_path,
tflite_dataloader,
# Skip yamnet output during TFLite evaluation.
postprocess_fn=lambda x: x[-1])
self.assertGreaterEqual(result['accuracy'], .5)
# Float model should be bigger than the dynamic range quantized model by
# a margin.
self.assertGreater(
os.path.getsize(output_path),
os.path.getsize(dynamic_range_output_path) + 1 * 1000 * 1000)
# Test serving model.
keras_model = task.create_serving_model()
self.assertAllEqual(keras_model.input_shape,
[None, train_spec.EXPECTED_WAVEFORM_LENGTH])
# Test exporting to the saved model.
task.export(train_spec.model_dir, export_format=ExportFormat.SAVED_MODEL)
new_model = tf.keras.models.load_model(
os.path.join(train_spec.model_dir, 'saved_model'))
self.assertAllEqual(new_model.input_shape,
[None, train_spec.EXPECTED_WAVEFORM_LENGTH])
output_path = os.path.join(train_spec.model_dir, 'saved_model')
self.assertTrue(os.path.isdir(output_path))
self.assertNotEqual(len(os.listdir(output_path)), 0)
