def test_windowed_inference(self, reduction, expected_mean):
"""Tests the training and inference stages with a linear model."""
# Create the basic decoder class, with a simple TF model.
decoder = infer_decoder.LinearRegressionDecoder(_linear_model,
reduction=reduction)
decoder.train(self._mixed_data, self._train_data)
speaker, _ = decoder.test_all(self._test_data)
window_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256]
windowed_means = np.zeros(len(window_sizes))
windowed_stds = np.zeros(len(window_sizes))
for i, window_size in enumerate(window_sizes):
results = []
# Evaluate performance on first half of the training data
for window_start in range(0, _NUM_TEST_POINTS // 2, window_size):
window_end = window_start + window_size
results.append(np.mean(speaker[window_start:window_end] > 0.5))
windowed_means[i] = np.mean(results)
windowed_stds[i] = np.std(results)
plt.clf()
plt.errorbar(window_sizes, windowed_means, windowed_stds)
plt.gca().set_xscale('log')
plt.title('Test_windowed_inference with %s' % reduction)
plt.savefig(
os.path.join(
os.environ.get('TMPDIR') or '/tmp',
'windowed_inference_%s.png' % reduction))
plt.xlabel('Window size (frames)')
self.assertAlmostEqual(np.mean(windowed_means),
expected_mean,
delta=0.05)
def test_one_window(self):
"""Tests the training and inference stages with a linear model."""
# Create the basic decoder class, with a simple TF model.
decoder = infer_decoder.LinearRegressionDecoder(_linear_model)
decoder.train(self._mixed_data, self._train_data)
batch_size = 101
for speaker, label in decoder.test_by_window(self._test_data,
batch_size):
self.assertEqual(speaker.shape, (batch_size, 1))
self.assertEqual(label.shape, (batch_size, 1))
def test_correlation_save_model(self):
num_batches = 50 # Arbitrary
batch_size = 340 # Arbitrary
total_points = num_batches * batch_size
x = np.random.randn(total_points, 3) + 1.2
y = x * 3 + 3.1
decoder = infer_decoder.LinearRegressionDecoder(_linear_model)
decoder.add_data_correlator(x, y)
x_new = np.random.randn(total_points, 3) + 1.2
y_new = x_new * 3 + 3.1
r = decoder.compute_correlation(x_new, y_new)
tmp_dir = flags.FLAGS.test_tmpdir or '/tmp'
corr_save_dir = os.path.join(tmp_dir, 'corr_params.json')
decoder.save_parameters(corr_save_dir)
decoder_loaded = infer_decoder.LinearRegressionDecoder(_linear_model)
decoder_loaded.restore_parameters(corr_save_dir)
r_loaded = decoder_loaded.compute_correlation(x_new, y_new)
np.testing.assert_equal(r_loaded, r)
def test_train_no_switches(self):
"""Tests the training and inference stages with a linear model."""
# Create the basic decoder class, with a simple TF model.
decoder = infer_decoder.LinearRegressionDecoder(_linear_model)
empty_dataset = tf.data.Dataset.from_tensor_slices(({
'input_1': [],
'input_2': []
}, []))
with self.assertRaisesRegex(ValueError, 'No data for class 0'):
decoder.train(empty_dataset, self._mixed_data)
with self.assertRaisesRegex(ValueError, 'No data for class 1'):
decoder.train(self._mixed_data, empty_dataset)
def test_correlation_calculation(self):
num_batches = 50 # Arbitrary
batch_size = 3400 # Arbitrary
total_points = num_batches * batch_size
x = np.random.randn(total_points, 3) + 1.2
y = x * 3 + 3.1
decoder = infer_decoder.LinearRegressionDecoder(_linear_model)
for i in range(num_batches):
s = i * batch_size
e = s + batch_size
decoder.add_data_correlator(x[s:e, :], y[s:e, :])
r = decoder.compute_correlation(x, y)
np.testing.assert_allclose(np.mean(r), 1, rtol=1e-5)
def test_inference(self, reduction):
"""Tests the training and inference stages with a linear model."""
# Create the basic decoder class, with a simple TF model.
decoder = infer_decoder.LinearRegressionDecoder(_linear_model,
reduction=reduction)
decoder.train(self._mixed_data, self._train_data)
speaker, labels = decoder.test_all(self._test_data)
plt.clf()
plt.plot(labels)
plt.plot(speaker)
plt.savefig(
os.path.join(
os.environ.get('TMPDIR') or '/tmp',
'inference_%s.png' % reduction))
print('test_inference_%s:' % reduction, speaker.shape, labels.shape)
self.assertGreater(np.mean(speaker[labels == 0]), 0.5)
self.assertLess(np.mean(speaker[labels == 1]), 0.5)
def test_training_and_inference(self,
regressor_name,
reduction,
tolerance=0.1,
window_size=1):
"""Tests the training and inference stages with a linear model."""
print('Training the %s regressor.' % regressor_name)
# Create the desired decoder class.
if regressor_name == 'linear':
decoder = infer_decoder.LinearRegressionDecoder(
_linear_model, reduction=reduction)
elif regressor_name == 'CCA':
decoder = infer_decoder.CCADecoder(_cca_model, reduction=reduction)
else:
raise ValueError('Unknown decoder name: %s' % regressor_name)
dprime = decoder.train(self._mixed_data,
self._train_data,
window_size=window_size)
logging.info('Infer training of %s data via %s gave a dprime of %g.',
regressor_name, reduction, dprime)
speaker, _ = decoder.test_all(self._test_data)
plt.clf()
plt.plot(speaker)
plt.savefig(
os.path.join(
os.environ.get('TMPDIR') or '/tmp',
'inference_train_%s_%s.png' % (regressor_name, reduction)))
self.assertGreater(np.mean(speaker[:(_NUM_TEST_POINTS // 2)]),
1.0 - tolerance)
self.assertLess(np.mean(speaker[(_NUM_TEST_POINTS // 2):]), tolerance)
# Make sure we can retrieve and save parameters (without errors)
decoder.decoding_model_params = decoder.decoding_model_params