def test_linear_regression_model_two(self):
"""Test simulated eeg regression. Do it twice to look for variable error.
"""
logging.info(
'\n\n******* test_simulated_linear_regression_two starting... '
'*******')
self.clear_model()
test_dataset, _, _ = self.create_linear_algebra_dataset()
logging.info('Creating the model....')
bmlr = brain_model.BrainModelLinearRegression(test_dataset)
logging.info('Training the model....')
bmlr.compile(optimizer=tf.keras.optimizers.RMSprop(learning_rate=1e-3),
loss=['mse'],
metrics=[brain_model.pearson_correlation_first])
bmlr.fit(test_dataset)
logging.info('Estimated W is: %s', bmlr.w_estimate)
logging.info('Estimated b is: %s', bmlr.b_estimate)
logging.info('Evaluating the model...')
metrics = bmlr.evaluate(test_dataset)
logging.info('test_simulated_linear_regression errors: %s', metrics)
logging.info('test_linear_regression_model returned these metrics: %s',
metrics)
r = metrics['pearson_correlation_first']
self.assertGreater(r, 0.99) # Attended reconstruction
# Now test the model predictions.
for x, y in test_dataset.take(1):
predictions = bmlr.predict(x)
np.testing.assert_allclose(y, predictions, rtol=1e-5)
# Do it again!
logging.info('Creating the model....')
bmlr = brain_model.BrainModelLinearRegression(test_dataset)
logging.info('Training the model....')
bmlr.compile(optimizer=tf.keras.optimizers.RMSprop(learning_rate=1e-3),
loss=['mse'],
metrics=[brain_model.pearson_correlation_first])
bmlr.fit(test_dataset)
logging.info('Estimated W is: %s', bmlr.w_estimate)
logging.info('Estimated b is: %s', bmlr.b_estimate)
logging.info('Evaluating the model...')
metrics = bmlr.evaluate(test_dataset)
logging.info('test_simulated_linear_regression errors: %s', metrics)
logging.info('test_linear_regression_model returned these metrics: %s',
metrics)
r = metrics['pearson_correlation_first']
self.assertGreater(r, 0.99) # Attended reconstruction
# Now test the model predictions.
for x, y in test_dataset.take(1):
predictions = bmlr.predict(x)
np.testing.assert_allclose(y, predictions, rtol=1e-5)
def test_simulated_dnn_regression(self):
"""Test simulated eeg regression.
This test starts with both attended and unattended audio, so we test the
reconstruction accuracy of both signals.
"""
logging.info('\n\n**********test_simulated_dnn_regression starting... '
'*******')
self.clear_model()
batch_size_request = 128
num_input_channels = 32
frame_rate = 100.0
test_brain_data = TestBrainData('input',
'output',
frame_rate,
pre_context=32,
post_context=0,
final_batch_size=batch_size_request)
test_dataset = self.create_simulated_eeg_data(test_brain_data,
num_input_channels,
mode='train')
model = brain_model.BrainModelLinearRegression(test_dataset)
model.compile(
optimizer=tf.keras.optimizers.RMSprop(learning_rate=1e-3),
loss=['mse'],
metrics=[brain_model.pearson_correlation_first])
model.fit(test_dataset)
metrics = model.evaluate(test_dataset)
logging.info('Test simulated linear regression metrics are: %s',
metrics)
logging.info('Test_simulated_linear_regression produced a MSE of %s',
metrics['loss'])
# Check attended reconstruction
self.assertGreater(metrics['pearson_correlation_first'], 0.95)
def create_brain_model(model_flags, input_dataset):
"""Creates the right kind of brain model.
Args:
model_flags: A DecodingOptions structure giving the desired model pararms.
input_dataset: Some models infer the data size from this dataset.
Returns:
The desired BrainModel object.
TODO Make models use input_dataset, so output_dim goes away.
"""
if not isinstance(model_flags, DecodingOptions):
raise TypeError('Model_flags must be a DecodingOptions, not a %s' %
type(model_flags))
if not isinstance(input_dataset, tf.data.Dataset):
raise TypeError('input_dataset must be a tf.data.Dataset, not %s' %
type(input_dataset))
if model_flags.dnn_regressor == 'fullyconnected':
logging.info('Create_brain_model creating a fullyconnected model.')
# Convert the string for hidden units into a list of numbers.
if not model_flags.hidden_units:
logging.info('Setting the number of hidden units to []')
hidden_units = []
else:
hidden_units = [
int(x) for x in model_flags.hidden_units.split('-')
]
bm = brain_model.BrainModelDNN(
input_dataset,
hidden_units,
tensorboard_dir=model_flags.tensorboard_dir)
elif model_flags.dnn_regressor == 'classifier':
logging.info('Create_brain_model creating a fullyconnected model.')
bm = brain_model.BrainModelClassifier(
input_dataset,
model_flags.hidden_units,
tensorboard_dir=model_flags.tensorboard_dir)
elif model_flags.dnn_regressor == 'linear':
logging.info('Create_brain_model creating a linear model.')
bm = brain_model.BrainModelLinearRegression(
input_dataset,
model_flags.regularization_lambda,
tensorboard_dir=model_flags.tensorboard_dir)
elif model_flags.dnn_regressor == 'cca':
logging.info('Create_brain_model creating a CCA model.')
bm = cca.BrainModelCCA(
input_dataset,
cca_dims=model_flags.cca_dimensions,
regularization_lambda=model_flags.regularization_lambda,
tensorboard_dir=model_flags.tensorboard_dir)
else:
raise TypeError('Unknown model type %s in create_brain_model.' % type)
bm.compile(learning_rate=model_flags.learning_rate)
return bm
def test_simulated_linear_regression(self):
"""Test simulated eeg regression.
This test starts with only attended, so we test the reconstruction accuracy
of only this signal.
"""
logging.info(
'\n\n**********test_simulated_linear_regression starting... '
'*******')
self.clear_model()
batch_size_request = 128
num_input_channels = 32
frame_rate = 100.0
test_brain_data = TestBrainData('input',
'output',
frame_rate,
pre_context=32,
post_context=0,
final_batch_size=batch_size_request)
self.create_simulated_eeg_data(test_brain_data,
num_input_channels,
mode='train',
num_output_channels=1)
test_dataset = test_brain_data.create_dataset('train')
model = brain_model.BrainModelLinearRegression(test_dataset)
model.compile(
optimizer=tf.keras.optimizers.RMSprop(learning_rate=1e-3),
loss=['mse'],
metrics=[brain_model.pearson_correlation_first])
model.fit(test_dataset)
metrics = model.evaluate(test_dataset)
logging.info('Test simulated linear regression metrics are: %s',
metrics)
logging.info('Test_simulated_linear_regression produced a MSE of %s',
metrics['loss'])
# Check attended reconstruction
self.assertGreater(metrics['pearson_correlation_first'], 0.99)
# Test inference with this model. Note, this depends on the dataset defined
# by this library to add the proper context to the data.
test_dataset = test_brain_data.create_dataset('program_test')
for next_element in test_dataset.take(1):
(input_dict, output) = next_element
for k in input_dict:
logging.info(' %s: %s', k, input_dict[k].shape)
logging.info('Output size is: %s', output.shape)
predictions = model.predict(tf.data.Dataset.from_tensors(input_dict))
edge_count = 10
signal_power = np.sum(output[edge_count:-edge_count]**2)
error = output - predictions
error_power = np.sum(error[edge_count:-edge_count]**2)
snr = 10 * np.log10(signal_power / error_power)
logging.info('Inference SNR is %s', snr)
self.assertGreater(snr, 16.0)
def create_model(self):
train_files = 'subj01'
test_files = 'subj02'
tf_dir = self._test_data_dir
params = {
'input_field': 'meg',
'output_field': 'envelope',
'pre_context': 0,
'post_context': 0,
'input2_pre_context': 0,
'input2_post_context': 0,
'attended_field': None,
}
audio_label = 'envelope'
bd = infer.create_brain_data(tf_dir, train_files, test_files, params,
audio_label)
bd_train = bd.create_dataset('train')
test_model = brain_model.BrainModelLinearRegression(bd_train)
learning_rate = 1e-3
my_optimizer = tf.keras.optimizers.RMSprop(learning_rate=learning_rate)
test_model.compile(optimizer=my_optimizer,
metrics=['mse'],
loss=['mse'])
test_model.fit(bd_train)
# results = test_model.evaluate(bd_train)
# print('Test results are:', results)
test_model.add_metadata(params, dataset=bd_train)
dprime, final_decoder = decoding.train_lda_model(
bd, test_model, params)
print('dprime after training is %g.' % dprime)
saved_model_dir = os.path.join(
os.environ.get('TMPDIR') or '/tmp', 'linear_saved_model')
logging.info('Writing saved model to %s', saved_model_dir)
test_model.save(saved_model_dir)
final_decoder.save_parameters(
os.path.join(saved_model_dir, 'decoder_model.json'))
return saved_model_dir, train_files, test_files
def test_save_model(self):
"""Test simple (small FIR with temporal shift) regression."""
self.clear_model()
pre_context = 16
post_context = 0
repeat_count_request = 1
batch_size_request = 128
frame_rate = 100.0
saved_model_dir = '/tmp/tf_saved_model.tf'
# Create the dataset and train the model.
num_input_channels = 1
test_brain_data = TestBrainData('input',
'output',
frame_rate,
pre_context=pre_context,
post_context=post_context,
repeat_count=repeat_count_request,
final_batch_size=batch_size_request)
self.load_simple_iir_dataset(test_brain_data,
num_input_channels=num_input_channels)
test_dataset = test_brain_data.create_dataset()
bmlr = brain_model.BrainModelLinearRegression(test_dataset)
bmlr.compile(optimizer=tf.keras.optimizers.RMSprop(learning_rate=1e-3),
loss=['mse'],
metrics=[brain_model.pearson_correlation_first])
bmlr.fit(test_dataset)
metrics = bmlr.evaluate(test_dataset)
logging.info('test_save_model evaluate metrics are: %s', metrics)
self.assertLess(metrics['loss'], 0.001)
self.assertGreater(metrics['pearson_correlation_first'], .99)
# Save some predictions for later.
for inputs, _ in test_dataset.take(1):
save_x = inputs
predict_y = bmlr.predict(save_x)
# Save the model on disk
bmlr.add_metadata(flags.FLAGS.flag_values_dict(), test_dataset)
bmlr.save(saved_model_dir, save_format='tf')
# Now restore the model.
# Done this way because of (and fixed in TF2.2?)
with tf.keras.utils.CustomObjectScope({
'pearson_correlation_first':
brain_model.pearson_correlation_first,
'BrainCcaLayer':
cca.BrainCcaLayer,
}):
new_model = tf.keras.models.load_model(saved_model_dir)
new_predictions = new_model.predict(save_x)
np.testing.assert_allclose(predict_y,
new_predictions,
rtol=1e-6,
atol=1e-6)
metadata = json.loads(new_model.telluride_metadata.numpy())
logging.info('New_model.telluride_metadata: %s', metadata)
self.assertIsInstance(metadata, dict)
self.assertEqual(metadata['telluride_test'], 'just for testing')
inputs = json.loads(new_model.telluride_inputs.numpy())
self.assertEqual(inputs['input_1'], list(save_x['input_1'].shape))
self.assertEqual(inputs['input_2'], list(save_x['input_2'].shape))
output = json.loads(new_model.telluride_output.numpy())
self.assertEqual(output, list(predict_y.shape))
def test_linear_regression(self):
# First test the basic linear regressor parameters using fixed weight and
# bias vectors.
desired_sample_count = 1000
w_true = [[1, 3], [2, 4]]
b_true = [[5, 6]]
x_train = np.random.rand(desired_sample_count, 2).astype(np.float32)
y_train = np.matmul(x_train, np.array(w_true)) + np.array(b_true)
y_train = tf.cast(y_train, tf.float32)
test_dataset = tf.data.Dataset.from_tensor_slices(({
'input_1': x_train
}, y_train))
test_dataset = test_dataset.batch(100).repeat(count=1)
(w_estimate, b_estimate, _, _,
_) = brain_model.calculate_linear_regressor_parameters_from_dataset(
test_dataset, lamb=0.0, use_offset=True)
logging.info('Estimated w: %s', w_estimate)
logging.info('Estimated b: %s', b_estimate)
self.assertTrue(np.allclose(w_true, w_estimate, atol=1e-05))
self.assertTrue(np.allclose(b_true, b_estimate, atol=1e-05))
(average_error, _, _, num_samples) = evaluate_regressor_from_dataset(
w_estimate, b_estimate, test_dataset)
self.assertLess(average_error, 1e-10)
self.assertEqual(num_samples, desired_sample_count)
# Now redo the test without the offset, and make sure the error is large.
(w_estimate, b_estimate, _, _,
_) = brain_model.calculate_linear_regressor_parameters_from_dataset(
test_dataset, lamb=0.0, use_offset=False)
logging.info('Estimated w: %s', w_estimate)
logging.info('Estimated b: %s', b_estimate)
# w will be different because the model no longer fits.
# self.assertTrue(np.allclose(w_true, w_estimate, atol=1e-05))
self.assertTrue(np.allclose(0.0, b_estimate, atol=1e-05))
(average_error, _, _, num_samples) = evaluate_regressor_from_dataset(
w_estimate, b_estimate, test_dataset)
self.assertGreater(average_error, 0.5)
# Create a TF model, initialized with the test dataset above, and make
# sure it is properly initialized.
model = brain_model.BrainModelLinearRegression(test_dataset)
model.compile(
optimizer=tf.keras.optimizers.RMSprop(learning_rate=1e-3),
loss=['mse'],
metrics=[brain_model.pearson_correlation_first])
model.fit(test_dataset)
w_estimate, b_estimate = model.weight_matrices
self.assertTrue(np.allclose(w_true, w_estimate, atol=1e-05))
self.assertTrue(np.allclose(b_true, b_estimate, atol=1e-05))
# Now test with a new evaluation dataset
x_eval = np.random.rand(1000, 2).astype(np.float32)
y_eval = np.matmul(x_eval, np.array(w_true)) + np.array(b_true)
eval_dataset = tf.data.Dataset.from_tensors(({
'input_1': x_eval
}, y_eval))
eval_metrics = model.evaluate(eval_dataset)
self.assertLess(eval_metrics['loss'], 1e-5)