def test_compute_sample_mean_tensor(self):
K.clear_session()
d = 12
n = 20
batch_size = n // 4
x = np.random.rand(n, d).astype(np.float32)
y = np.sin(2 * np.pi * x[:, 0]).reshape((-1, 1)).astype(np.float32)
# Linear regression
model = keras.models.Sequential()
model.add(keras.layers.Dense(1, use_bias=False, input_shape=(d,)))
model.add(keras.layers.Activation('relu'))
model.add(keras.layers.Dense(10))
model.add(keras.layers.Activation('relu'))
model.add(keras.layers.Dense(1))
model.compile(loss='mean_squared_error', optimizer=keras.optimizers.SGD())
tfutils.keras_compute_tensors(model, x, y, model.total_loss)
grad_t = tfutils.flatten_tensor_list(
tf.gradients(model.total_loss, model.trainable_weights))
grad = tfutils.keras_compute_tensors(model, x, y, grad_t)
batches = tfutils.MiniBatchMaker(x, y, batch_size)
actual_grad = tfutils.compute_sample_mean_tensor(model, batches, grad_t)
self.assertTrue(np.allclose(grad, actual_grad))
def test_full_hessian_measurement(self):
"""Test that the Hessian is computed correctly."""
K.clear_session()
n = 10
p = 4
x = np.random.rand(n).astype(np.float32)
y = np.sin(2 * np.pi * x).reshape((-1, 1)).astype(np.float32)
features = np.zeros((n, p)).astype(np.float32)
for order in range(p):
features[:, order] = np.power(x, order)
# Linear regression
model = keras.models.Sequential()
model.add(keras.layers.Dense(1, use_bias=False, input_shape=(p,)))
model.compile(loss='mean_squared_error', optimizer=keras.optimizers.SGD())
hess_t = tfutils.hessians(model.total_loss, model.trainable_weights[0])[0]
hess = tfutils.keras_compute_tensors(model, features, y, hess_t).reshape(
p, p)
batch_size = n // 4
batches = tfutils.MiniBatchMaker(features, y, batch_size)
meas = measurements.FullHessianMeasurement(MockRecorder(), model, 1, batches,
None, 1)
actual_hess = meas.compute_hessian()
self.assertTrue(np.allclose(hess, actual_hess))
self.assertFalse(np.allclose(hess, 2 * actual_hess))
def test_gradient_measurement(self):
"""Test that the full-batch gradient is computed correctly."""
K.clear_session()
d = 12
n = 20
batch_size = n // 4
x = np.random.rand(n, d).astype(np.float32)
y = np.sin(2 * np.pi * x[:, 0]).reshape((-1, 1)).astype(np.float32)
x_test = np.random.rand(n, d).astype(np.float32)
y_test = np.sin(2 * np.pi * x_test[:, 0]).reshape(
(-1, 1)).astype(np.float32)
# Linear regression
model = keras.models.Sequential()
model.add(keras.layers.Dense(1, use_bias=False, input_shape=(d, )))
model.add(keras.layers.Activation('relu'))
model.add(keras.layers.Dense(10))
model.add(keras.layers.Activation('relu'))
model.add(keras.layers.Dense(1))
model.compile(loss='mean_squared_error',
optimizer=keras.optimizers.SGD())
tfutils.keras_compute_tensors(model, x, y, model.total_loss)
grad_t = tfutils.flatten_tensor_list(
tf.gradients(model.total_loss, model.trainable_weights))
grad = tfutils.keras_compute_tensors(model, x, y, grad_t)
train_batches = tfutils.MiniBatchMaker(x, y, batch_size)
test_batches = tfutils.MiniBatchMaker(x_test, y_test, batch_size)
meas = measurements.GradientMeasurement(
MockRecorder(), model,
measurements.Frequency(freq=1, stepwise=False), train_batches,
test_batches)
meas.on_epoch_begin(0)
meas.on_batch_begin(0)
meas.on_batch_end(0)
meas.on_epoch_end(0)
actual_grad = meas.full_batch_g
self.assertTrue(np.allclose(grad, actual_grad))
def get_batch_makers(batch_size):
"""Returns train and test mini-batch makers."""
train_batches = tfutils.MiniBatchMaker(x_train, y_train, batch_size)
test_batches = tfutils.MiniBatchMaker(x_test, y_test, batch_size)
return train_batches, test_batches