def testGoodKernelApproximationAmortized(self):
# Parameters.
num_points = 20
input_dim = 5
mapped_dim = 5000
stddev = 5.0
points_shape = [1, input_dim]
points = [
random_ops.random_uniform(shape=points_shape, maxval=1.0)
for _ in xrange(num_points)
]
normalized_points = [nn.l2_normalize(point, dim=1) for point in points]
total_absolute_error = 0.0
with self.cached_session():
rffm = RandomFourierFeatureMapper(input_dim, mapped_dim, stddev, seed=0)
# Cache mappings so that they are not computed multiple times.
cached_mappings = dict((point, rffm.map(point))
for point in normalized_points)
for x in normalized_points:
mapped_x = cached_mappings[x]
for y in normalized_points:
mapped_y = cached_mappings[y]
exact_kernel_value = _compute_exact_rbf_kernel(x, y, stddev)
approx_kernel_value = _inner_product(mapped_x, mapped_y)
abs_error = math_ops.abs(exact_kernel_value - approx_kernel_value)
total_absolute_error += abs_error
self.assertAllClose(
[[0.0]],
total_absolute_error.eval() / (num_points * num_points),
atol=0.02)
def testInvalidInputShape(self):
x = constant_op.constant([[2.0, 1.0]])
with self.test_session():
rffm = RandomFourierFeatureMapper(3, 10)
with self.assertRaisesWithPredicateMatch(
dense_kernel_mapper.InvalidShapeError,
r'Invalid dimension: expected 3 input features, got 2 instead.'):
rffm.map(x)
def testInvalidInputShape(self):
x = constant_op.constant([[2.0, 1.0]])
with self.test_session():
rffm = RandomFourierFeatureMapper(3, 10)
with self.assertRaisesWithPredicateMatch(
dense_kernel_mapper.InvalidShapeError,
r'Invalid dimension: expected 3 input features, got 2 instead.'):
rffm.map(x)
def testSameOmegaReused(self):
x = constant_op.constant([[2.0, 1.0, 0.0]])
with self.test_session():
rffm = RandomFourierFeatureMapper(3, 100)
mapped_x = rffm.map(x)
mapped_x_copy = rffm.map(x)
# Two different evaluations of tensors output by map on the same input
# are identical because the same parameters are used for the mappings.
self.assertAllClose(mapped_x.eval(), mapped_x_copy.eval(), atol=0.001)
def testSameOmegaReused(self):
x = constant_op.constant([[2.0, 1.0, 0.0]])
with self.test_session():
rffm = RandomFourierFeatureMapper(3, 100)
mapped_x = rffm.map(x)
mapped_x_copy = rffm.map(x)
# Two different evaluations of tensors output by map on the same input
# are identical because the same parameters are used for the mappings.
self.assertAllClose(mapped_x.eval(), mapped_x_copy.eval(), atol=0.001)
def testMappedShape(self):
x1 = constant_op.constant([[2.0, 1.0, 0.0]])
x2 = constant_op.constant([[1.0, -1.0, 2.0], [-1.0, 10.0, 1.0],
[4.0, -2.0, -1.0]])
with self.test_session():
rffm = RandomFourierFeatureMapper(3, 10, 1.0)
mapped_x1 = rffm.map(x1)
mapped_x2 = rffm.map(x2)
self.assertEqual([1, 10], mapped_x1.get_shape())
self.assertEqual([3, 10], mapped_x2.get_shape())
def testMappedShape(self):
x1 = constant_op.constant([[2.0, 1.0, 0.0]])
x2 = constant_op.constant([[1.0, -1.0, 2.0], [-1.0, 10.0, 1.0],
[4.0, -2.0, -1.0]])
with self.test_session():
rffm = RandomFourierFeatureMapper(3, 10, 1.0)
mapped_x1 = rffm.map(x1)
mapped_x2 = rffm.map(x2)
self.assertEqual([1, 10], mapped_x1.get_shape())
self.assertEqual([3, 10], mapped_x2.get_shape())
def testBadKernelApproximation(self):
x = constant_op.constant([[2.0, 1.0, 0.0]])
y = constant_op.constant([[1.0, -1.0, 2.0]])
stddev = 3.0
with self.test_session():
# The mapped dimension is fairly small, so the kernel approximation is
# very rough.
rffm = RandomFourierFeatureMapper(3, 100, stddev, seed=0)
mapped_x = rffm.map(x)
mapped_y = rffm.map(y)
exact_kernel_value = _compute_exact_rbf_kernel(x, y, stddev)
approx_kernel_value = _inner_product(mapped_x, mapped_y)
self.assertAllClose(
exact_kernel_value.eval(), approx_kernel_value.eval(), atol=0.2)
def testBadKernelApproximation(self):
x = constant_op.constant([[2.0, 1.0, 0.0]])
y = constant_op.constant([[1.0, -1.0, 2.0]])
stddev = 3.0
with self.test_session():
# The mapped dimension is fairly small, so the kernel approximation is
# very rough.
rffm = RandomFourierFeatureMapper(3, 100, stddev, seed=0)
mapped_x = rffm.map(x)
mapped_y = rffm.map(y)
exact_kernel_value = _compute_exact_rbf_kernel(x, y, stddev)
approx_kernel_value = _inner_product(mapped_x, mapped_y)
self.assertAllClose(
exact_kernel_value.eval(), approx_kernel_value.eval(), atol=0.2)
def testMulticlassDataWithAndWithoutKernels(self):
"""Tests classifier w/ and w/o kernels on multiclass data."""
feature_column = layers.real_valued_column('feature', dimension=4)
# Metrics for linear classifier (no kernels).
linear_classifier = kernel_estimators.KernelLinearClassifier(
feature_columns=[feature_column], n_classes=3)
linear_classifier.fit(input_fn=test_data.iris_input_multiclass_fn,
steps=50)
linear_metrics = linear_classifier.evaluate(
input_fn=test_data.iris_input_multiclass_fn, steps=1)
linear_loss = linear_metrics['loss']
linear_accuracy = linear_metrics['accuracy']
# Using kernel mappers allows to discover non-linearities in data (via RBF
# kernel approximation), reduces loss and increases accuracy.
kernel_mappers = {
feature_column: [
RandomFourierFeatureMapper(input_dim=4,
output_dim=50,
stddev=1.0,
name='rffm')
]
}
kernel_linear_classifier = kernel_estimators.KernelLinearClassifier(
feature_columns=[], n_classes=3, kernel_mappers=kernel_mappers)
kernel_linear_classifier.fit(
input_fn=test_data.iris_input_multiclass_fn, steps=50)
kernel_linear_metrics = kernel_linear_classifier.evaluate(
input_fn=test_data.iris_input_multiclass_fn, steps=1)
kernel_linear_loss = kernel_linear_metrics['loss']
kernel_linear_accuracy = kernel_linear_metrics['accuracy']
self.assertLess(kernel_linear_loss, linear_loss)
self.assertGreater(kernel_linear_accuracy, linear_accuracy)
def testTwoMapperObjects(self):
x = constant_op.constant([[2.0, 1.0, 0.0]])
y = constant_op.constant([[1.0, -1.0, 2.0]])
stddev = 3.0
with self.test_session():
# The mapped dimension is fairly small, so the kernel approximation is
# very rough.
rffm1 = RandomFourierFeatureMapper(3, 100, stddev)
rffm2 = RandomFourierFeatureMapper(3, 100, stddev)
mapped_x1 = rffm1.map(x)
mapped_y1 = rffm1.map(y)
mapped_x2 = rffm2.map(x)
mapped_y2 = rffm2.map(y)
approx_kernel_value1 = _inner_product(mapped_x1, mapped_y1)
approx_kernel_value2 = _inner_product(mapped_x2, mapped_y2)
self.assertAllClose(
approx_kernel_value1.eval(), approx_kernel_value2.eval(), atol=0.01)
def testGoodKernelApproximationAmortized(self):
# Parameters.
num_points = 20
input_dim = 5
mapped_dim = 5000
stddev = 5.0
# TODO(sibyl-vie3Poto): Reduce test's running time before moving to third_party. One
# possible way to speed the test up is to compute both the approximate and
# the exact kernel matrix directly using matrix operations instead of
# computing the values for each pair of points separately.
points_shape = [1, input_dim]
points = [
random_ops.random_uniform(shape=points_shape, maxval=1.0)
for _ in xrange(num_points)
]
normalized_points = [nn.l2_normalize(point, dim=1) for point in points]
total_absolute_error = 0.0
with self.test_session():
rffm = RandomFourierFeatureMapper(input_dim,
mapped_dim,
stddev,
seed=0)
# Cache mappings so that they are not computed multiple times.
cached_mappings = dict(
(point, rffm.map(point)) for point in normalized_points)
for x in normalized_points:
mapped_x = cached_mappings[x]
for y in normalized_points:
mapped_y = cached_mappings[y]
exact_kernel_value = _compute_exact_rbf_kernel(
x, y, stddev)
approx_kernel_value = _inner_product(mapped_x, mapped_y)
abs_error = math_ops.abs(exact_kernel_value -
approx_kernel_value)
total_absolute_error += abs_error
self.assertAllClose([[0.0]],
total_absolute_error.eval() /
(num_points * num_points),
atol=0.02)
def testGoodKernelApproximationAmortized(self):
# Parameters.
num_points = 20
input_dim = 5
mapped_dim = 5000
stddev = 5.0
points_shape = [1, input_dim]
points = [
random_ops.random_uniform(shape=points_shape, maxval=1.0)
for _ in xrange(num_points)
]
normalized_points = [nn.l2_normalize(point, dim=1) for point in points]
total_absolute_error = 0.0
with self.cached_session():
rffm = RandomFourierFeatureMapper(input_dim,
mapped_dim,
stddev,
seed=0)
# Cache mappings so that they are not computed multiple times.
cached_mappings = dict(
(point, rffm.map(point)) for point in normalized_points)
for x in normalized_points:
mapped_x = cached_mappings[x]
for y in normalized_points:
mapped_y = cached_mappings[y]
exact_kernel_value = _compute_exact_rbf_kernel(
x, y, stddev)
approx_kernel_value = _inner_product(mapped_x, mapped_y)
abs_error = math_ops.abs(exact_kernel_value -
approx_kernel_value)
total_absolute_error += abs_error
self.assertAllClose([[0.0]],
total_absolute_error.eval() /
(num_points * num_points),
atol=0.02)
def testGoodKernelApproximationAmortized(self):
# Parameters.
num_points = 20
input_dim = 5
mapped_dim = 5000
stddev = 5.0
# TODO(sibyl-vie3Poto): Reduce test's running time before moving to third_party. One
# possible way to speed the test up is to compute both the approximate and
# the exact kernel matrix directly using matrix operations instead of
# computing the values for each pair of points separately.
points_shape = [1, input_dim]
points = [
random_ops.random_uniform(shape=points_shape, maxval=1.0)
for _ in xrange(num_points)
]
normalized_points = [nn.l2_normalize(point, dim=1) for point in points]
total_absolute_error = 0.0
with self.test_session():
rffm = RandomFourierFeatureMapper(input_dim, mapped_dim, stddev, seed=0)
# Cache mappings so that they are not computed multiple times.
cached_mappings = dict((point, rffm.map(point))
for point in normalized_points)
for x in normalized_points:
mapped_x = cached_mappings[x]
for y in normalized_points:
mapped_y = cached_mappings[y]
exact_kernel_value = _compute_exact_rbf_kernel(x, y, stddev)
approx_kernel_value = _inner_product(mapped_x, mapped_y)
abs_error = math_ops.abs(exact_kernel_value - approx_kernel_value)
total_absolute_error += abs_error
self.assertAllClose(
[[0.0]],
total_absolute_error.eval() / (num_points * num_points),
atol=0.02)
def testLinearlyInseparableBinaryDataWithAndWithoutKernels(self):
"""Tests classifier w/ and w/o kernels on non-linearly-separable data."""
multi_dim_feature = layers.real_valued_column('multi_dim_feature',
dimension=2)
# Data points are non-linearly separable so there will be at least one
# mis-classified sample (accuracy < 0.8). In fact, the loss is minimized for
# w1=w2=0.0, in which case each example incurs a loss of ln(2). The overall
# (average) loss should then be ln(2) and the logits should be approximately
# 0.0 for each sample.
logreg_classifier = kernel_estimators.KernelLinearClassifier(
feature_columns=[multi_dim_feature])
logreg_classifier.fit(input_fn=_linearly_inseparable_binary_input_fn,
steps=50)
logreg_metrics = logreg_classifier.evaluate(
input_fn=_linearly_inseparable_binary_input_fn, steps=1)
logreg_loss = logreg_metrics['loss']
logreg_accuracy = logreg_metrics['accuracy']
logreg_predictions = logreg_classifier.predict(
input_fn=_linearly_inseparable_binary_input_fn, as_iterable=False)
self.assertAlmostEqual(logreg_loss, np.log(2), places=3)
self.assertLess(logreg_accuracy, 0.8)
self.assertAllClose(logreg_predictions['logits'],
[[0.0], [0.0], [0.0], [0.0]])
# Using kernel mappers allows to discover non-linearities in data. Mapping
# the data to a higher dimensional feature space using approx RBF kernels,
# substantially reduces the loss and leads to perfect classification
# accuracy.
kernel_mappers = {
multi_dim_feature:
[RandomFourierFeatureMapper(2, 30, 0.6, 1, 'rffm')]
}
kernelized_logreg_classifier = kernel_estimators.KernelLinearClassifier(
feature_columns=[], kernel_mappers=kernel_mappers)
kernelized_logreg_classifier.fit(
input_fn=_linearly_inseparable_binary_input_fn, steps=50)
kernelized_logreg_metrics = kernelized_logreg_classifier.evaluate(
input_fn=_linearly_inseparable_binary_input_fn, steps=1)
kernelized_logreg_loss = kernelized_logreg_metrics['loss']
kernelized_logreg_accuracy = kernelized_logreg_metrics['accuracy']
self.assertLess(kernelized_logreg_loss, 0.2)
self.assertEqual(kernelized_logreg_accuracy, 1.0)
def testVariablesWithAndWithoutKernels(self):
"""Tests variables w/ and w/o kernel."""
multi_dim_feature = layers.real_valued_column('multi_dim_feature',
dimension=2)
linear_classifier = kernel_estimators.KernelLinearClassifier(
feature_columns=[multi_dim_feature])
linear_classifier.fit(input_fn=_linearly_inseparable_binary_input_fn,
steps=50)
linear_variables = linear_classifier.get_variable_names()
self.assertIn('linear/multi_dim_feature/weight', linear_variables)
self.assertIn('linear/bias_weight', linear_variables)
linear_weights = linear_classifier.get_variable_value(
'linear/multi_dim_feature/weight')
linear_bias = linear_classifier.get_variable_value(
'linear/bias_weight')
kernel_mappers = {
multi_dim_feature:
[RandomFourierFeatureMapper(2, 30, 0.6, 1, 'rffm')]
}
kernel_linear_classifier = kernel_estimators.KernelLinearClassifier(
feature_columns=[], kernel_mappers=kernel_mappers)
kernel_linear_classifier.fit(
input_fn=_linearly_inseparable_binary_input_fn, steps=50)
kernel_linear_variables = kernel_linear_classifier.get_variable_names()
self.assertIn('linear/multi_dim_feature_MAPPED/weight',
kernel_linear_variables)
self.assertIn('linear/bias_weight', kernel_linear_variables)
kernel_linear_weights = kernel_linear_classifier.get_variable_value(
'linear/multi_dim_feature_MAPPED/weight')
kernel_linear_bias = kernel_linear_classifier.get_variable_value(
'linear/bias_weight')
# The feature column used for linear classification (no kernels) has
# dimension 2 so the model will learn a 2-dimension weights vector (and a
# scalar for the bias). In the kernelized model, the features are mapped to
# a 30-dimensional feature space and so the weights variable will also have
# dimension 30.
self.assertEqual(2, len(linear_weights))
self.assertEqual(1, len(linear_bias))
self.assertEqual(30, len(kernel_linear_weights))
self.assertEqual(1, len(kernel_linear_bias))
def testClassifierWithAndWithoutKernelsNoRealValuedColumns(self):
"""Tests kernels have no effect for non-real valued columns ."""
def input_fn():
return {
'price':
constant_op.constant([[0.4], [0.6], [0.3]]),
'country':
sparse_tensor.SparseTensor(values=['IT', 'US', 'GB'],
indices=[[0, 0], [1, 3], [2, 1]],
dense_shape=[3, 5]),
}, constant_op.constant([[1], [0], [1]])
price = layers.real_valued_column('price')
country = layers.sparse_column_with_hash_bucket('country',
hash_bucket_size=5)
linear_classifier = kernel_estimators.KernelLinearClassifier(
feature_columns=[price, country])
linear_classifier.fit(input_fn=input_fn, steps=100)
linear_metrics = linear_classifier.evaluate(input_fn=input_fn, steps=1)
linear_loss = linear_metrics['loss']
linear_accuracy = linear_metrics['accuracy']
kernel_mappers = {
country: [RandomFourierFeatureMapper(2, 30, 0.6, 1, 'rffm')]
}
kernel_linear_classifier = kernel_estimators.KernelLinearClassifier(
feature_columns=[price, country], kernel_mappers=kernel_mappers)
kernel_linear_classifier.fit(input_fn=input_fn, steps=100)
kernel_linear_metrics = kernel_linear_classifier.evaluate(
input_fn=input_fn, steps=1)
kernel_linear_loss = kernel_linear_metrics['loss']
kernel_linear_accuracy = kernel_linear_metrics['accuracy']
# The kernel mapping is applied to a non-real-valued feature column and so
# it should have no effect on the model. The loss and accuracy of the
# "kernelized" model should match the loss and accuracy of the initial model
# (without kernels).
self.assertAlmostEqual(linear_loss, kernel_linear_loss, delta=0.01)
self.assertAlmostEqual(linear_accuracy,
kernel_linear_accuracy,
delta=0.01)
def testTwoMapperObjects(self):
x = constant_op.constant([[2.0, 1.0, 0.0]])
y = constant_op.constant([[1.0, -1.0, 2.0]])
stddev = 3.0
with self.test_session():
# The mapped dimension is fairly small, so the kernel approximation is
# very rough.
rffm1 = RandomFourierFeatureMapper(3, 100, stddev)
rffm2 = RandomFourierFeatureMapper(3, 100, stddev)
mapped_x1 = rffm1.map(x)
mapped_y1 = rffm1.map(y)
mapped_x2 = rffm2.map(x)
mapped_y2 = rffm2.map(y)
approx_kernel_value1 = _inner_product(mapped_x1, mapped_y1)
approx_kernel_value2 = _inner_product(mapped_x2, mapped_y2)
self.assertAllClose(
approx_kernel_value1.eval(), approx_kernel_value2.eval(), atol=0.01)