def _apply(self, x1, x2, param_expansion_ndims=0):
print('\nLinear._apply')
print('x1: ', type(x1), x1.dtype, x1.shape)
print('x2: ', type(x2), x2.dtype, x2.shape)
if self.origin is not None:
x1 = x1 - self.origin
x2 = x2 - self.origin
print('x1: ', type(x1), x1.dtype, x1.shape)
print('x2: ', type(x2), x2.dtype, x2.shape)
print('x2.T: ', type(tf.transpose(x2)),
tf.transpose(x2).dtype,
tf.transpose(x2).shape)
dot_prod = tf.tensordot(x1, tf.transpose(x2), axes=1)
dot_prod = tf.reshape(dot_prod, [x1.shape[0], x2.shape[1]])
# dot_prod = tf.matmul(x1, x2, transpose_b=True)
print('dot_prod: ', type(dot_prod), dot_prod.dtype, dot_prod.shape)
if self.bias is not None:
bias = util.pad_shape_right_with_ones(self.bias,
param_expansion_ndims)
dot_prod += bias**2
print('dot_prod: ', type(dot_prod), dot_prod.dtype, dot_prod.shape)
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(self.amplitude,
param_expansion_ndims)
dot_prod *= amplitude**2
print('dot_prod: ', type(dot_prod), dot_prod.dtype, dot_prod.shape)
return dot_prod
def _apply(self, x1, x2, param_expansion_ndims=0):
if self.shift is None:
dot_prod = util.sum_rightmost_ndims_preserving_shape(
x1 * x2, ndims=self.feature_ndims)
else:
dot_prod = util.sum_rightmost_ndims_preserving_shape(
(x1 - self.shift) * (x2 - self.shift),
ndims=self.feature_ndims)
if self.exponent is not None:
exponent = util.pad_shape_right_with_ones(
self.exponent, param_expansion_ndims)
dot_prod **= exponent
if self.slope_variance is not None:
slope_variance = util.pad_shape_right_with_ones(
self.slope_variance, param_expansion_ndims)
dot_prod *= slope_variance ** 2.
if self.bias_variance is not None:
bias_variance = util.pad_shape_right_with_ones(
self.bias_variance, param_expansion_ndims)
dot_prod += bias_variance ** 2.
return dot_prod
def testPadShapeRightWithOnesDynamicShape(self):
# Test partially unknown shape
x = tf.placeholder_with_default(np.ones([3], np.float32), [None])
expanded = util.pad_shape_right_with_ones(x, 3)
self.assertAllEqual(expanded.shape.as_list(), [None, 1, 1, 1])
self.assertAllEqual(self.evaluate(expanded).shape, [3, 1, 1, 1])
# Test totally unknown shape
x = tf.placeholder_with_default(np.ones([3], np.float32), None)
expanded = util.pad_shape_right_with_ones(x, 3)
self.assertIsNone(expanded.shape.ndims)
self.assertAllEqual(self.evaluate(expanded).shape, [3, 1, 1, 1])
def testPadShapeRightWithOnesDynamicShape(self):
# Test partially unknown shape
x = tf.placeholder_with_default(np.ones([3], np.float32), [None])
expanded = util.pad_shape_right_with_ones(x, 3)
self.assertAllEqual(expanded.shape.as_list(), [None, 1, 1, 1])
self.assertAllEqual(self.evaluate(expanded).shape, [3, 1, 1, 1])
# Test totally unknown shape
x = tf.placeholder_with_default(np.ones([3], np.float32), None)
expanded = util.pad_shape_right_with_ones(x, 3)
self.assertIsNone(expanded.shape.ndims)
self.assertAllEqual(self.evaluate(expanded).shape, [3, 1, 1, 1])
def _apply(self, x1, x2, param_expansion_ndims=0):
exponent = -0.5 * util.sum_rightmost_ndims_preserving_shape(
tf.math.squared_difference(x1, x2), self.feature_ndims)
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, param_expansion_ndims)
exponent /= length_scale**2
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(self.amplitude,
param_expansion_ndims)
exponent += 2. * tf.math.log(amplitude)
return tf.exp(exponent)
def _apply(self, x1, x2, param_expansion_ndims=0):
exponent = -0.5 * util.sum_rightmost_ndims_preserving_shape(
tf.squared_difference(x1, x2), self.feature_ndims)
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, param_expansion_ndims)
exponent /= length_scale**2
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(
self.amplitude, param_expansion_ndims)
exponent += 2. * tf.log(amplitude)
return tf.exp(exponent)
def _apply(self, x1, x2, param_expansion_ndims=0):
norm = tf.sqrt(
util.sum_rightmost_ndims_preserving_shape(
tf.squared_difference(x1, x2), self.feature_ndims))
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, ndims=param_expansion_ndims)
norm /= length_scale
result = tf.exp(-norm)
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(
self.amplitude, ndims=param_expansion_ndims)
result *= amplitude**2
return result
def _apply(self, x1, x2, param_expansion_ndims=0):
# Use util.sqrt_with_finite_grads to avoid NaN gradients when `x1 == x2`.
norm = util.sqrt_with_finite_grads(
util.sum_rightmost_ndims_preserving_shape(
tf.squared_difference(x1, x2), self.feature_ndims))
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, ndims=param_expansion_ndims)
norm /= length_scale
log_result = -norm
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(
self.amplitude, ndims=param_expansion_ndims)
log_result += 2. * tf.log(amplitude)
return tf.exp(log_result)
def _apply(self, x1, x2, param_expansion_ndims=0):
# Use util.sqrt_with_finite_grads to avoid NaN gradients when `x1 == x2`.
norm = util.sqrt_with_finite_grads(
util.sum_rightmost_ndims_preserving_shape(
tf.math.squared_difference(x1, x2), self.feature_ndims))
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, ndims=param_expansion_ndims)
norm /= length_scale
log_result = -norm
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(
self.amplitude, ndims=param_expansion_ndims)
log_result += 2. * tf.math.log(amplitude)
return tf.exp(log_result)
def _apply(self, x1, x2, param_expansion_ndims=0):
x1 = tf.convert_to_tensor(x1)
x2 = tf.convert_to_tensor(x2)
multiplier = kernels_util.pad_shape_right_with_ones(
self._multiplier, param_expansion_ndims)
return multiplier * tf.reduce_sum(x1 + x2, axis=-1)
def _apply(self, x1, x2, param_expansion_ndims=0):
x1 = tf.convert_to_tensor(value=x1)
x2 = tf.convert_to_tensor(value=x2)
multiplier = kernels_util.pad_shape_right_with_ones(
self._multiplier, param_expansion_ndims)
return multiplier * tf.reduce_sum(input_tensor=x1 + x2, axis=-1)
def _apply(self, x1, x2, param_expansion_ndims=0):
# Use util.sqrt_with_finite_grads to avoid NaN gradients when `x1 == x2`.
norm = util.sqrt_with_finite_grads(
util.sum_rightmost_ndims_preserving_shape(
tf.squared_difference(x1, x2), self.feature_ndims))
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, ndims=param_expansion_ndims)
norm /= length_scale
series_term = np.sqrt(5) * norm
result = (1. + series_term + series_term**2 / 3.) * tf.exp(-series_term)
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(self.amplitude,
param_expansion_ndims)
result *= amplitude**2
return result
def testPadShapeRightWithOnesCanBeGraphNoop(self):
# First ensure graph actually *is* changed when we use non-trivial ndims.
# Use an explicitly created graph, to make sure no whacky test fixture graph
# reuse is going on in the background.
g = tf.Graph()
with g.as_default():
x = tf.constant(np.ones([3], np.float32))
graph_def = g.as_graph_def()
x = util.pad_shape_right_with_ones(x, 3)
self.assertNotEqual(graph_def, g.as_graph_def())
# Now verify that graphdef is unchanged (no extra ops) when we pass ndims=0.
g = tf.Graph()
with g.as_default():
x = tf.constant(np.ones([3], np.float32))
graph_def = g.as_graph_def()
x = util.pad_shape_right_with_ones(x, 0)
self.assertEqual(graph_def, g.as_graph_def())
def _apply(self, x1, x2, param_expansion_ndims=0):
x1 = tf.convert_to_tensor(x1)
x2 = tf.convert_to_tensor(x2)
exponent = -.5 * util.sum_rightmost_ndims_preserving_shape(
tf.squared_difference(x1, x2), self.feature_ndims)
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, param_expansion_ndims)
exponent /= length_scale**2
exponential = tf.exp(exponent)
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(self.amplitude,
param_expansion_ndims)
return amplitude * exponential
else:
return exponential
def testPadShapeRightWithOnesCanBeGraphNoop(self):
# First ensure graph actually *is* changed when we use non-trivial ndims.
# Use an explicitly created graph, to make sure no whacky test fixture graph
# reuse is going on in the background.
g = tf.Graph()
with g.as_default():
x = tf.constant(np.ones([3], np.float32))
graph_def = g.as_graph_def()
x = util.pad_shape_right_with_ones(x, 3)
self.assertNotEqual(graph_def, g.as_graph_def())
# Now verify that graphdef is unchanged (no extra ops) when we pass ndims=0.
g = tf.Graph()
with g.as_default():
x = tf.constant(np.ones([3], np.float32))
graph_def = g.as_graph_def()
x = util.pad_shape_right_with_ones(x, 0)
self.assertEqual(graph_def, g.as_graph_def())
def _apply(self, x1, x2, param_expansion_ndims=0):
# Use util.sqrt_with_finite_grads to avoid NaN gradients when `x1 == x2`.norm = util.sqrt_with_finite_grads(
#x1 = B,Np,D -> B,Np,1,D
#x2 = B,N,D -> B,1,N,D
#B, Np,N
with tf.control_dependencies([
tf.assert_equal(
tf.shape(self.heights)[-1] + 1,
tf.shape(self.edgescales)[-1])
]):
norm = util.sqrt_with_finite_grads(
util.sum_rightmost_ndims_preserving_shape(
tf.squared_difference(x1, x2), self.feature_ndims))
#B(1),1,Np,N
norm = tf.expand_dims(norm, -(param_expansion_ndims + 1))
#B(1), H+1, 1, 1
edgescales = util.pad_shape_right_with_ones(
self.edgescales, ndims=param_expansion_ndims)
norm *= edgescales
norm *= 2 * np.pi
zeros = tf.zeros(tf.shape(self.heights)[:-1],
dtype=self.heights.dtype)[..., None]
# B(1),1+H+1
heights = tf.concat([zeros, self.heights, zeros], axis=-1)
# B(1), H+1
dheights = heights[..., :-1] - heights[..., 1:]
#B(1), H+1, 1, 1
dheights = util.pad_shape_right_with_ones(dheights,
ndims=param_expansion_ndims)
#B(1), H+1, 1, 1
dheights *= edgescales
def _sinc(x):
return tf.sin(x) * tf.reciprocal(x)
#B(1), H+1, N, Np
sincs = tf.where(tf.less(norm, tf.constant(1e-15, dtype=norm.dtype)),
tf.ones_like(norm), _sinc(norm))
#B(1), H+1, N, Np
result = dheights * sincs
#B(1), N,Np
return tf.reduce_sum(result, axis=-3)
def _apply(self, x1, x2, param_expansion_ndims=0):
dot_prod = tf.tensordot(x1, tf.transpose(x2), axes=1)
dot_prod = tf.reshape(dot_prod, [x1.shape[0], x2.shape[1]])
if self.bias is not None:
bias = util.pad_shape_right_with_ones(self.bias,
param_expansion_ndims)
dot_prod += bias**2.
if self.power is not None:
power = util.pad_shape_right_with_ones(self.power,
param_expansion_ndims)
dot_prod **= power
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(self.amplitude,
param_expansion_ndims)
dot_prod *= amplitude**2.
return dot_prod
def _apply(self, x1, x2, param_expansion_ndims=0):
difference = util.sum_rightmost_ndims_preserving_shape(
tf.squared_difference(x1, x2), ndims=self.feature_ndims)
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, ndims=param_expansion_ndims)
difference /= (length_scale ** 2)
scale_mixture_rate = 1.
if self.scale_mixture_rate is not None:
scale_mixture_rate = util.pad_shape_right_with_ones(
self.scale_mixture_rate, ndims=param_expansion_ndims)
difference /= (2 * scale_mixture_rate)
result = (1. + difference) ** -scale_mixture_rate
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(
self.amplitude, ndims=param_expansion_ndims)
result *= amplitude ** 2
return result
def _apply(self, x1, x2, param_expansion_ndims=0):
difference = util.sum_rightmost_ndims_preserving_shape(
tf.squared_difference(x1, x2), ndims=self.feature_ndims)
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, ndims=param_expansion_ndims)
difference /= length_scale**2
scale_mixture_rate = 1.
if self.scale_mixture_rate is not None:
scale_mixture_rate = util.pad_shape_right_with_ones(
self.scale_mixture_rate, ndims=param_expansion_ndims)
difference /= scale_mixture_rate
result = (1. + difference)**-scale_mixture_rate
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(
self.amplitude, ndims=param_expansion_ndims)
result *= amplitude**2
return result
def _apply(self, x1, x2, param_expansion_ndims=0):
difference = np.pi * tf.abs(x1 - x2)
if self.period is not None:
# period acts as a batch of periods, and hence we must additionally
# pad the shape with self.feature_ndims number of ones.
period = util.pad_shape_right_with_ones(
self.period, ndims=(param_expansion_ndims + self.feature_ndims))
difference /= period
log_kernel = util.sum_rightmost_ndims_preserving_shape(
-2 * tf.sin(difference) ** 2, ndims=self.feature_ndims)
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, ndims=param_expansion_ndims)
log_kernel /= length_scale ** 2
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(
self.amplitude, ndims=param_expansion_ndims)
log_kernel += 2. * tf.math.log(amplitude)
return tf.exp(log_kernel)
def _apply(self, x1, x2, param_expansion_ndims=0):
difference = np.pi * tf.abs(x1 - x2)
if self.period is not None:
# period acts as a batch of periods, and hence we must additionally
# pad the shape with self.feature_ndims number of ones.
period = util.pad_shape_right_with_ones(
self.period, ndims=(param_expansion_ndims + self.feature_ndims))
difference /= period
log_kernel = util.sum_rightmost_ndims_preserving_shape(
-2 * tf.sin(difference) ** 2, ndims=self.feature_ndims)
if self.length_scale is not None:
length_scale = util.pad_shape_right_with_ones(
self.length_scale, ndims=param_expansion_ndims)
log_kernel /= length_scale ** 2
if self.amplitude is not None:
amplitude = util.pad_shape_right_with_ones(
self.amplitude, ndims=param_expansion_ndims)
log_kernel += 2. * tf.log(amplitude)
return tf.exp(log_kernel)
def testPadShapeRightWithOnes(self):
# Test nominal behavior.
x = np.ones([3], np.float32)
self.assertAllEqual(
self.evaluate(util.pad_shape_right_with_ones(x, 3)).shape,
[3, 1, 1, 1])
def testPadShapeRightWithOnes(self):
# Test nominal behavior.
x = np.ones([3], np.float32)
self.assertAllEqual(
self.evaluate(util.pad_shape_right_with_ones(x, 3)).shape,
[3, 1, 1, 1])
