def test_gated(self):
np.random.seed(1234)
kernel = np.random.normal(size=(5, 6)).astype(np.float64)
bias = np.random.normal(size=(6, )).astype(np.float64)
x = np.random.normal(size=(11, 7, 5)).astype(np.float64)
normalizer_fn = lambda x: x * 2. + 1.
activation_fn = lambda x: x * 1.5 - 3.
self.assertGreater(
np.min(
np.abs(
normalizer_fn(activation_fn(x)) -
activation_fn(normalizer_fn(x)))), 1.)
with self.test_session() as sess:
normalized_kernel = l2_normalize(kernel, axis=0)
output, gate = np.split(normalizer_fn(np.dot(x,
normalized_kernel)),
2,
axis=-1)
ans = activation_fn(output) * safe_sigmoid(gate + 1.1)
self.assertEqual(ans.shape, (11, 7, 3))
np.testing.assert_allclose(sess.run(
dense(tf.constant(x, dtype=tf.float64),
3,
kernel=tf.constant(kernel),
bias=tf.constant(bias),
activation_fn=activation_fn,
normalizer_fn=normalizer_fn,
weight_norm=True,
gated=True,
gate_sigmoid_bias=1.1)),
ans,
rtol=1e-5)
def test_normalization_and_activation(self):
np.random.seed(1234)
kernel = np.random.normal(size=(5, 3)).astype(np.float64)
bias = np.random.normal(size=(3,)).astype(np.float64)
x = np.random.normal(size=(11, 7, 5)).astype(np.float64)
normalizer_fn = lambda x: x * 2. + 1.
activation_fn = lambda x: x * 1.5 - 3.
self.assertGreater(
np.min(np.abs(normalizer_fn(activation_fn(x)) -
activation_fn(normalizer_fn(x)))),
1.
)
with self.test_session() as sess:
# test weight_norm + normalizer + activation
normalized_kernel = l2_normalize(kernel, axis=0)
ans = activation_fn(normalizer_fn(np.dot(x, normalized_kernel)))
self.assertEqual(ans.shape, (11, 7, 3))
np.testing.assert_allclose(
sess.run(
dense(
tf.constant(x, dtype=tf.float64), 3,
kernel=tf.constant(kernel),
bias=tf.constant(bias),
activation_fn=activation_fn,
normalizer_fn=normalizer_fn,
weight_norm=True
)
),
ans,
rtol=1e-5
)
# test weight_norm + normalizer + activation, use_bias is True
ans = activation_fn(
normalizer_fn(np.dot(x, normalized_kernel) + bias))
self.assertEqual(ans.shape, (11, 7, 3))
np.testing.assert_allclose(
sess.run(
dense(
tf.constant(x, dtype=tf.float64), 3,
kernel=tf.constant(kernel),
bias=tf.constant(bias),
activation_fn=activation_fn,
normalizer_fn=normalizer_fn,
weight_norm=True,
use_bias=True
)
),
ans,
rtol=1e-5
)
def test_gated(self):
assert_allclose = functools.partial(np.testing.assert_allclose,
rtol=1e-5,
atol=1e-5)
with mock.patch('tensorflow.nn.conv2d', patched_conv2d), \
self.test_session() as sess:
np.random.seed(1234)
x = np.random.normal(size=[17, 11, 32, 31, 5]).astype(np.float32)
kernel = np.random.random(size=[3, 4, 5, 14]).astype(np.float32)
normalized_kernel = l2_normalize(kernel, axis=(0, 1, 2))
kernel = kernel.astype(np.float32)
bias = np.random.random(size=[14]).astype(np.float32)
normalizer_fn = lambda x: x * 1.5 - 3.
activation_fn = lambda x: x * 2. + 1.
assert_allclose(
self.run_conv2d(x,
7, (3, 4),
'same',
kernel,
bias,
1,
1,
channels_last=True,
weight_norm=True,
normalizer_fn=normalizer_fn,
activation_fn=activation_fn,
gated=True,
gate_sigmoid_bias=1.1),
self.conv2d_ans(x,
'same',
normalized_kernel,
None,
1,
1,
normalizer_fn=normalizer_fn,
activation_fn=activation_fn,
gated=True,
gate_sigmoid_bias=1.1))
def check(self, x, padding, kernel, bias, strides):
"""Integrated tests for specific argument combinations."""
assert_allclose = functools.partial(np.testing.assert_allclose,
rtol=1e-5,
atol=1e-5)
strides = (strides, ) * 2 if is_integer(strides) else tuple(strides)
x_shape = (x.shape[-3], x.shape[-2])
x_channels = x.shape[-1]
kernel_size = kernel.shape[0], kernel.shape[1]
# compute the input for the deconv
y = Conv2dTestCase.conv2d_ans(x, padding, kernel, None, strides, 1)
y_shape = (y.shape[-3], y.shape[-2])
y_channels = y.shape[-1]
# test explicit output_shape, NHWC
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
None,
strides,
channels_last=True,
use_bias=False)
self.assertEqual(deconv_out.shape, x.shape)
# memorize the linear output for later tests
linear_out = np.copy(deconv_out)
# test explicit output_shape, NCHW
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
None,
strides,
channels_last=False,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test explicit dynamic output_shape, NHWC
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
tf.constant(x_shape),
padding,
kernel,
None,
strides,
channels_last=True,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test explicit dynamic output_shape, NCHW
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
tf.constant(x_shape),
padding,
kernel,
None,
strides,
channels_last=False,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test dynamic input, explicit dynamic output_shape, NHWC
ph = tf.placeholder(dtype=tf.float32,
shape=(None, ) * (len(y.shape) - 3) +
(None, None, y_channels))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
tf.constant(x_shape),
padding,
kernel,
None,
strides,
channels_last=True,
ph=ph,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test dynamic input, explicit dynamic output_shape, NCHW
ph = tf.placeholder(dtype=tf.float32,
shape=(None, ) * (len(y.shape) - 3) +
(y_channels, None, None))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
tf.constant(x_shape),
padding,
kernel,
None,
strides,
channels_last=False,
ph=ph,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# if the given payload shape matches the auto-inferred shape
# further test not giving explicit output_shape
def axis_matches(i):
return x_shape[i] == get_deconv_output_length(
y_shape[i], kernel_size[i], strides[i], padding)
if all(axis_matches(i) for i in (0, 1)):
# test static input, implicit output_shape, NHWC
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
None,
padding,
kernel,
None,
strides,
channels_last=True,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test static input, implicit output_shape, NCHW
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
None,
padding,
kernel,
None,
strides,
channels_last=False,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test dynamic input, implicit output_shape, NHWC
ph = tf.placeholder(dtype=tf.float32,
shape=(None, ) * (len(y.shape) - 3) +
(None, None, y_channels))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
None,
padding,
kernel,
None,
strides,
channels_last=True,
ph=ph,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test dynamic input, implicit output_shape, NCHW
ph = tf.placeholder(dtype=tf.float32,
shape=(None, ) * (len(y.shape) - 3) +
(y_channels, None, None))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
None,
padding,
kernel,
None,
strides,
channels_last=False,
ph=ph,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test normalization and activation
activation_fn = lambda x: x * 2. + 1.
normalizer_fn = lambda x: x * 1.5 - 3.
ans = activation_fn(normalizer_fn(linear_out))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
bias,
strides,
channels_last=True,
normalizer_fn=normalizer_fn,
activation_fn=activation_fn)
assert_allclose(deconv_out, ans)
# test normalization and activation and force using bias
ans = activation_fn(normalizer_fn(linear_out + bias))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
bias,
strides,
channels_last=False,
use_bias=True,
normalizer_fn=normalizer_fn,
activation_fn=activation_fn)
assert_allclose(deconv_out, ans)
# test weight norm
normalized_kernel = l2_normalize(kernel, axis=(0, 1, 2))
ans = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
normalized_kernel,
None,
strides,
channels_last=True,
use_bias=False)
deconv_out = Deconv2dTestCase.run_deconv2d(
y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
None,
strides,
channels_last=False,
use_bias=False,
weight_norm=True,
# this can force not using scale in weight_norm
normalizer_fn=(lambda x: x))
assert_allclose(deconv_out, ans)
# test gated
activation_fn = lambda x: x * 2. + 1.
normalizer_fn = lambda x: x * 1.5 - 3.
output, gate = np.split(normalizer_fn(linear_out), 2, axis=-1)
ans = activation_fn(output) * safe_sigmoid(gate + 1.1)
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels // 2,
kernel_size,
x_shape,
padding,
kernel,
bias,
strides,
channels_last=True,
normalizer_fn=normalizer_fn,
activation_fn=activation_fn,
gated=True,
gate_sigmoid_bias=1.1)
assert_allclose(deconv_out, ans)