Пример #1
0
    def test_gated(self):
        np.random.seed(1234)
        kernel = np.random.normal(size=(5, 6)).astype(np.float64)
        bias = np.random.normal(size=(6, )).astype(np.float64)
        x = np.random.normal(size=(11, 7, 5)).astype(np.float64)

        normalizer_fn = lambda x: x * 2. + 1.
        activation_fn = lambda x: x * 1.5 - 3.

        self.assertGreater(
            np.min(
                np.abs(
                    normalizer_fn(activation_fn(x)) -
                    activation_fn(normalizer_fn(x)))), 1.)

        with self.test_session() as sess:
            normalized_kernel = l2_normalize(kernel, axis=0)
            output, gate = np.split(normalizer_fn(np.dot(x,
                                                         normalized_kernel)),
                                    2,
                                    axis=-1)
            ans = activation_fn(output) * safe_sigmoid(gate + 1.1)
            self.assertEqual(ans.shape, (11, 7, 3))
            np.testing.assert_allclose(sess.run(
                dense(tf.constant(x, dtype=tf.float64),
                      3,
                      kernel=tf.constant(kernel),
                      bias=tf.constant(bias),
                      activation_fn=activation_fn,
                      normalizer_fn=normalizer_fn,
                      weight_norm=True,
                      gated=True,
                      gate_sigmoid_bias=1.1)),
                                       ans,
                                       rtol=1e-5)
Пример #2
0
    def test_normalization_and_activation(self):
        np.random.seed(1234)
        kernel = np.random.normal(size=(5, 3)).astype(np.float64)
        bias = np.random.normal(size=(3,)).astype(np.float64)
        x = np.random.normal(size=(11, 7, 5)).astype(np.float64)

        normalizer_fn = lambda x: x * 2. + 1.
        activation_fn = lambda x: x * 1.5 - 3.

        self.assertGreater(
            np.min(np.abs(normalizer_fn(activation_fn(x)) -
                          activation_fn(normalizer_fn(x)))),
            1.
        )

        with self.test_session() as sess:
            # test weight_norm + normalizer + activation
            normalized_kernel = l2_normalize(kernel, axis=0)
            ans = activation_fn(normalizer_fn(np.dot(x, normalized_kernel)))
            self.assertEqual(ans.shape, (11, 7, 3))
            np.testing.assert_allclose(
                sess.run(
                    dense(
                        tf.constant(x, dtype=tf.float64), 3,
                        kernel=tf.constant(kernel),
                        bias=tf.constant(bias),
                        activation_fn=activation_fn,
                        normalizer_fn=normalizer_fn,
                        weight_norm=True
                    )
                ),
                ans,
                rtol=1e-5
            )

            # test weight_norm + normalizer + activation, use_bias is True
            ans = activation_fn(
                normalizer_fn(np.dot(x, normalized_kernel) + bias))
            self.assertEqual(ans.shape, (11, 7, 3))
            np.testing.assert_allclose(
                sess.run(
                    dense(
                        tf.constant(x, dtype=tf.float64), 3,
                        kernel=tf.constant(kernel),
                        bias=tf.constant(bias),
                        activation_fn=activation_fn,
                        normalizer_fn=normalizer_fn,
                        weight_norm=True,
                        use_bias=True
                    )
                ),
                ans,
                rtol=1e-5
            )
Пример #3
0
    def test_gated(self):
        assert_allclose = functools.partial(np.testing.assert_allclose,
                                            rtol=1e-5,
                                            atol=1e-5)
        with mock.patch('tensorflow.nn.conv2d', patched_conv2d), \
                self.test_session() as sess:
            np.random.seed(1234)

            x = np.random.normal(size=[17, 11, 32, 31, 5]).astype(np.float32)
            kernel = np.random.random(size=[3, 4, 5, 14]).astype(np.float32)
            normalized_kernel = l2_normalize(kernel, axis=(0, 1, 2))
            kernel = kernel.astype(np.float32)
            bias = np.random.random(size=[14]).astype(np.float32)

            normalizer_fn = lambda x: x * 1.5 - 3.
            activation_fn = lambda x: x * 2. + 1.

            assert_allclose(
                self.run_conv2d(x,
                                7, (3, 4),
                                'same',
                                kernel,
                                bias,
                                1,
                                1,
                                channels_last=True,
                                weight_norm=True,
                                normalizer_fn=normalizer_fn,
                                activation_fn=activation_fn,
                                gated=True,
                                gate_sigmoid_bias=1.1),
                self.conv2d_ans(x,
                                'same',
                                normalized_kernel,
                                None,
                                1,
                                1,
                                normalizer_fn=normalizer_fn,
                                activation_fn=activation_fn,
                                gated=True,
                                gate_sigmoid_bias=1.1))
Пример #4
0
    def check(self, x, padding, kernel, bias, strides):
        """Integrated tests for specific argument combinations."""
        assert_allclose = functools.partial(np.testing.assert_allclose,
                                            rtol=1e-5,
                                            atol=1e-5)
        strides = (strides, ) * 2 if is_integer(strides) else tuple(strides)

        x_shape = (x.shape[-3], x.shape[-2])
        x_channels = x.shape[-1]
        kernel_size = kernel.shape[0], kernel.shape[1]

        # compute the input for the deconv
        y = Conv2dTestCase.conv2d_ans(x, padding, kernel, None, strides, 1)
        y_shape = (y.shape[-3], y.shape[-2])
        y_channels = y.shape[-1]

        # test explicit output_shape, NHWC
        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels,
                                                   kernel_size,
                                                   x_shape,
                                                   padding,
                                                   kernel,
                                                   None,
                                                   strides,
                                                   channels_last=True,
                                                   use_bias=False)
        self.assertEqual(deconv_out.shape, x.shape)

        # memorize the linear output for later tests
        linear_out = np.copy(deconv_out)

        # test explicit output_shape, NCHW
        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels,
                                                   kernel_size,
                                                   x_shape,
                                                   padding,
                                                   kernel,
                                                   None,
                                                   strides,
                                                   channels_last=False,
                                                   use_bias=False)
        assert_allclose(deconv_out, linear_out)

        # test explicit dynamic output_shape, NHWC
        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels,
                                                   kernel_size,
                                                   tf.constant(x_shape),
                                                   padding,
                                                   kernel,
                                                   None,
                                                   strides,
                                                   channels_last=True,
                                                   use_bias=False)
        assert_allclose(deconv_out, linear_out)

        # test explicit dynamic output_shape, NCHW
        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels,
                                                   kernel_size,
                                                   tf.constant(x_shape),
                                                   padding,
                                                   kernel,
                                                   None,
                                                   strides,
                                                   channels_last=False,
                                                   use_bias=False)
        assert_allclose(deconv_out, linear_out)

        # test dynamic input, explicit dynamic output_shape, NHWC
        ph = tf.placeholder(dtype=tf.float32,
                            shape=(None, ) * (len(y.shape) - 3) +
                            (None, None, y_channels))
        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels,
                                                   kernel_size,
                                                   tf.constant(x_shape),
                                                   padding,
                                                   kernel,
                                                   None,
                                                   strides,
                                                   channels_last=True,
                                                   ph=ph,
                                                   use_bias=False)
        assert_allclose(deconv_out, linear_out)

        # test dynamic input, explicit dynamic output_shape, NCHW
        ph = tf.placeholder(dtype=tf.float32,
                            shape=(None, ) * (len(y.shape) - 3) +
                            (y_channels, None, None))
        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels,
                                                   kernel_size,
                                                   tf.constant(x_shape),
                                                   padding,
                                                   kernel,
                                                   None,
                                                   strides,
                                                   channels_last=False,
                                                   ph=ph,
                                                   use_bias=False)
        assert_allclose(deconv_out, linear_out)

        # if the given payload shape matches the auto-inferred shape
        # further test not giving explicit output_shape
        def axis_matches(i):
            return x_shape[i] == get_deconv_output_length(
                y_shape[i], kernel_size[i], strides[i], padding)

        if all(axis_matches(i) for i in (0, 1)):
            # test static input, implicit output_shape, NHWC
            deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                       x_channels,
                                                       kernel_size,
                                                       None,
                                                       padding,
                                                       kernel,
                                                       None,
                                                       strides,
                                                       channels_last=True,
                                                       use_bias=False)
            assert_allclose(deconv_out, linear_out)

            # test static input, implicit output_shape, NCHW
            deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                       x_channels,
                                                       kernel_size,
                                                       None,
                                                       padding,
                                                       kernel,
                                                       None,
                                                       strides,
                                                       channels_last=False,
                                                       use_bias=False)
            assert_allclose(deconv_out, linear_out)

            # test dynamic input, implicit output_shape, NHWC
            ph = tf.placeholder(dtype=tf.float32,
                                shape=(None, ) * (len(y.shape) - 3) +
                                (None, None, y_channels))
            deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                       x_channels,
                                                       kernel_size,
                                                       None,
                                                       padding,
                                                       kernel,
                                                       None,
                                                       strides,
                                                       channels_last=True,
                                                       ph=ph,
                                                       use_bias=False)
            assert_allclose(deconv_out, linear_out)

            # test dynamic input, implicit output_shape, NCHW
            ph = tf.placeholder(dtype=tf.float32,
                                shape=(None, ) * (len(y.shape) - 3) +
                                (y_channels, None, None))
            deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                       x_channels,
                                                       kernel_size,
                                                       None,
                                                       padding,
                                                       kernel,
                                                       None,
                                                       strides,
                                                       channels_last=False,
                                                       ph=ph,
                                                       use_bias=False)
            assert_allclose(deconv_out, linear_out)

        # test normalization and activation
        activation_fn = lambda x: x * 2. + 1.
        normalizer_fn = lambda x: x * 1.5 - 3.
        ans = activation_fn(normalizer_fn(linear_out))

        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels,
                                                   kernel_size,
                                                   x_shape,
                                                   padding,
                                                   kernel,
                                                   bias,
                                                   strides,
                                                   channels_last=True,
                                                   normalizer_fn=normalizer_fn,
                                                   activation_fn=activation_fn)
        assert_allclose(deconv_out, ans)

        # test normalization and activation and force using bias
        ans = activation_fn(normalizer_fn(linear_out + bias))
        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels,
                                                   kernel_size,
                                                   x_shape,
                                                   padding,
                                                   kernel,
                                                   bias,
                                                   strides,
                                                   channels_last=False,
                                                   use_bias=True,
                                                   normalizer_fn=normalizer_fn,
                                                   activation_fn=activation_fn)
        assert_allclose(deconv_out, ans)

        # test weight norm
        normalized_kernel = l2_normalize(kernel, axis=(0, 1, 2))
        ans = Deconv2dTestCase.run_deconv2d(y,
                                            x_channels,
                                            kernel_size,
                                            x_shape,
                                            padding,
                                            normalized_kernel,
                                            None,
                                            strides,
                                            channels_last=True,
                                            use_bias=False)
        deconv_out = Deconv2dTestCase.run_deconv2d(
            y,
            x_channels,
            kernel_size,
            x_shape,
            padding,
            kernel,
            None,
            strides,
            channels_last=False,
            use_bias=False,
            weight_norm=True,
            # this can force not using scale in weight_norm
            normalizer_fn=(lambda x: x))
        assert_allclose(deconv_out, ans)

        # test gated
        activation_fn = lambda x: x * 2. + 1.
        normalizer_fn = lambda x: x * 1.5 - 3.
        output, gate = np.split(normalizer_fn(linear_out), 2, axis=-1)
        ans = activation_fn(output) * safe_sigmoid(gate + 1.1)

        deconv_out = Deconv2dTestCase.run_deconv2d(y,
                                                   x_channels // 2,
                                                   kernel_size,
                                                   x_shape,
                                                   padding,
                                                   kernel,
                                                   bias,
                                                   strides,
                                                   channels_last=True,
                                                   normalizer_fn=normalizer_fn,
                                                   activation_fn=activation_fn,
                                                   gated=True,
                                                   gate_sigmoid_bias=1.1)
        assert_allclose(deconv_out, ans)