def testReuseVariables(self): height, width = 3, 3 images = random_ops.random_uniform((5, height, width, 4), seed=1) normalization_ops.layer_norm(images, scale=True, scope='IN') normalization_ops.layer_norm(images, scale=True, scope='IN', reuse=True) beta = get_variables('beta') gamma = get_variables('gamma') self.assertEqual(1, len(beta)) self.assertEqual(1, len(gamma))
def testValueCorrectWithReuseVars(self): height, width = 3, 3 image_shape = (10, height, width, 4) images = random_ops.random_uniform(image_shape, seed=1) output_train = normalization_ops.layer_norm(images, scope='IN') output_eval = normalization_ops.layer_norm(images, scope='IN', reuse=True) with self.cached_session() as sess: sess.run(variables.global_variables_initializer()) # output_train and output_eval should be the same. train_np, eval_np = sess.run([output_train, output_eval]) self.assertAllClose(train_np, eval_np)
def testCreateVariables_NCHW(self): height, width = 3, 3 images = random_ops.random_uniform((5, 2, height, width), seed=1) normalization_ops.layer_norm(images, channels_axis=-3, center=True, scale=True) beta = get_variables('beta')[0] gamma = get_variables('gamma')[0] self.assertEqual('LayerNorm/beta', beta.op.name) self.assertEqual('LayerNorm/gamma', gamma.op.name)
def doOutputTest(self, input_shape, channels_axis=None, reduction_axes=None, tol=1e-1): # Select the axis for the channel and the dimensions along which statistics # are accumulated. if channels_axis < 0: channels_axis += len(input_shape) reduced_axes = [channels_axis + 1] for a in reduction_axes: if a < 0: a += len(input_shape) if a < channels_axis: reduced_axes.append(a) else: reduced_axes.append(a + 1) reduced_axes = tuple(reduced_axes) channels = input_shape[channels_axis] # Calculate the final shape for the output Tensor. axes_before_channels = input_shape[:channels_axis] axes_after_channels = input_shape[channels_axis + 1:] outputs_shape = (axes_before_channels + [1, channels] + axes_after_channels) # Calculate the final shape for the output statistics. reduced_shape = [] for i, a in enumerate(outputs_shape): if i not in reduced_axes: reduced_shape.append(a) mu = 1.0 sigma = 1.0 # Determine shape of Tensor after normalization. expected_mean = np.zeros(reduced_shape) expected_var = np.ones(reduced_shape) inputs = random_ops.random_normal(input_shape, seed=0) * sigma + mu output_op = normalization_ops.layer_norm(inputs, center=False, scale=False, channels_axis=channels_axis, training=True) with self.cached_session() as sess: sess.run(variables.global_variables_initializer()) outputs = sess.run(output_op) # Make sure that there are no NaNs self.assertFalse(np.isnan(outputs).any()) outputs = np.swapaxes(outputs, 0, channels_axis) outputs = np.reshape(outputs, outputs_shape) mean = np.mean(outputs, axis=reduced_axes, dtype=np.float32) var = np.var(outputs, axis=reduced_axes, dtype=np.float32) # The mean and variance of each example should be close to 0 and 1 # respectively. self.assertAllClose(expected_mean, mean, rtol=tol, atol=tol) self.assertAllClose(expected_var, var, rtol=tol, atol=tol)
def testCreateOpNoScaleCenter(self): height, width = 3, 3 images = random_ops.random_uniform((5, height, width, 3), dtype=dtypes.float32, seed=1) output = normalization_ops.layer_norm(images, center=False, scale=False) self.assertListEqual([5, height, width, 3], output.shape.as_list()) self.assertEqual(0, len(get_variables('beta'))) self.assertEqual(0, len(get_variables('gamma')))
def testCreateOp(self): height, width = 3, 3 images = random_ops.random_uniform((5, height, width, 2), seed=1) output = normalization_ops.layer_norm(images, channels_axis=-1) self.assertListEqual([5, height, width, 2], output.shape.as_list())
def testParamsShapeNotFullyDefinedChannelsAxis(self): inputs = array_ops.placeholder(dtypes.float32, shape=(1, 3, 4, None)) with self.assertRaisesRegex(ValueError, 'undefined channel dimension'): normalization_ops.layer_norm(inputs, channels_axis=-1)
def testUnknownShape(self): inputs = array_ops.placeholder(dtypes.float32) with self.assertRaisesRegex(ValueError, 'undefined rank'): normalization_ops.layer_norm(inputs)
def testAxisIsBad(self): inputs = array_ops.placeholder(dtypes.float32, shape=(1, 2, 4, 5)) with self.assertRaisesRegex(ValueError, 'Axis is out of bounds.'): normalization_ops.layer_norm(inputs, channels_axis=5)
def func(a, b): x = tf.matmul(a, b) x = normalization_ops.layer_norm(x) x = nn_ops.gelu(x) return x