def testCreateVariables_NHWC(self): height, width = 3, 3 images = random_ops.random_uniform((5, height, width, 8), seed=1) normalization_ops.group_norm(images, groups=4, channels_axis=-1, center=True, scale=True) beta = get_variables('beta')[0] gamma = get_variables('gamma')[0] self.assertEqual('GroupNorm/beta', beta.op.name) self.assertEqual('GroupNorm/gamma', gamma.op.name)
def testReuseVariables(self): height, width = 3, 3 images = random_ops.random_uniform((5, height, width, 4), seed=1) normalization_ops.group_norm(images, groups=2, scale=True, scope='IN') normalization_ops.group_norm(images, groups=2, scale=True, scope='IN', reuse=True) beta = get_variables('beta') gamma = get_variables('gamma') self.assertEqual(1, len(beta)) self.assertEqual(1, len(gamma))
def testValueCorrectWithReuseVars(self): height, width = 3, 3 image_shape = (10, height, width, 4) images = random_ops.random_uniform(image_shape, seed=1) output_train = normalization_ops.group_norm(images, groups=2, scope='IN') output_eval = normalization_ops.group_norm(images, groups=2, scope='IN', reuse=True) with self.cached_session() as sess: sess.run(variables.global_variables_initializer()) # output_train and output_eval should be the same. train_np, eval_np = sess.run([output_train, output_eval]) self.assertAllClose(train_np, eval_np)
def norm(x, opts, is_training=True, zero_gamma_init=False): # We initialise the parameters depending on the gamma_init argument if zero_gamma_init: p_init = { 'gamma': tf.zeros_initializer(), 'beta': tf.zeros_initializer() } else: p_init = { 'gamma': tf.ones_initializer(), 'beta': tf.zeros_initializer() } norm_type = ("GROUP" if opts["group_norm"] else "BATCH" if opts["batch_norm"] else None) if norm_type == "BATCH": x = tf.layers.batch_normalization(x, fused=True, center=True, scale=True, training=is_training, trainable=True, momentum=opts["BN_decay"], epsilon=1e-5, gamma_initializer=p_init['gamma'], beta_initializer=p_init['beta']) elif norm_type == "GROUP": # Adding arguments to group norm x = normalization_ops.group_norm(x, groups=opts["groups"], param_initializers=p_init) tf.add_to_collection("activations", x) return x
def norm(x: tf.Tensor, norm_type='BATCH', groups=32, training=False) -> tf.Tensor: """ Batch-normalization :param x: (tf.Tensor) input tensor :param norm_type: (str) type of normalization :param groups: (int) size of group :param training: (boolean) training flag :return: """ if norm_type == 'BATCH': # Perhaps use tf.nn.fused_batch_norm instead. x = tf.layers.batch_normalization(x, fused=True, center=True, scale=True, training=training, trainable=training, momentum=0.997, epsilon=1e-5) elif norm_type == 'GROUP': x = normalization_ops.group_norm(x, groups=groups, center=True, scale=True, training=training, trainable=training, channels_axis=-1) return x
def testCreateOp(self): height, width, groups = 3, 3, 4 images = random_ops.random_uniform((5, height, width, 2 * groups), seed=1) output = normalization_ops.group_norm(images, groups=groups, channels_axis=-1) self.assertListEqual([5, height, width, 2 * groups], output.shape.as_list())
def norm(self, x, type='BATCH', groups=32, training=False): if type == 'BATCH': # Perhaps use tf.nn.fused_batch_norm instead. x = tf.layers.batch_normalization(x, fused=True, center=True, scale=True, training=training, trainable=training, momentum=0.997, epsilon=1e-5) elif type == 'GROUP': x = normalization_ops.group_norm(x, groups=groups, center=True, scale=True, training=training, trainable=training, channels_axis=-1) return x
def testCreateOpNoScaleCenter(self): height, width, groups = 3, 3, 7 images = random_ops.random_uniform((5, height, width, 3 * groups), dtype=dtypes.float32, seed=1) output = normalization_ops.group_norm(images, groups=groups, center=False, scale=False) self.assertListEqual([5, height, width, 3 * groups], output.shape.as_list()) self.assertEqual(0, len(get_variables('beta'))) self.assertEqual(0, len(get_variables('gamma')))
def norm(x, is_training=True, norm_type=None, epsilon=1e-6, groups=None): if norm_type == 'batch': x = batch_norm(x, center=True, scale=True, training=is_training, trainable=True, epsilon=epsilon) elif norm_type == 'group': x = normalization_ops.group_norm(x, groups=min(int(x.get_shape()[3]), groups), epsilon=epsilon) else: raise ValueError("{} not a valid norm_type".format(norm_type)) return x
def norm(x, opts, is_training=True): norm_type = 'GROUP' if opts[ "group_norm"] else 'BATCH' if opts['batch_norm'] else None if norm_type == 'BATCH': x = tf.layers.batch_normalization(x, fused=True, center=True, scale=True, training=is_training, trainable=True, momentum=opts["BN_decay"], epsilon=1e-5) elif norm_type == 'GROUP': x = normalization_ops.group_norm(x, groups=opts['groups']) tf.add_to_collection('activations', x) return x
def testDefunInput(self): shape = [10, 10, 10, 30] @function.Defun() def f(): return array_ops.ones(shape) inputs = f() inputs.set_shape(shape) output_op = normalization_ops.group_norm(inputs, groups=5, center=False, scale=False, channels_axis=-1, training=True) with self.cached_session() as sess: sess.run(variables.global_variables_initializer()) outputs = sess.run(output_op) self.assertAllClose(outputs, np.zeros(shape))
def norm(x, opts, is_training=True): norm_type = ("GROUP" if opts["group_norm"] else "BATCH" if opts["batch_norm"] else None) if norm_type == "BATCH": x = tf.layers.batch_normalization( x, fused=True, center=True, scale=True, training=is_training, trainable=True, momentum=opts["BN_decay"], epsilon=1e-5, ) elif norm_type == "GROUP": x = normalization_ops.group_norm(x, groups=opts["groups"]) tf.add_to_collection("activations", x) return x
def testParamsShapeNotFullyDefinedChannelsAxis(self): inputs = array_ops.placeholder(dtypes.float32, shape=(1, 3, 4, None)) with self.assertRaisesRegex(ValueError, 'undefined channel dimension'): normalization_ops.group_norm(inputs, channels_axis=-1)
def doOutputTest(self, input_shape, channels_axis=None, reduction_axes=None, groups=2, tol=1e-1): # Select the axis for the channel and the dimensions along which statistics # are accumulated. if channels_axis < 0: channels_axis += len(input_shape) reduced_axes = [channels_axis + 1] for a in reduction_axes: if a < 0: a += len(input_shape) if a < channels_axis: reduced_axes.append(a) else: reduced_axes.append(a + 1) reduced_axes = tuple(reduced_axes) channels = input_shape[channels_axis] group_size = channels // groups # Calculate the final shape for the output Tensor. axes_before_channels = input_shape[:channels_axis] axes_after_channels = input_shape[channels_axis + 1:] outputs_shape = (axes_before_channels + [groups, group_size] + axes_after_channels) # Calculate the final shape for the output statistics. reduced_shape = [] for i, a in enumerate(outputs_shape): if i not in reduced_axes: reduced_shape.append(a) mu = 1.0 sigma = 1.0 # Determine shape of Tensor after normalization. expected_mean = np.zeros(reduced_shape) expected_var = np.ones(reduced_shape) inputs = random_ops.random_normal(input_shape, seed=0) * sigma + mu output_op = normalization_ops.group_norm(inputs, groups=groups, center=False, scale=False, channels_axis=channels_axis, training=True) with self.cached_session() as sess: sess.run(variables.global_variables_initializer()) outputs = sess.run(output_op) # Make sure that there are no NaNs self.assertFalse(np.isnan(outputs).any()) # Implementation detail - in Poplibs group norm, the groups are not # contiguous, but strided - we replicate that here # Move the channels to the first dimension for inputs, gamma and beta outputs = np.swapaxes(outputs, 0, channels_axis) reshuffled_outputs = np.empty(outputs.shape, outputs.dtype) for from_idx in range(channels): to_idx = (from_idx % groups) * group_size + from_idx // groups reshuffled_outputs[to_idx] = outputs[from_idx] outputs = np.swapaxes(reshuffled_outputs, 0, channels_axis) outputs = np.reshape(outputs, outputs_shape) mean = np.mean(outputs, axis=reduced_axes, dtype=np.float32) var = np.var(outputs, axis=reduced_axes, dtype=np.float32) # The mean and variance of each example should be close to 0 and 1 # respectively. self.assertAllClose(expected_mean, mean, rtol=tol, atol=tol) self.assertAllClose(expected_var, var, rtol=tol, atol=tol)
def testInvalidGroupSize(self): inputs = array_ops.placeholder(dtypes.float32, shape=(5, 2, 10, 10)) with self.assertRaisesRegex(ValueError, 'Invalid groups 10 for 2 channels.'): normalization_ops.group_norm(inputs, groups=10, channels_axis=-3)
def testBadCommensurateGroup(self): inputs = array_ops.placeholder(dtypes.float32, shape=(5, 4, 10, 10)) with self.assertRaisesRegex( ValueError, '4 channels is not commensurate with ' '3 groups.'): normalization_ops.group_norm(inputs, groups=3, channels_axis=-3)
def testAxisIsBad(self): inputs = array_ops.placeholder(dtypes.float32, shape=(1, 2, 4, 5)) with self.assertRaisesRegex(ValueError, 'Axis is out of bounds.'): normalization_ops.group_norm(inputs, channels_axis=5)
def testUnknownShape(self): inputs = array_ops.placeholder(dtypes.float32) with self.assertRaisesRegex(ValueError, 'undefined rank'): normalization_ops.group_norm(inputs)