def run_reshape_test(axis, group, input_shape, expected_shape):
group_layer = GroupNormalization(groups=group, axis=axis)
group_layer._set_number_of_groups_for_instance_norm(input_shape)
inputs = np.ones(input_shape)
tensor_input_shape = tf.convert_to_tensor(input_shape)
reshaped_inputs, group_shape = group_layer._reshape_into_groups(
inputs, (10, 10, 10), tensor_input_shape)
for i in range(len(expected_shape)):
self.assertEqual(int(group_shape[i]), expected_shape[i])
def test_apply_normalization(self):
input_shape = (1, 4)
expected_shape = (1, 2, 2)
reshaped_inputs = tf.constant([[[2.0, 2.0], [3.0, 3.0]]])
layer = GroupNormalization(groups=2, axis=1, scale=False, center=False)
normalized_input = layer._apply_normalization(reshaped_inputs,
input_shape)
self.assertTrue(
tf.reduce_all(
tf.equal(normalized_input,
tf.constant([[[0.0, 0.0], [0.0, 0.0]]]))))
def test_weights(self):
# Check if weights get initialized correctly
layer = GroupNormalization(groups=1, scale=False, center=False)
layer.build((None, 3, 4))
self.assertEqual(len(layer.trainable_weights), 0)
self.assertEqual(len(layer.weights), 0)
layer = LayerNormalization()
layer.build((None, 3, 4))
self.assertEqual(len(layer.trainable_weights), 2)
self.assertEqual(len(layer.weights), 2)
layer = InstanceNormalization()
layer.build((None, 3, 4))
self.assertEqual(len(layer.trainable_weights), 2)
self.assertEqual(len(layer.weights), 2)
def _test_specific_layer(self, inputs, axis, groups, center, scale):
input_shape = inputs.shape
# Get Output from Keras model
layer = GroupNormalization(axis=axis,
groups=groups,
center=center,
scale=scale)
model = tf.keras.models.Sequential()
model.add(layer)
outputs = model.predict(inputs)
self.assertFalse(np.isnan(outputs).any())
# Create shapes
if groups is -1:
groups = input_shape[axis]
np_inputs = inputs.numpy()
reshaped_dims = list(np_inputs.shape)
reshaped_dims[axis] = reshaped_dims[axis] // groups
reshaped_dims.insert(1, groups)
reshaped_inputs = np.reshape(np_inputs, tuple(reshaped_dims))
# Calculate mean and variance
mean = np.mean(reshaped_inputs,
axis=tuple(range(2, len(reshaped_dims))),
keepdims=True)
variance = np.var(reshaped_inputs,
axis=tuple(range(2, len(reshaped_dims))),
keepdims=True)
# Get gamma and beta initalized by layer
gamma, beta = layer._get_reshaped_weights(input_shape)
if gamma is None:
gamma = 1.0
if beta is None:
beta = 0.0
# Get ouput from Numpy
zeroed = reshaped_inputs - mean
rsqrt = 1 / np.sqrt(variance + 1e-5)
output_test = gamma * zeroed * rsqrt + beta
# compare outputs
output_test = np.reshape(output_test, input_shape.as_list())
self.assertAlmostEqual(np.mean(output_test - outputs), 0, places=7)
def test_groupnorm_flat(self):
# Check basic usage of groupnorm_flat
# Testing for 1 == LayerNorm, 16 == GroupNorm, -1 == InstanceNorm
groups = [-1, 16, 1]
shape = (64, )
for i in groups:
model = self._create_and_fit_Sequential_model(
GroupNormalization(groups=i), shape)
self.assertTrue(hasattr(model.layers[0], 'gamma'))
self.assertTrue(hasattr(model.layers[0], 'beta'))
def test_regularizations(self):
layer = GroupNormalization(gamma_regularizer='l1',
beta_regularizer='l1',
groups=4,
axis=2)
layer.build((None, 4, 4))
self.assertEqual(len(layer.losses), 2)
max_norm = tf.keras.constraints.max_norm
layer = GroupNormalization(gamma_constraint=max_norm,
beta_constraint=max_norm)
layer.build((None, 3, 4))
self.assertEqual(layer.gamma.constraint, max_norm)
self.assertEqual(layer.beta.constraint, max_norm)
def test_initializer(self):
# Check if the initializer for gamma and beta is working correctly
layer = GroupNormalization(groups=32,
beta_initializer='random_normal',
beta_constraint='NonNeg',
gamma_initializer='random_normal',
gamma_constraint='NonNeg')
model = self._create_and_fit_Sequential_model(layer, (64, ))
weights = np.array(model.layers[0].get_weights())
negativ = weights[weights < 0.0]
self.assertTrue(len(negativ) == 0)
def test_groupnorm_conv(self):
# Check if Axis is working for CONV nets
# Testing for 1 == LayerNorm, 5 == GroupNorm, -1 == InstanceNorm
groups = [-1, 5, 1]
for i in groups:
model = tf.keras.models.Sequential()
model.add(
GroupNormalization(axis=1, groups=i, input_shape=(20, 20, 3)))
model.add(tf.keras.layers.Conv2D(5, (1, 1), padding='same'))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(1, activation='softmax'))
model.compile(optimizer=tf.keras.optimizers.RMSprop(0.01),
loss='mse')
x = np.random.randint(1000, size=(10, 20, 20, 3))
y = np.random.randint(1000, size=(10, 1))
a = model.fit(x=x, y=y, epochs=1)
self.assertTrue(hasattr(model.layers[0], 'gamma'))
def test_axis_error(self):
with self.assertRaises(ValueError):
GroupNormalization(axis=0)