def test_apply_normalization():
input_shape = (1, 4)
reshaped_inputs = tf.constant([[[2.0, 2.0], [3.0, 3.0]]])
layer = GroupNormalization(groups=2, axis=1, scale=False, center=False)
normalized_input = layer._apply_normalization(reshaped_inputs, input_shape)
np.testing.assert_equal(normalized_input,
np.array([[[0.0, 0.0], [0.0, 0.0]]]))
def run_reshape_test(axis, group, input_shape, expected_shape):
group_layer = GroupNormalization(groups=group, axis=axis)
group_layer._set_number_of_groups_for_instance_norm(input_shape)
inputs = np.ones(input_shape)
tensor_input_shape = tf.convert_to_tensor(input_shape)
reshaped_inputs, group_shape = group_layer._reshape_into_groups(
inputs, (10, 10, 10), tensor_input_shape)
for i in range(len(expected_shape)):
assert group_shape[i] == expected_shape[i]
def test_apply_normalization(self):
input_shape = (1, 4)
expected_shape = (1, 2, 2)
reshaped_inputs = tf.constant([[[2.0, 2.0], [3.0, 3.0]]])
layer = GroupNormalization(groups=2, axis=1, scale=False, center=False)
normalized_input = layer._apply_normalization(reshaped_inputs,
input_shape)
self.assertTrue(
np.all(
np.equal(self.evaluate(normalized_input),
np.array([[[0.0, 0.0], [0.0, 0.0]]]))))
def _test_specific_layer(self, inputs, axis, groups, center, scale):
input_shape = inputs.shape
# Get Output from Keras model
layer = GroupNormalization(axis=axis,
groups=groups,
center=center,
scale=scale)
model = tf.keras.models.Sequential()
model.add(layer)
outputs = model.predict(inputs, steps=1)
self.assertFalse(np.isnan(outputs).any())
# Create shapes
if groups is -1:
groups = input_shape[axis]
np_inputs = self.evaluate(inputs)
reshaped_dims = list(np_inputs.shape)
reshaped_dims[axis] = reshaped_dims[axis] // groups
reshaped_dims.insert(axis, groups)
reshaped_inputs = np.reshape(np_inputs, tuple(reshaped_dims))
group_reduction_axes = list(range(1, len(reshaped_dims)))
axis = -2 if axis == -1 else axis - 1
group_reduction_axes.pop(axis)
# Calculate mean and variance
mean = np.mean(reshaped_inputs,
axis=tuple(group_reduction_axes),
keepdims=True)
variance = np.var(reshaped_inputs,
axis=tuple(group_reduction_axes),
keepdims=True)
# Get gamma and beta initalized by layer
gamma, beta = layer._get_reshaped_weights(input_shape)
if gamma is None:
gamma = 1.0
if beta is None:
beta = 0.0
# Get ouput from Numpy
zeroed = reshaped_inputs - mean
rsqrt = 1 / np.sqrt(variance + 1e-5)
output_test = gamma * zeroed * rsqrt + beta
# compare outputs
output_test = tf.reshape(output_test, input_shape)
self.assertAlmostEqual(self.evaluate(
tf.reduce_mean(output_test - outputs)),
0,
places=7)
def test_simple(self):
normalizers = [
GroupNormalization(groups=1, center=False, scale=False),
GroupNormalization(groups=-1, center=False, scale=False),
PoolNormalization2D(pool_size=(-1, 3))
]
layer = AdaptiveMultiNormalization(layers=normalizers)
x = tf.convert_to_tensor(normal(size=(1, 8, 8, 8)).astype(np.float16))
y = tf.convert_to_tensor(normal(size=(1, 2, 3, 4)).astype(np.float16))
res = layer([x, y])
self.assertShapeEqual(x.numpy(), res)
y = tf.convert_to_tensor(normal(size=(1, 4)).astype(np.float16))
res = layer([x, y])
self.assertShapeEqual(x.numpy(), res)
def test_weights(self):
# Check if weights get initialized correctly
layer = GroupNormalization(groups=1, scale=False, center=False)
layer.build((None, 3, 4))
self.assertEqual(len(layer.trainable_weights), 0)
self.assertEqual(len(layer.weights), 0)
layer = LayerNormalization()
layer.build((None, 3, 4))
self.assertEqual(len(layer.trainable_weights), 2)
self.assertEqual(len(layer.weights), 2)
layer = InstanceNormalization()
layer.build((None, 3, 4))
self.assertEqual(len(layer.trainable_weights), 2)
self.assertEqual(len(layer.weights), 2)
def test_regularizations(self):
layer = GroupNormalization(
gamma_regularizer='l1', beta_regularizer='l1', groups=4, axis=2)
layer.build((None, 4, 4))
self.assertEqual(len(layer.losses), 2)
max_norm = tf.keras.constraints.max_norm
layer = GroupNormalization(
gamma_constraint=max_norm, beta_constraint=max_norm)
layer.build((None, 3, 4))
self.assertEqual(layer.gamma.constraint, max_norm)
self.assertEqual(layer.beta.constraint, max_norm)
def test_groupnorm_flat():
# Check basic usage of groupnorm_flat
# Testing for 1 == LayerNorm, 16 == GroupNorm, -1 == InstanceNorm
groups = [-1, 16, 1]
shape = (64, )
for i in groups:
model = _create_and_fit_sequential_model(GroupNormalization(groups=i),
shape)
assert hasattr(model.layers[0], "gamma")
assert hasattr(model.layers[0], "beta")
def test_groupnorm_flat(self):
# Check basic usage of groupnorm_flat
# Testing for 1 == LayerNorm, 16 == GroupNorm, -1 == InstanceNorm
groups = [-1, 16, 1]
shape = (64, )
for i in groups:
model = self._create_and_fit_Sequential_model(
GroupNormalization(groups=i), shape)
self.assertTrue(hasattr(model.layers[0], 'gamma'))
self.assertTrue(hasattr(model.layers[0], 'beta'))
def test_regularizations():
layer = GroupNormalization(gamma_regularizer="l1",
beta_regularizer="l1",
groups=4,
axis=2)
layer.build((None, 4, 4))
assert len(layer.losses) == 2
max_norm = tf.keras.constraints.max_norm
layer = GroupNormalization(gamma_constraint=max_norm,
beta_constraint=max_norm)
layer.build((None, 3, 4))
assert layer.gamma.constraint == max_norm
assert layer.beta.constraint == max_norm
def test_weights():
# Check if weights get initialized correctly
layer = GroupNormalization(groups=1, scale=False, center=False)
layer.build((None, 3, 4))
assert len(layer.trainable_weights) == 0
assert len(layer.weights) == 0
layer = InstanceNormalization()
layer.build((None, 3, 4))
assert len(layer.trainable_weights) == 2
assert len(layer.weights) == 2
def test_initializer(self):
# Check if the initializer for gamma and beta is working correctly
layer = GroupNormalization(groups=32,
beta_initializer='random_normal',
beta_constraint='NonNeg',
gamma_initializer='random_normal',
gamma_constraint='NonNeg')
model = self._create_and_fit_Sequential_model(layer, (64, ))
weights = np.array(model.layers[0].get_weights())
negativ = weights[weights < 0.0]
self.assertTrue(len(negativ) == 0)
def test_group_norm_compute_output_shape(center, scale):
target_variables_len = [center, scale].count(True)
target_trainable_variables_len = [center, scale].count(True)
layer1 = GroupNormalization(groups=2, center=center, scale=scale)
layer1.build(input_shape=[8, 28, 28, 16]) # build()
assert len(layer1.variables) == target_variables_len
assert len(layer1.trainable_variables) == target_trainable_variables_len
layer2 = GroupNormalization(groups=2, center=center, scale=scale)
layer2.compute_output_shape(input_shape=[8, 28, 28,
16]) # compute_output_shape()
assert len(layer2.variables) == target_variables_len
assert len(layer2.trainable_variables) == target_trainable_variables_len
layer3 = GroupNormalization(groups=2, center=center, scale=scale)
layer3(tf.random.normal(shape=[8, 28, 28, 16])) # call()
assert len(layer3.variables) == target_variables_len
assert len(layer3.trainable_variables) == target_trainable_variables_len
def test_groupnorm_correctness_1d():
np.random.seed(0x2020)
model = tf.keras.models.Sequential()
norm = GroupNormalization(input_shape=(10, ), groups=2)
model.add(norm)
model.compile(loss="mse", optimizer="rmsprop")
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
model.fit(x, x, epochs=5, verbose=0)
out = model.predict(x)
out -= norm.beta.numpy()
out /= norm.gamma.numpy()
np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1)
np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
def test_groupnorm_conv(self):
# Check if Axis is working for CONV nets
# Testing for 1 == LayerNorm, 5 == GroupNorm, -1 == InstanceNorm
groups = [-1, 5, 1]
for i in groups:
model = tf.keras.models.Sequential()
model.add(GroupNormalization(axis=1, groups=i, input_shape=(20, 20, 3)))
model.add(tf.keras.layers.Conv2D(5, (1, 1), padding="same"))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(1, activation="softmax"))
model.compile(optimizer=tf.keras.optimizers.RMSprop(0.01), loss="mse")
x = np.random.randint(1000, size=(10, 20, 20, 3))
y = np.random.randint(1000, size=(10, 1))
model.fit(x=x, y=y, epochs=1)
self.assertTrue(hasattr(model.layers[0], "gamma"))
def test_initializer():
# Check if the initializer for gamma and beta is working correctly
layer = GroupNormalization(
groups=32,
beta_initializer="random_normal",
beta_constraint="NonNeg",
gamma_initializer="random_normal",
gamma_constraint="NonNeg",
)
model = _create_and_fit_sequential_model(layer, (64, ))
weights = np.array(model.layers[0].get_weights())
negativ = weights[weights < 0.0]
assert len(negativ) == 0
def test_groupnorm_convnet():
np.random.seed(0x2020)
model = tf.keras.models.Sequential()
norm = GroupNormalization(axis=1, input_shape=(3, 4, 4), groups=3)
model.add(norm)
model.compile(loss="mse", optimizer="sgd")
# centered = 5.0, variance = 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
model.fit(x, x, epochs=4, verbose=0)
out = model.predict(x)
out -= np.reshape(norm.beta.numpy(), (1, 3, 1, 1))
out /= np.reshape(norm.gamma.numpy(), (1, 3, 1, 1))
np.testing.assert_allclose(np.mean(out, axis=(0, 2, 3), dtype=np.float32),
(0.0, 0.0, 0.0),
atol=1e-1)
np.testing.assert_allclose(np.std(out, axis=(0, 2, 3), dtype=np.float32),
(1.0, 1.0, 1.0),
atol=1e-1)
def test_groupnorm_convnet_no_center_no_scale():
np.random.seed(0x2020)
model = tf.keras.models.Sequential()
norm = GroupNormalization(axis=-1,
groups=2,
center=False,
scale=False,
input_shape=(3, 4, 4))
model.add(norm)
model.compile(loss="mse", optimizer="sgd")
# centered and variance are 5.0 and 10.0, respectively
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
model.fit(x, x, epochs=4, verbose=0)
out = model.predict(x)
np.testing.assert_allclose(np.mean(out, axis=(0, 2, 3), dtype=np.float32),
(0.0, 0.0, 0.0),
atol=1e-1)
np.testing.assert_allclose(np.std(out, axis=(0, 2, 3), dtype=np.float32),
(1.0, 1.0, 1.0),
atol=1e-1)
def test_groupnorm_2d_different_groups():
np.random.seed(0x2020)
groups = [2, 1, 10]
for i in groups:
model = tf.keras.models.Sequential()
norm = GroupNormalization(axis=1, groups=i, input_shape=(10, 3))
model.add(norm)
# centered and variance are 5.0 and 10.0, respectively
model.compile(loss="mse", optimizer="rmsprop")
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 3))
model.fit(x, x, epochs=5, verbose=0)
out = model.predict(x)
out -= np.reshape(norm.beta.numpy(), (1, 10, 1))
out /= np.reshape(norm.gamma.numpy(), (1, 10, 1))
np.testing.assert_allclose(out.mean(axis=(0, 1), dtype=np.float32),
(0.0, 0.0, 0.0),
atol=1e-1)
np.testing.assert_allclose(out.std(axis=(0, 1), dtype=np.float32),
(1.0, 1.0, 1.0),
atol=1e-1)
def test_axis_error(self):
with self.assertRaises(ValueError):
GroupNormalization(axis=0)
def test_axis_error():
with pytest.raises(ValueError):
GroupNormalization(axis=0)
