def test_apply_normalization(): input_shape = (1, 4) reshaped_inputs = tf.constant([[[2.0, 2.0], [3.0, 3.0]]]) layer = GroupNormalization(groups=2, axis=1, scale=False, center=False) normalized_input = layer._apply_normalization(reshaped_inputs, input_shape) np.testing.assert_equal(normalized_input, np.array([[[0.0, 0.0], [0.0, 0.0]]]))
def run_reshape_test(axis, group, input_shape, expected_shape): group_layer = GroupNormalization(groups=group, axis=axis) group_layer._set_number_of_groups_for_instance_norm(input_shape) inputs = np.ones(input_shape) tensor_input_shape = tf.convert_to_tensor(input_shape) reshaped_inputs, group_shape = group_layer._reshape_into_groups( inputs, (10, 10, 10), tensor_input_shape) for i in range(len(expected_shape)): assert group_shape[i] == expected_shape[i]
def test_apply_normalization(self): input_shape = (1, 4) expected_shape = (1, 2, 2) reshaped_inputs = tf.constant([[[2.0, 2.0], [3.0, 3.0]]]) layer = GroupNormalization(groups=2, axis=1, scale=False, center=False) normalized_input = layer._apply_normalization(reshaped_inputs, input_shape) self.assertTrue( np.all( np.equal(self.evaluate(normalized_input), np.array([[[0.0, 0.0], [0.0, 0.0]]]))))
def _test_specific_layer(self, inputs, axis, groups, center, scale): input_shape = inputs.shape # Get Output from Keras model layer = GroupNormalization(axis=axis, groups=groups, center=center, scale=scale) model = tf.keras.models.Sequential() model.add(layer) outputs = model.predict(inputs, steps=1) self.assertFalse(np.isnan(outputs).any()) # Create shapes if groups is -1: groups = input_shape[axis] np_inputs = self.evaluate(inputs) reshaped_dims = list(np_inputs.shape) reshaped_dims[axis] = reshaped_dims[axis] // groups reshaped_dims.insert(axis, groups) reshaped_inputs = np.reshape(np_inputs, tuple(reshaped_dims)) group_reduction_axes = list(range(1, len(reshaped_dims))) axis = -2 if axis == -1 else axis - 1 group_reduction_axes.pop(axis) # Calculate mean and variance mean = np.mean(reshaped_inputs, axis=tuple(group_reduction_axes), keepdims=True) variance = np.var(reshaped_inputs, axis=tuple(group_reduction_axes), keepdims=True) # Get gamma and beta initalized by layer gamma, beta = layer._get_reshaped_weights(input_shape) if gamma is None: gamma = 1.0 if beta is None: beta = 0.0 # Get ouput from Numpy zeroed = reshaped_inputs - mean rsqrt = 1 / np.sqrt(variance + 1e-5) output_test = gamma * zeroed * rsqrt + beta # compare outputs output_test = tf.reshape(output_test, input_shape) self.assertAlmostEqual(self.evaluate( tf.reduce_mean(output_test - outputs)), 0, places=7)
def test_simple(self): normalizers = [ GroupNormalization(groups=1, center=False, scale=False), GroupNormalization(groups=-1, center=False, scale=False), PoolNormalization2D(pool_size=(-1, 3)) ] layer = AdaptiveMultiNormalization(layers=normalizers) x = tf.convert_to_tensor(normal(size=(1, 8, 8, 8)).astype(np.float16)) y = tf.convert_to_tensor(normal(size=(1, 2, 3, 4)).astype(np.float16)) res = layer([x, y]) self.assertShapeEqual(x.numpy(), res) y = tf.convert_to_tensor(normal(size=(1, 4)).astype(np.float16)) res = layer([x, y]) self.assertShapeEqual(x.numpy(), res)
def test_weights(self): # Check if weights get initialized correctly layer = GroupNormalization(groups=1, scale=False, center=False) layer.build((None, 3, 4)) self.assertEqual(len(layer.trainable_weights), 0) self.assertEqual(len(layer.weights), 0) layer = LayerNormalization() layer.build((None, 3, 4)) self.assertEqual(len(layer.trainable_weights), 2) self.assertEqual(len(layer.weights), 2) layer = InstanceNormalization() layer.build((None, 3, 4)) self.assertEqual(len(layer.trainable_weights), 2) self.assertEqual(len(layer.weights), 2)
def test_regularizations(self): layer = GroupNormalization( gamma_regularizer='l1', beta_regularizer='l1', groups=4, axis=2) layer.build((None, 4, 4)) self.assertEqual(len(layer.losses), 2) max_norm = tf.keras.constraints.max_norm layer = GroupNormalization( gamma_constraint=max_norm, beta_constraint=max_norm) layer.build((None, 3, 4)) self.assertEqual(layer.gamma.constraint, max_norm) self.assertEqual(layer.beta.constraint, max_norm)
def test_groupnorm_flat(): # Check basic usage of groupnorm_flat # Testing for 1 == LayerNorm, 16 == GroupNorm, -1 == InstanceNorm groups = [-1, 16, 1] shape = (64, ) for i in groups: model = _create_and_fit_sequential_model(GroupNormalization(groups=i), shape) assert hasattr(model.layers[0], "gamma") assert hasattr(model.layers[0], "beta")
def test_groupnorm_flat(self): # Check basic usage of groupnorm_flat # Testing for 1 == LayerNorm, 16 == GroupNorm, -1 == InstanceNorm groups = [-1, 16, 1] shape = (64, ) for i in groups: model = self._create_and_fit_Sequential_model( GroupNormalization(groups=i), shape) self.assertTrue(hasattr(model.layers[0], 'gamma')) self.assertTrue(hasattr(model.layers[0], 'beta'))
def test_regularizations(): layer = GroupNormalization(gamma_regularizer="l1", beta_regularizer="l1", groups=4, axis=2) layer.build((None, 4, 4)) assert len(layer.losses) == 2 max_norm = tf.keras.constraints.max_norm layer = GroupNormalization(gamma_constraint=max_norm, beta_constraint=max_norm) layer.build((None, 3, 4)) assert layer.gamma.constraint == max_norm assert layer.beta.constraint == max_norm
def test_weights(): # Check if weights get initialized correctly layer = GroupNormalization(groups=1, scale=False, center=False) layer.build((None, 3, 4)) assert len(layer.trainable_weights) == 0 assert len(layer.weights) == 0 layer = InstanceNormalization() layer.build((None, 3, 4)) assert len(layer.trainable_weights) == 2 assert len(layer.weights) == 2
def test_initializer(self): # Check if the initializer for gamma and beta is working correctly layer = GroupNormalization(groups=32, beta_initializer='random_normal', beta_constraint='NonNeg', gamma_initializer='random_normal', gamma_constraint='NonNeg') model = self._create_and_fit_Sequential_model(layer, (64, )) weights = np.array(model.layers[0].get_weights()) negativ = weights[weights < 0.0] self.assertTrue(len(negativ) == 0)
def test_group_norm_compute_output_shape(center, scale): target_variables_len = [center, scale].count(True) target_trainable_variables_len = [center, scale].count(True) layer1 = GroupNormalization(groups=2, center=center, scale=scale) layer1.build(input_shape=[8, 28, 28, 16]) # build() assert len(layer1.variables) == target_variables_len assert len(layer1.trainable_variables) == target_trainable_variables_len layer2 = GroupNormalization(groups=2, center=center, scale=scale) layer2.compute_output_shape(input_shape=[8, 28, 28, 16]) # compute_output_shape() assert len(layer2.variables) == target_variables_len assert len(layer2.trainable_variables) == target_trainable_variables_len layer3 = GroupNormalization(groups=2, center=center, scale=scale) layer3(tf.random.normal(shape=[8, 28, 28, 16])) # call() assert len(layer3.variables) == target_variables_len assert len(layer3.trainable_variables) == target_trainable_variables_len
def test_groupnorm_correctness_1d(): np.random.seed(0x2020) model = tf.keras.models.Sequential() norm = GroupNormalization(input_shape=(10, ), groups=2) model.add(norm) model.compile(loss="mse", optimizer="rmsprop") x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10)) model.fit(x, x, epochs=5, verbose=0) out = model.predict(x) out -= norm.beta.numpy() out /= norm.gamma.numpy() np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1) np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
def test_groupnorm_conv(self): # Check if Axis is working for CONV nets # Testing for 1 == LayerNorm, 5 == GroupNorm, -1 == InstanceNorm groups = [-1, 5, 1] for i in groups: model = tf.keras.models.Sequential() model.add(GroupNormalization(axis=1, groups=i, input_shape=(20, 20, 3))) model.add(tf.keras.layers.Conv2D(5, (1, 1), padding="same")) model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Dense(1, activation="softmax")) model.compile(optimizer=tf.keras.optimizers.RMSprop(0.01), loss="mse") x = np.random.randint(1000, size=(10, 20, 20, 3)) y = np.random.randint(1000, size=(10, 1)) model.fit(x=x, y=y, epochs=1) self.assertTrue(hasattr(model.layers[0], "gamma"))
def test_initializer(): # Check if the initializer for gamma and beta is working correctly layer = GroupNormalization( groups=32, beta_initializer="random_normal", beta_constraint="NonNeg", gamma_initializer="random_normal", gamma_constraint="NonNeg", ) model = _create_and_fit_sequential_model(layer, (64, )) weights = np.array(model.layers[0].get_weights()) negativ = weights[weights < 0.0] assert len(negativ) == 0
def test_groupnorm_convnet(): np.random.seed(0x2020) model = tf.keras.models.Sequential() norm = GroupNormalization(axis=1, input_shape=(3, 4, 4), groups=3) model.add(norm) model.compile(loss="mse", optimizer="sgd") # centered = 5.0, variance = 10.0 x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4)) model.fit(x, x, epochs=4, verbose=0) out = model.predict(x) out -= np.reshape(norm.beta.numpy(), (1, 3, 1, 1)) out /= np.reshape(norm.gamma.numpy(), (1, 3, 1, 1)) np.testing.assert_allclose(np.mean(out, axis=(0, 2, 3), dtype=np.float32), (0.0, 0.0, 0.0), atol=1e-1) np.testing.assert_allclose(np.std(out, axis=(0, 2, 3), dtype=np.float32), (1.0, 1.0, 1.0), atol=1e-1)
def test_groupnorm_convnet_no_center_no_scale(): np.random.seed(0x2020) model = tf.keras.models.Sequential() norm = GroupNormalization(axis=-1, groups=2, center=False, scale=False, input_shape=(3, 4, 4)) model.add(norm) model.compile(loss="mse", optimizer="sgd") # centered and variance are 5.0 and 10.0, respectively x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4)) model.fit(x, x, epochs=4, verbose=0) out = model.predict(x) np.testing.assert_allclose(np.mean(out, axis=(0, 2, 3), dtype=np.float32), (0.0, 0.0, 0.0), atol=1e-1) np.testing.assert_allclose(np.std(out, axis=(0, 2, 3), dtype=np.float32), (1.0, 1.0, 1.0), atol=1e-1)
def test_groupnorm_2d_different_groups(): np.random.seed(0x2020) groups = [2, 1, 10] for i in groups: model = tf.keras.models.Sequential() norm = GroupNormalization(axis=1, groups=i, input_shape=(10, 3)) model.add(norm) # centered and variance are 5.0 and 10.0, respectively model.compile(loss="mse", optimizer="rmsprop") x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 3)) model.fit(x, x, epochs=5, verbose=0) out = model.predict(x) out -= np.reshape(norm.beta.numpy(), (1, 10, 1)) out /= np.reshape(norm.gamma.numpy(), (1, 10, 1)) np.testing.assert_allclose(out.mean(axis=(0, 1), dtype=np.float32), (0.0, 0.0, 0.0), atol=1e-1) np.testing.assert_allclose(out.std(axis=(0, 1), dtype=np.float32), (1.0, 1.0, 1.0), atol=1e-1)
def test_axis_error(self): with self.assertRaises(ValueError): GroupNormalization(axis=0)
def test_axis_error(): with pytest.raises(ValueError): GroupNormalization(axis=0)