def __init__(self):
super(HasMapping, self).__init__()
self.layer_dict = data_structures.Mapping(output=core.Dense(7))
self.layer_dict["norm"] = data_structures.List()
self.layer_dict["dense"] = data_structures.List()
self.layer_dict["dense"].extend([
core.Dense(5),
core.Dense(6, kernel_regularizer=math_ops.reduce_sum)
])
self.layer_dict["norm"].append(normalization.BatchNormalization())
self.layer_dict["norm"].append(normalization.BatchNormalization())
def my_func():
layer = normalization.BatchNormalization()
x = array_ops.ones((10, 1))
y = layer(x, training=True)
# Updates should be tracked in a `wrap_function`.
self.assertLen(layer.updates, 2)
return y
def test_v1_fused_attribute(self):
norm = normalization.BatchNormalization()
inp = keras.layers.Input((4, 4, 4))
norm(inp)
self.assertEqual(norm.fused, True)
norm = normalization.BatchNormalization(fused=False)
self.assertEqual(norm.fused, False)
inp = keras.layers.Input(shape=(4, 4, 4))
norm(inp)
self.assertEqual(norm.fused, False)
norm = normalization.BatchNormalization(virtual_batch_size=2)
self.assertEqual(norm.fused, True)
inp = keras.layers.Input(shape=(2, 2, 2))
norm(inp)
self.assertEqual(norm.fused, False)
def benchmark_layers_batch_norm_nonfused_train(self):
layer = normalization.BatchNormalization(fused=False)
x = array_ops.ones((1, 1, 1, 1))
def fn():
layer(x, training=True)
self._run(fn, 10000)
def benchmark_layers_normalization_batch_normalization_overhead(self):
layer = normalization.BatchNormalization()
x = array_ops.ones((1, 1))
def fn():
layer(x, training=True)
self._run(fn, 10000)
def benchmark_layers_batch_norm_fused_inf(self):
layer = normalization.BatchNormalization(fused=True)
x = array_ops.ones((1, 1, 1, 1))
def fn():
layer(x)
self._run(fn, 10000)
def DenseBlock(self, inputs, outdim):
inputshape = K.int_shape(inputs)
bn = normalization.BatchNormalization(epsilon=2e-05, axis=3, momentum=0.9, weights=None, beta_initializer='zero', gamma_initializer='one')(inputs)
act = Activation('relu')(bn)
# act = Dropout(rate=0.2)(act)
conv1 = Conv2D(outdim, (3, 3), activation=None, padding='same', kernel_regularizer=regularizers.l2(0.0001))(act)
if inputshape[3] != outdim:
shortcut = Conv2D(outdim, (1, 1), padding='same', kernel_regularizer=regularizers.l2(0.0001))(inputs)
else:
shortcut = inputs
result1 = add([conv1, shortcut])
bn = normalization.BatchNormalization(epsilon=2e-05, axis=3, momentum=0.9, weights=None, beta_initializer='zero', gamma_initializer='one')(result1)
act = Activation('relu')(bn)
# act = Dropout(rate=0.2)(act)
conv2 = Conv2D(outdim, (3, 3), activation=None, padding='same', kernel_regularizer=regularizers.l2(0.0001))(act)
result = add([result1, conv2, shortcut])
result = Activation('relu')(result)
return result
def build_model(self):
inputs = Input((self.patch_height, self.patch_width, 1))
conv1 = Conv2D(32, (1, 1), activation=None, padding='same')(inputs)
conv1 = normalization.BatchNormalization(epsilon=2e-05, axis=3, momentum=0.9, weights=None, beta_initializer='zero', gamma_initializer='one')(conv1)
conv1 = Activation('relu')(conv1)
conv1 = self.DenseBlock(conv1, 32) # 48
conv1 = self.se_block(ratio=2)(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = self.DenseBlock(pool1, 64) # 24
conv2 = self.se_block(ratio=2)(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = self.DenseBlock(pool2, 64) # 12
conv3 = self.se_block(ratio=2)(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = self.DenseBlock(pool3, 64) # 12
conv4 = self.se_block(ratio=2)(conv4)
up1 = Conv2DTranspose(64, (3, 3), strides=2, activation='relu', padding='same', kernel_regularizer=regularizers.l2(0.0001))(conv4)
up1 = concatenate([up1, conv3], axis=3)
conv5 = self.DenseBlock(up1, 64)
conv5 = self.se_block(ratio=2)(conv5)
up2 = Conv2DTranspose(64, (3, 3), strides=2, activation='relu', padding='same', kernel_regularizer=regularizers.l2(0.0001))(conv5)
up2 = concatenate([up2, conv2], axis=3)
conv6 = self.DenseBlock(up2, 64)
conv6 = self.se_block(ratio=2)(conv6)
up3 = Conv2DTranspose(32, (3, 3), strides=2, activation='relu', padding='same', kernel_regularizer=regularizers.l2(0.0001))(conv6)
up3 = concatenate([up3, conv1], axis=3)
conv7 = self.DenseBlock(up3, 32)
conv7 = self.se_block(ratio=2)(conv7)
conv8 = Conv2D(self.num_seg_class + 1, (1, 1), activation='relu', padding='same', kernel_regularizer=regularizers.l2(0.0001))(conv7)
# conv6 = normalization.BatchNormalization(epsilon=1e-06, mode=1, axis=-1, momentum=0.9, weights=None, beta_init='zero', gamma_init='one')(conv6)
# for tensorflow
conv8 = core.Reshape((self.patch_height * self.patch_width, self.num_seg_class + 1))(conv8)
# for theano
# conv8 = core.Reshape(((self.num_seg_class + 1), self.patch_height * self.patch_width))(conv8)
# conv8 = core.Permute((2, 1))(conv8)
############
act = Activation('softmax')(conv8)
model = Model(inputs=inputs, outputs=act)
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['categorical_accuracy'])
plot_model(model, to_file=os.path.join(self.config.checkpoint, "model.png"), show_shapes=True)
self.model = model
def test_updates_in_wrap_function(self):
layer = normalization.BatchNormalization()
def my_func():
x = array_ops.ones((10, 1))
return layer(x, training=True)
wrapped_fn = wrap_function.wrap_function(my_func, [])
wrapped_fn()
# Updates should be tracked in a `wrap_function`.
self.assertLen(layer.updates, 2)
def __init__(self):
super(HasList, self).__init__()
self.layer_list = data_structures.List([core.Dense(3)])
self.layer_list.append(core.Dense(4))
self.layer_list.extend([
core.Dense(5),
core.Dense(6, kernel_regularizer=math_ops.reduce_sum)
])
self.layer_list += [
core.Dense(7, bias_regularizer=math_ops.reduce_sum),
core.Dense(8)
]
self.layer_list += (data_structures.List([core.Dense(9)]) +
data_structures.List([core.Dense(10)]))
self.layer_list.extend(
data_structures.List(list([core.Dense(11)]) + [core.Dense(12)]))
self.layers_with_updates = data_structures.List(
(normalization.BatchNormalization(), ))
def f(inputs):
conv = Conv2D(filters, strides, padding=padding)(inputs)
conv = normalization.BatchNormalization(
epsilon=2e-05,
axis=3,
momentum=0.9,
weights=None,
beta_initializer='RandomNormal',
gamma_initializer='one')(conv)
conv = LeakyReLU(alpha=0.3)(conv)
conv = Dropout(0.2)(conv)
conv = Conv2D(filters, strides, dilation_rate=2,
padding=padding)(conv)
conv = LeakyReLU(alpha=0.3)(conv)
conv = Conv2D(filters, strides, dilation_rate=4,
padding=padding)(conv)
conv = LeakyReLU(alpha=0.3)(conv)
return conv
def __init__(self):
super(HasTuple, self).__init__()
self.layer_list = (core.Dense(3), core.Dense(4),
core.Dense(5,
kernel_regularizer=math_ops.reduce_sum))
self.layers_with_updates = (normalization.BatchNormalization(), )
def __init__(
self,
# DepthwiseConv2D params
kernel_size,
strides=(1, 1),
padding='valid',
depth_multiplier=1,
data_format=None,
depthwise_initializer='glorot_uniform',
depthwise_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
depthwise_constraint=None,
bias_constraint=None,
name=None,
# BatchNormalization params
axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
renorm=False,
renorm_clipping=None,
renorm_momentum=0.99,
fused=None,
trainable=True,
virtual_batch_size=None,
adjustment=None,
# Post-batchnorm activation instance.
post_activation=None,
# quantization params
is_quantized=True,
**kwargs):
super(_DepthwiseConvBatchNorm2D, self).__init__(
kernel_size,
strides=strides,
padding=padding,
depth_multiplier=depth_multiplier,
data_format=data_format,
use_bias=False,
depthwise_initializer=depthwise_initializer,
depthwise_regularizer=depthwise_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
depthwise_constraint=depthwise_constraint,
bias_constraint=bias_constraint,
name=name,
**kwargs)
self.batchnorm = normalization.BatchNormalization(
axis=axis,
momentum=momentum,
epsilon=epsilon,
center=center,
scale=scale,
beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer,
moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer,
beta_regularizer=beta_regularizer,
gamma_regularizer=gamma_regularizer,
beta_constraint=beta_constraint,
gamma_constraint=gamma_constraint,
renorm=renorm,
renorm_clipping=renorm_clipping,
renorm_momentum=renorm_momentum,
fused=fused,
trainable=trainable,
virtual_batch_size=virtual_batch_size,
adjustment=adjustment,
)
self.post_activation = activations.get(post_activation)
self.is_quantized = is_quantized
if self.is_quantized:
self.weight_quantizer = default_8bit_quantizers.Default8BitConvWeightsQuantizer(
)
self.activation_quantizer = quantizers.MovingAverageQuantizer(
num_bits=8, per_axis=False, symmetric=False, narrow_range=False)
def __init__(
self,
# Conv2D params
filters,
kernel_size,
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
name=None,
# BatchNormalization params
axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
renorm=False,
renorm_clipping=None,
renorm_momentum=0.99,
fused=None,
trainable=True,
virtual_batch_size=None,
adjustment=None,
# Post-batchnorm activation.
post_activation=None,
# quantization params
is_quantized=True,
**kwargs):
super(_ConvBatchNorm2D, self).__init__(
filters,
kernel_size,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
use_bias=False,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
kernel_constraint=kernel_constraint,
bias_constraint=bias_constraint,
name=name,
**kwargs)
self.batchnorm = normalization.BatchNormalization(
axis=axis,
momentum=momentum,
epsilon=epsilon,
center=center,
scale=scale,
beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer,
moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer,
beta_regularizer=beta_regularizer,
gamma_regularizer=gamma_regularizer,
beta_constraint=beta_constraint,
gamma_constraint=gamma_constraint,
renorm=renorm,
renorm_clipping=renorm_clipping,
renorm_momentum=renorm_momentum,
fused=fused,
trainable=trainable,
virtual_batch_size=virtual_batch_size,
adjustment=adjustment,
)
# Named as post_activation to not conflict with Layer self.activation.
self.post_activation = activations.get(post_activation)
self.is_quantized = is_quantized
if self.is_quantized:
self.weight_quantizer = tflite_quantizers.ConvWeightsQuantizer()
self.activation_quantizer = quantizers.MovingAverageQuantizer(
num_bits=8, per_axis=False, symmetric=False, narrow_range=False)
