def build_conv2_supermask(args):
kwargs = {}
if args.signed_constant:
kwargs['signed_constant'] = True
kwargs['const_multiplier'] = args.signed_constant_multiplier
if args.dynamic_scaling:
kwargs['dynamic_scaling'] = True
return SequentialNetwork([
MaskedConv2D(64,
3,
kernel_initializer=glorot_normal,
sigmoid_bias=args.sigmoid_bias,
round_mask=args.round_mask,
padding='same',
kernel_regularizer=l2reg(args.l2),
name='conv2D_1',
**kwargs),
Activation('relu'),
MaskedConv2D(64,
3,
kernel_initializer=glorot_normal,
sigmoid_bias=args.sigmoid_bias,
round_mask=args.round_mask,
padding='same',
kernel_regularizer=l2reg(args.l2),
name='conv2D_2',
**kwargs),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
Flatten(),
MaskedDense(256,
kernel_initializer=glorot_normal,
sigmoid_bias=args.sigmoid_bias,
round_mask=args.round_mask,
activation=relu,
kernel_regularizer=l2reg(args.l2),
name='fc_1',
**kwargs),
MaskedDense(256,
kernel_initializer=glorot_normal,
sigmoid_bias=args.sigmoid_bias,
round_mask=args.round_mask,
activation=relu,
kernel_regularizer=l2reg(args.l2),
name='fc_2',
**kwargs),
MaskedDense(10,
kernel_initializer=glorot_normal,
sigmoid_bias=args.sigmoid_bias,
round_mask=args.round_mask,
activation=None,
kernel_regularizer=l2reg(args.l2),
name='fc_3',
**kwargs)
])
def build_network_fc_special(args):
return SequentialNetwork([
Flatten(),
Dense(100, kernel_initializer=he_normal, activation=None, kernel_regularizer=l2reg(args.l2), name='fc_1'),
BatchNormalization(momentum=0, name='batch_norm_1'),
Activation('relu'),
Dense(50, kernel_initializer=he_normal, activation=None, kernel_regularizer=l2reg(args.l2), name='fc_2'),
BatchNormalization(momentum=0, name='batch_norm_1'),
Activation('relu'),
Dense(5, kernel_initializer=he_normal, activation=None, kernel_regularizer=l2reg(args.l2), name='fc_3')
])
def build_frozen_conv2_lottery(args, init_values, mask_values):
return SequentialNetwork([
FreezeConv2D(64, 3, init_values[0], init_values[1], mask_values[0], mask_values[1], kernel_initializer=glorot_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_1'),
Activation('relu'),
FreezeConv2D(64, 3, init_values[2], init_values[3], mask_values[2], mask_values[3], kernel_initializer=glorot_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_2'),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
Flatten(),
FreezeDense(256, init_values[4], init_values[5], mask_values[4], mask_values[5], kernel_initializer=glorot_normal, activation=relu, kernel_regularizer=l2reg(args.l2), name='fc_1'),
FreezeDense(256, init_values[6], init_values[7], mask_values[6], mask_values[7], kernel_initializer=glorot_normal, activation=relu, kernel_regularizer=l2reg(args.l2), name='fc_2'),
FreezeDense(10, init_values[8], init_values[9], mask_values[8], mask_values[9], kernel_initializer=glorot_normal, activation=None, kernel_regularizer=l2reg(args.l2), name='fc_3')
])
def build_conv2_lottery(args):
return SequentialNetwork([
Conv2D(64, 3, kernel_initializer=glorot_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_1'),
Activation('relu'),
Conv2D(64, 3, kernel_initializer=glorot_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_2'),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
Flatten(),
Dense(256, kernel_initializer=glorot_normal, activation=relu, kernel_regularizer=l2reg(args.l2), name='fc_1'),
Dense(256, kernel_initializer=glorot_normal, activation=relu, kernel_regularizer=l2reg(args.l2), name='fc_2'),
Dense(10, kernel_initializer=glorot_normal, activation=None, kernel_regularizer=l2reg(args.l2), name='fc_3')
])
def build_lenet_conv(args): # ok this is a slightly modified lenet
return SequentialNetwork([
Conv2D(20, 5, kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_1'),
# BatchNormalization(momentum=0.0, name='batch_norm_1'),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
Conv2D(40, 5, kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_2'),
# BatchNormalization(momentum=0.0, name='batch_norm_2'),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
Flatten(),
Dropout(0.25),
Dense(400, kernel_initializer=he_normal, activation=relu, kernel_regularizer=l2reg(args.l2), name='fc_1'),
Dropout(0.5),
Dense(10, kernel_initializer=he_normal, activation=None, kernel_regularizer=l2reg(args.l2), name='fc_2')
])
def build_resnet(args):
return SequentialNetwork([
# pre-blocks
Conv2D(16, 3, kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_1'),
BatchNormalization(momentum=0.0, name='batch_norm_1'),
Activation('relu'),
# set 1
ResidualBlock(3, 16, first_stride=(1, 1), name_prefix='1A_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 16, first_stride=(1, 1), name_prefix='1B_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 16, first_stride=(1, 1), name_prefix='1C_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
# set 2
ResidualBlock(3, 32, first_stride=(2, 2), name_prefix='2A_', identity=False, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 32, first_stride=(1, 1), name_prefix='2B_', identity=True, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 32, first_stride=(1, 1), name_prefix='2C_', identity=True, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
tf.layers.Conv2DTranspose(16, 2, strides=(2, 2)),
# tf.layers.Conv2DTranspose(1, 15, padding='valid'),
# set 3
ResidualBlock(3, 64, first_stride=(2, 2), name_prefix='3A_', identity=False, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 64, first_stride=(1, 1), name_prefix='3B_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 64, first_stride=(1, 1), name_prefix='3C_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
# post-blocks
# GlobalAveragePooling2D(),
tf.layers.Conv2DTranspose(16, 1, strides=(2, 2)),
ResidualBlock(3, 64, first_stride=(2, 2), name_prefix='4A_', identity=False, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 64, first_stride=(1, 1), name_prefix='4B_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 64, first_stride=(1, 1), name_prefix='4C_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
# tf.layers.Conv2DTranspose(32, 1, strides=(2, 2)),
tf.layers.Conv2DTranspose(16, 1, strides=(2, 2)),
tf.layers.Conv2DTranspose(8, 1, strides=(1, 1)),
tf.layers.Conv2DTranspose(1, 1, strides=(1, 1)),
# tf.layers.Conv2DTranspose(1, 5, padding='valid'),
# tf.layers.Conv2DTranspose(1, 11, padding='valid'),
# tf.layers.Conv2DTranspose(1, 15, padding='valid'),
# tf.layers.Conv2DTranspose(1, 21, padding='valid'),
# tf.layers.Conv2DTranspose(1, 42, padding='valid', name='probs'),
Activation('sigmoid', name='mask')
# Dense(10, kernel_initializer=he_normal, activation=None, kernel_regularizer=l2reg(args.l2_special), name='fc_last')
])
def __init__(self, kernel_size, filters, first_stride, name_prefix='', identity=True, resize=1, l2=0, l2_shortcut=0, *args, **kwargs):
super(ResidualBlock, self).__init__(*args, **kwargs)
self.identity = identity
self.conv1 = self.track_layer(Conv2D(int(filters * resize), kernel_size, strides=first_stride, kernel_initializer=he_normal,
padding='same', kernel_regularizer=l2reg(l2), name=name_prefix+'conv2D_1'))
self.bn1 = self.track_layer(BatchNormalization(momentum=0.0, name=name_prefix+'batch_norm_1'))
self.act1 = self.track_layer(Activation('relu'))
self.conv2 = self.track_layer(Conv2D(filters, kernel_size, strides=(1, 1), kernel_initializer=he_normal,
padding='same', kernel_regularizer=l2reg(l2), name=name_prefix+'conv2D_2'))
self.bn2 = self.track_layer(BatchNormalization(momentum=0.0, name=name_prefix+'batch_norm_2'))
if not self.identity:
self.conv_shortcut = self.track_layer(Conv2D(filters, (1, 1), strides=first_stride, kernel_initializer=he_normal,
padding='same', kernel_regularizer=l2reg(l2_shortcut), name=name_prefix+'shortcut_conv'))
self.add_layer = self.track_layer(tfkeras.layers.Add())
self.act2 = self.track_layer(Activation('relu')) # TODO need relu on last block?
def build_linknet_2(args):
layers = conv_bn_relu(32, 3, stride=1, name="block1_conv1")
for layer in conv_bn_relu(32, 3, stride=1, name="block1_conv2"):
layers.append(layer)
layers.append(MaxPooling2D((2, 2), strides=(2, 2), padding="same", name="block1_pool"))
layers.append(Activation('relu'))
layers.append(Flatten())
layers.append(Dense(400, kernel_initializer=he_normal, activation=relu, kernel_regularizer=l2reg(0), name='fc_1'))
layers.append(Dense(10, kernel_initializer=he_normal, activation=None, kernel_regularizer=l2reg(0), name='fc_2'))
return SequentialNetwork(layers)
def build_masked_conv4_lottery(args, mask_values):
return SequentialNetwork([
MaskedConv2D(64, 3, mask_values[0], mask_values[1], kernel_initializer=glorot_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_1'),
Activation('relu'),
MaskedConv2D(64, 3, mask_values[2], mask_values[3], kernel_initializer=glorot_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_2'),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
MaskedConv2D(128, 3, mask_values[4], mask_values[5], kernel_initializer=glorot_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_3'),
Activation('relu'),
MaskedConv2D(128, 3, mask_values[6], mask_values[7], kernel_initializer=glorot_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_4'),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
Flatten(),
# Dropout(0.5),
MaskedDense(256, mask_values[8], mask_values[9], kernel_initializer=glorot_normal, activation=relu, kernel_regularizer=l2reg(args.l2), name='fc_1'),
# Dropout(0.5),
MaskedDense(256, mask_values[10], mask_values[11], kernel_initializer=glorot_normal, activation=relu, kernel_regularizer=l2reg(args.l2), name='fc_2'),
# Dropout(0.5),
MaskedDense(10, mask_values[12], mask_values[13], kernel_initializer=glorot_normal, activation=None, kernel_regularizer=l2reg(args.l2), name='fc_3')
])
def build_basic_model(args):
return SequentialNetwork([
Conv2D(16, 2, padding='same', name='conv2D_1'),
# BatchNormalization(momentum=0.0, name='batch_norm_1'),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
Conv2D(32, 2, padding='same', name='conv2D_2'),
# BatchNormalization(momentum=0.0, name='batch_norm_2'),
Activation('relu'),
MaxPooling2D((2, 2), (2, 2)),
Conv2D(64, 2, padding='same', name='conv2D_3'),
Activation('relu'),
MaxPooling2D((2,2), (2,2)),
GlobalAveragePooling2D(),
Dense(1000, activation=relu),
Dropout(0.2),
Dense(1000, activation=relu, name='fc_1'),
Dropout(0.2),
Dense(5, activation= None, name='fc_2')
])
def build_all_cnn(args):
return SequentialNetwork([
#Dropout(0.1),
Conv2D(96, (3, 3), kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_1'),
BatchNormalization(momentum=0.0, name='batch_norm_1'),
Activation('relu'),
Conv2D(96, (3, 3), kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_2'),
BatchNormalization(momentum=0.0, name='batch_norm_2'),
Activation('relu'),
Conv2D(96, (3, 3), kernel_initializer=he_normal, padding='same', strides=(2, 2), name='conv2D_strided_1'),
Dropout(0.5),
Conv2D(192, (3, 3), kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_3'),
BatchNormalization(momentum=0.0, name='batch_norm_3'),
Activation('relu'),
Conv2D(192, (3, 3), kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_4'),
BatchNormalization(momentum=0.0, name='batch_norm_4'),
Activation('relu'),
Conv2D(192, (3, 3), kernel_initializer=he_normal, padding='same', strides=(2, 2), name='conv2D_strided_2'),
Dropout(0.5),
Conv2D(192, (3, 3), kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_5'),
BatchNormalization(momentum=0.0, name='batch_norm_5'),
Activation('relu'),
Conv2D(192, (1, 1), kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_6'),
BatchNormalization(momentum=0.0, name='batch_norm_6'),
Activation('relu'),
Conv2D(10, (1, 1), kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_7'),
GlobalAveragePooling2D()
])
def conv_bn_relu(num_channel,
kernel_size,
stride,
name,
padding='same',
activation='relu'):
return [
Conv2D(filters=num_channel,
kernel_size=(kernel_size, kernel_size),
strides=stride,
padding=padding,
kernel_initializer="he_normal",
name=name + "_conv"),
BatchNormalization(name=name + '_bn'),
Activation(activation) # , name=name + '_relu'
]
def __init__(self, n_filters, stride=1, downsample=None, name=None):
super(ResidualBlockLinkNet, self).__init__(name=name)
if downsample is None:
self.shortcut = None
else:
self.shortcut = self.track_layers(downsample)
self.conv_bn_relu = self.track_layers(
conv_bn_relu(n_filters,
kernel_size=3,
stride=stride,
name=name + '/cvbnrelu'))
self.conv1 = self.track_layer(
Conv2D(n_filters, (3, 3), name=name + '_conv2', padding='same'))
self.bn1 = self.track_layer(BatchNormalization(name=name + '_bn'))
self.add_layer = self.track_layer(tfkeras.layers.Add())
self.act1 = self.track_layer(Activation('relu'))
def __init__(self, classes=1, dropout=0.5, feature_scale=4):
super(LinkNet, self).__init__()
self.conv_bn_relu_1 = self.track_layers(conv_bn_relu(32, 3, stride=1, name="block1_conv1"))
self.conv_bn_relu_2 = self.track_layers(conv_bn_relu(32, 3, stride=1, name="block1_conv2"))
self.maxPool = MaxPooling2D((2, 2), strides=(2, 2), padding="same", name="block1_pool")
layers = [2, 2, 2, 2, 2]
filters = [64, 128, 256, 512, 32]
enc1 = self.track_layers(encoder(m=32, n=filters[0], blocks=layers[0], stride=1, name='encoder1'))
enc2 = self.track_layers(encoder(m=filters[0], n=filters[1], blocks=layers[1], stride=2, name='encoder2'))
enc3 = self.track_layers(encoder(m=filters[1], n=filters[2], blocks=layers[2], stride=2, name='encoder3'))
enc4 = self.track_layers(encoder(m=filters[2], n=filters[3], blocks=layers[3], stride=2, name='encoder4'))
enc5 = self.track_layers(encoder(m=filters[3], n=filters[4], blocks=layers[4], stride=2, name='encoder5'))
self.decoder = self.track_layer(LinkNetDecoder(enc1, enc2, enc3, enc4, enc5, filters, feature_scale))
# self.dropout = tfkeras.layers.SpatialDropout2D(dropout)
self.conv1 = self.track_layer(Conv2D(filters=classes, kernel_size=(1, 1), padding='same', name='prediction'))
self.act = self.track_layer(Activation('sigmoid', name='mask'))
def deconv_bn_relu(num_channels,
kernel_size,
name,
transposed_conv,
activation='relu'):
layers = []
if transposed_conv:
layers.append(
tf.layers.Conv2DTranspose(num_channels,
kernel_size=(4, 4),
strides=(2, 2),
padding="same"))
else:
layers.append(tf.keras.layers.UpSampling2D())
layers.append(
Conv2D(num_channels,
kernel_size=(kernel_size, kernel_size),
kernel_initializer="he_normal",
padding='same'))
layers.append(BatchNormalization(name=name + '_bn'))
layers.append(Activation(activation))
return layers
def build_resnet(args):
return SequentialNetwork([
# pre-blocks
Conv2D(16, 3, kernel_initializer=he_normal, padding='same', kernel_regularizer=l2reg(args.l2), name='conv2D_1'),
BatchNormalization(momentum=0.0, name='batch_norm_1'),
Activation('relu'),
# set 1
ResidualBlock(3, 16, first_stride=(1, 1), name_prefix='1A_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 16, first_stride=(1, 1), name_prefix='1B_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 16, first_stride=(1, 1), name_prefix='1C_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
# set 2
ResidualBlock(3, 32, first_stride=(2, 2), name_prefix='2A_', identity=False, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 32, first_stride=(1, 1), name_prefix='2B_', identity=True, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 32, first_stride=(1, 1), name_prefix='2C_', identity=True, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
# set 3
ResidualBlock(3, 64, first_stride=(2, 2), name_prefix='3A_', identity=False, resize=args.resize_less, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 64, first_stride=(1, 1), name_prefix='3B_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
ResidualBlock(3, 64, first_stride=(1, 1), name_prefix='3C_', identity=True, resize=args.resize_more, l2=args.l2, l2_shortcut=args.l2),
# post-blocks
GlobalAveragePooling2D(),
Dense(10, kernel_initializer=he_normal, activation=None, kernel_regularizer=l2reg(args.l2_special), name='fc_last')
])
