def append_gan_network(self, x_in, true_X_input):
# Append the inputs to the output of the SRResNet
x = merge([x_in, true_X_input], mode='concat', concat_axis=0)
# Normalize the inputs via custom VGG Normalization layer
x = Normalize(type="gan", value=255., name="gan_normalize")(x)
x = Convolution2D(64,
self.k,
self.k,
border_mode='same',
name='gan_conv1_1')(x)
x = LeakyReLU(0.3, name="gan_lrelu1_1")(x)
x = Convolution2D(64,
self.k,
self.k,
border_mode='same',
name='gan_conv1_2',
subsample=(2, 2))(x)
x = LeakyReLU(0.3, name='gan_lrelu1_2')(x)
x = BatchNormalization(mode=self.mode, axis=1,
name='gan_batchnorm1_1')(x)
filters = [128, 256] if self.small_model else [128, 256, 512]
for i, nb_filters in enumerate(filters):
for j in range(2):
subsample = (2, 2) if j == 1 else (1, 1)
x = Convolution2D(nb_filters,
self.k,
self.k,
border_mode='same',
subsample=subsample,
name='gan_conv%d_%d' % (i + 2, j + 1))(x)
x = LeakyReLU(0.3, name='gan_lrelu_%d_%d' % (i + 2, j + 1))(x)
x = BatchNormalization(mode=self.mode,
axis=1,
name='gan_batchnorm%d_%d' %
(i + 2, j + 1))(x)
x = Flatten(name='gan_flatten')(x)
output_dim = 128 if self.small_model else 1024
x = Dense(output_dim, name='gan_dense1')(x)
x = LeakyReLU(0.3, name='gan_lrelu5')(x)
gan_regulrizer = AdversarialLossRegularizer(
weight=self.adversarial_loss_weight)
x = Dense(1,
activation="sigmoid",
activity_regularizer=gan_regulrizer,
name='gan_output')(x)
return x
def mnist_feature_learner(name, images, DIM, dropout=None, is_training=None):
initializer = 'glorot_uniform'
normalizer = 'BN'
output = Conv2D('{}.feature.Conv.1'.format(name), images, DIM, initializer=initializer)
output = tf.nn.max_pool(output, (1,1,2,2), (1,1,2,2), 'VALID', data_format='NCHW')
output = Normalize('{}.feature.NORM.1'.format(name), output, method=normalizer, bn_is_training=is_training)
output = tf.nn.relu(output)
output = Conv2D('{}.feature.Conv.2'.format(name), output, 2*DIM, initializer=initializer)
output = tf.nn.max_pool(output, (1,1,2,2), (1,1,2,2), 'VALID', data_format='NCHW')
output = Normalize('{}.feature.NORM.2'.format(name), output, method=normalizer, bn_is_training=is_training)
output = tf.nn.relu(output)
output = Conv2D('{}.feature.Conv.3'.format(name), output, 4*DIM, initializer=initializer)
output = tf.nn.max_pool(output, (1,1,2,2), (1,1,2,2), 'VALID', data_format='NCHW')
output = Normalize('{}.feature.NORM.3'.format(name), output, method=normalizer, bn_is_training=is_training)
output = tf.nn.relu(output)
output = tf.reduce_mean(output, [2,3])
return output
def append_vgg_network(self, x_in, true_X_input, pre_train=False):
# Append the initial inputs to the outputs of the SRResNet
x = merge([x_in, true_X_input], mode='concat', concat_axis=0)
# Normalize the inputs via custom VGG Normalization layer
x = Normalize(name="normalize_vgg")(x)
# Begin adding the VGG layers
x = Convolution2D(64,
3,
3,
activation='relu',
name='vgg_conv1_1',
border_mode='same')(x)
x = Convolution2D(64,
3,
3,
activation='relu',
name='vgg_conv1_2',
border_mode='same')(x)
x = MaxPooling2D(name='vgg_maxpool1')(x)
x = Convolution2D(128,
3,
3,
activation='relu',
name='vgg_conv2_1',
border_mode='same')(x)
if pre_train:
vgg_regularizer2 = ContentVGGRegularizer(weight=self.vgg_weight)
x = Convolution2D(128,
3,
3,
activation='relu',
name='vgg_conv2_2',
border_mode='same',
activity_regularizer=vgg_regularizer2)(x)
else:
x = Convolution2D(128,
3,
3,
activation='relu',
name='vgg_conv2_2',
border_mode='same')(x)
x = MaxPooling2D(name='vgg_maxpool2')(x)
x = Convolution2D(256,
3,
3,
activation='relu',
name='vgg_conv3_1',
border_mode='same')(x)
x = Convolution2D(256,
3,
3,
activation='relu',
name='vgg_conv3_2',
border_mode='same')(x)
x = Convolution2D(256,
3,
3,
activation='relu',
name='vgg_conv3_3',
border_mode='same')(x)
x = MaxPooling2D(name='vgg_maxpool3')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
name='vgg_conv4_1',
border_mode='same')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
name='vgg_conv4_2',
border_mode='same')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
name='vgg_conv4_3',
border_mode='same')(x)
x = MaxPooling2D(name='vgg_maxpool4')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
name='vgg_conv5_1',
border_mode='same')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
name='vgg_conv5_2',
border_mode='same')(x)
if not pre_train:
vgg_regularizer5 = ContentVGGRegularizer(weight=self.vgg_weight)
x = Convolution2D(512,
3,
3,
activation='relu',
name='vgg_conv5_3',
border_mode='same',
activity_regularizer=vgg_regularizer5)(x)
else:
x = Convolution2D(512,
3,
3,
activation='relu',
name='vgg_conv5_3',
border_mode='same')(x)
x = MaxPooling2D(name='vgg_maxpool5')(x)
return x
def SSD300(input_shape, num_classes=21):
"""SSD300 architecture.
# Arguments
input_shape: Shape of the input image,
expected to be either (300, 300, 3) or (3, 300, 300)(not tested).
num_classes: Number of classes including background.
# References
https://arxiv.org/abs/1512.02325
"""
print('begin building networks')
kernel_size = (3, 3)
net = {}
# Block 1
input_tensor = Input(shape=input_shape)
img_size = (input_shape[1], input_shape[0])
net['input'] = input_tensor
net['conv1_1'] = Conv2D(64,
kernel_size,
activation='relu',
padding='same',
name='conv1_1')(net['input'])
net['conv1_2'] = Conv2D(64,
kernel_size,
activation='relu',
padding='same',
name='conv1_2')(net['conv1_1'])
net['pool1'] = MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='pool1')(net['conv1_2'])
# Block 2
net['conv2_1'] = Conv2D(128,
kernel_size,
activation='relu',
padding='same',
name='conv2_1')(net['pool1'])
net['conv2_2'] = Conv2D(128,
kernel_size,
activation='relu',
padding='same',
name='conv2_2')(net['conv2_1'])
net['pool2'] = MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='pool2')(net['conv2_2'])
# Block 3
net['conv3_1'] = Conv2D(256,
kernel_size,
activation='relu',
padding='same',
name='conv3_1')(net['pool2'])
net['conv3_2'] = Conv2D(256,
kernel_size,
activation='relu',
padding='same',
name='conv3_2')(net['conv3_1'])
net['conv3_3'] = Conv2D(256,
kernel_size,
activation='relu',
padding='same',
name='conv3_3')(net['conv3_2'])
net['pool3'] = MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='pool3')(net['conv3_3'])
# Block 4
net['conv4_1'] = Conv2D(512,
kernel_size,
activation='relu',
padding='same',
name='conv4_1')(net['pool3'])
net['conv4_2'] = Conv2D(512,
kernel_size,
activation='relu',
padding='same',
name='conv4_2')(net['conv4_1'])
net['conv4_3'] = Conv2D(512,
kernel_size,
activation='relu',
padding='same',
name='conv4_3')(net['conv4_2'])
net['pool4'] = MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='pool4')(net['conv4_3'])
# Block 5
net['conv5_1'] = Conv2D(512,
kernel_size,
activation='relu',
padding='same',
name='conv5_1')(net['pool4'])
net['conv5_2'] = Conv2D(512,
kernel_size,
activation='relu',
padding='same',
name='conv5_2')(net['conv5_1'])
net['conv5_3'] = Conv2D(512,
kernel_size,
activation='relu',
padding='same',
name='conv5_3')(net['conv5_2'])
net['pool5'] = MaxPooling2D((3, 3),
strides=(1, 1),
padding='same',
name='pool5')(net['conv5_3'])
# FC6
net['fc6'] = Conv2D(1024,
kernel_size,
dilation_rate=(6, 6),
activation='relu',
padding='same',
name='fc6')(net['pool5'])
# x = Dropout(0.5, name='drop6')(x)
# FC7
net['fc7'] = Conv2D(1024, (1, 1),
activation='relu',
padding='same',
name='fc7')(net['fc6'])
# x = Dropout(0.5, name='drop7')(x)
# Block 6
net['conv6_1'] = Conv2D(256, (1, 1),
activation='relu',
padding='same',
name='conv6_1')(net['fc7'])
net['conv6_2'] = Conv2D(512,
kernel_size,
strides=(2, 2),
activation='relu',
padding='same',
name='conv6_2')(net['conv6_1'])
# Block 7
net['conv7_1'] = Conv2D(128, (1, 1),
activation='relu',
padding='same',
name='conv7_1')(net['conv6_2'])
net['conv7_2'] = ZeroPadding2D()(net['conv7_1'])
net['conv7_2'] = Conv2D(256,
kernel_size,
strides=(2, 2),
activation='relu',
padding='valid',
name='conv7_2')(net['conv7_2'])
# Block 8
net['conv8_1'] = Conv2D(128, (1, 1),
activation='relu',
padding='same',
name='conv8_1')(net['conv7_2'])
net['conv8_2'] = Conv2D(256,
kernel_size,
strides=(2, 2),
activation='relu',
padding='same',
name='conv8_2')(net['conv8_1'])
# Last Pool
net['pool6'] = GlobalAveragePooling2D(name='pool6')(net['conv8_2'])
print('base network built')
# Prediction from conv4_3
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm')(net['conv4_3'])
num_priors = 3
x = Conv2D(num_priors * 4,
kernel_size,
padding='same',
name='conv4_3_norm_mbox_loc')(net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc'] = x
flatten = Flatten(name='conv4_3_norm_mbox_loc_flat')
net['conv4_3_norm_mbox_loc_flat'] = flatten(net['conv4_3_norm_mbox_loc'])
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes,
kernel_size,
padding='same',
name=name)(net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf'] = x
flatten = Flatten(name='conv4_3_norm_mbox_conf_flat')
net['conv4_3_norm_mbox_conf_flat'] = flatten(net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size,
30.0,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv4_3_norm_mbox_priorbox')
net['conv4_3_norm_mbox_priorbox'] = priorbox(net['conv4_3_norm'])
print('conv4_3_norm_mbox_priorbox built')
# Prediction from fc7
num_priors = 6
net['fc7_mbox_loc'] = Conv2D(num_priors * 4,
kernel_size,
padding='same',
name='fc7_mbox_loc')(net['fc7'])
flatten = Flatten(name='fc7_mbox_loc_flat')
net['fc7_mbox_loc_flat'] = flatten(net['fc7_mbox_loc'])
name = 'fc7_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
net['fc7_mbox_conf'] = Conv2D(num_priors * num_classes, (3, 3),
padding='same',
name=name)(net['fc7'])
flatten = Flatten(name='fc7_mbox_conf_flat')
net['fc7_mbox_conf_flat'] = flatten(net['fc7_mbox_conf'])
priorbox = PriorBox(img_size,
60.0,
max_size=114.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='fc7_mbox_priorbox')
net['fc7_mbox_priorbox'] = priorbox(net['fc7'])
print('fc7_mbox_priorbox built')
# Prediction from conv6_2
num_priors = 6
x = Conv2D(num_priors * 4,
kernel_size,
padding='same',
name='conv6_2_mbox_loc')(net['conv6_2'])
net['conv6_2_mbox_loc'] = x
flatten = Flatten(name='conv6_2_mbox_loc_flat')
net['conv6_2_mbox_loc_flat'] = flatten(net['conv6_2_mbox_loc'])
name = 'conv6_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes,
kernel_size,
padding='same',
name=name)(net['conv6_2'])
net['conv6_2_mbox_conf'] = x
flatten = Flatten(name='conv6_2_mbox_conf_flat')
net['conv6_2_mbox_conf_flat'] = flatten(net['conv6_2_mbox_conf'])
priorbox = PriorBox(img_size,
114.0,
max_size=168.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv6_2_mbox_priorbox')
net['conv6_2_mbox_priorbox'] = priorbox(net['conv6_2'])
print('conv6_2_mbox_priorbox built')
# Prediction from conv7_2
num_priors = 6
x = Conv2D(num_priors * 4,
kernel_size,
padding='same',
name='conv7_2_mbox_loc')(net['conv7_2'])
net['conv7_2_mbox_loc'] = x
flatten = Flatten(name='conv7_2_mbox_loc_flat')
net['conv7_2_mbox_loc_flat'] = flatten(net['conv7_2_mbox_loc'])
name = 'conv7_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes,
kernel_size,
padding='same',
name=name)(net['conv7_2'])
net['conv7_2_mbox_conf'] = x
flatten = Flatten(name='conv7_2_mbox_conf_flat')
net['conv7_2_mbox_conf_flat'] = flatten(net['conv7_2_mbox_conf'])
priorbox = PriorBox(img_size,
168.0,
max_size=222.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv7_2_mbox_priorbox')
net['conv7_2_mbox_priorbox'] = priorbox(net['conv7_2'])
print('conv7_2_mbox_priorbox built')
# Prediction from conv8_2
num_priors = 6
x = Conv2D(num_priors * 4,
kernel_size,
padding='same',
name='conv8_2_mbox_loc')(net['conv8_2'])
net['conv8_2_mbox_loc'] = x
flatten = Flatten(name='conv8_2_mbox_loc_flat')
net['conv8_2_mbox_loc_flat'] = flatten(net['conv8_2_mbox_loc'])
name = 'conv8_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes,
kernel_size,
padding='same',
name=name)(net['conv8_2'])
net['conv8_2_mbox_conf'] = x
flatten = Flatten(name='conv8_2_mbox_conf_flat')
net['conv8_2_mbox_conf_flat'] = flatten(net['conv8_2_mbox_conf'])
priorbox = PriorBox(img_size,
222.0,
max_size=276.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv8_2_mbox_priorbox')
net['conv8_2_mbox_priorbox'] = priorbox(net['conv8_2'])
print('conv8_2_mbox_priorbox built')
# Prediction from pool6
num_priors = 6
x = Dense(num_priors * 4, name='pool6_mbox_loc_flat')(net['pool6'])
net['pool6_mbox_loc_flat'] = x
name = 'pool6_mbox_conf_flat'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Dense(num_priors * num_classes, name=name)(net['pool6'])
net['pool6_mbox_conf_flat'] = x
priorbox = PriorBox(img_size,
276.0,
max_size=330.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='pool6_mbox_priorbox')
if K.image_dim_ordering() == 'tf':
target_shape = (1, 1, 256)
else:
target_shape = (256, 1, 1)
net['pool6_reshaped'] = Reshape(target_shape,
name='pool6_reshaped')(net['pool6'])
net['pool6_mbox_priorbox'] = priorbox(net['pool6_reshaped'])
print('pool6_mbox_priorbox built')
# Gather all predictions
net['mbox_loc'] = concatenate([
net['conv4_3_norm_mbox_loc_flat'], net['fc7_mbox_loc_flat'],
net['conv6_2_mbox_loc_flat'], net['conv7_2_mbox_loc_flat'],
net['conv8_2_mbox_loc_flat'], net['pool6_mbox_loc_flat']
],
axis=1,
name='mbox_loc')
net['mbox_conf'] = concatenate([
net['conv4_3_norm_mbox_conf_flat'], net['fc7_mbox_conf_flat'],
net['conv6_2_mbox_conf_flat'], net['conv7_2_mbox_conf_flat'],
net['conv8_2_mbox_conf_flat'], net['pool6_mbox_conf_flat']
],
axis=1,
name='mbox_conf')
net['mbox_priorbox'] = concatenate([
net['conv4_3_norm_mbox_priorbox'], net['fc7_mbox_priorbox'],
net['conv6_2_mbox_priorbox'], net['conv7_2_mbox_priorbox'],
net['conv8_2_mbox_priorbox'], net['pool6_mbox_priorbox']
],
axis=1,
name='mbox_priorbox')
print('gathering all prediction layers built')
if hasattr(net['mbox_loc'], '_keras_shape'):
# divide 4 for [xmin, ymin, xmax, ymax]
num_boxes = net['mbox_loc']._keras_shape[-1] // 4
elif hasattr(net['mbox_loc'], 'int_shape'):
num_boxes = K.int_shape(net['mbox_loc'])[-1] // 4
net['mbox_loc'] = Reshape((num_boxes, 4),
name='mbox_loc_final')(net['mbox_loc'])
net['mbox_conf'] = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(net['mbox_conf'])
net['mbox_conf'] = Activation('softmax',
name='mbox_conf_final')(net['mbox_conf'])
net['predictions'] = concatenate(
[net['mbox_loc'], net['mbox_conf'], net['mbox_priorbox']],
axis=2,
name='predictions')
print('prediction layers built')
model = Model(net['input'], net['predictions'])
return model
def mini_SSD300(input_shape=(300,300,3), num_classes=21):
"""SSD300 architecture.
# Arguments
input_shape: Shape of the input image,
expected to be either (300, 300, 3) or (3, 300, 300)(not tested).
num_classes: Number of classes including background.
# References
https://arxiv.org/abs/1512.02325
"""
net = {}
# Block 1
input_tensor = input_tensor = Input(shape=input_shape)
img_size = (input_shape[1], input_shape[0])
net['input'] = input_tensor
net['conv1_1'] = Convolution2D(64, 3, 3,
activation='relu',
border_mode='same',
name='conv1_1')(net['input'])
net['conv1_2'] = Convolution2D(64, 3, 3,
activation='relu',
border_mode='same',
name='conv1_2')(net['conv1_1'])
net['pool1'] = MaxPooling2D((2, 2), strides=(2, 2), border_mode='same',
name='pool1')(net['conv1_2'])
# Block 2
net['conv2_1'] = Convolution2D(128, 3, 3,
activation='relu',
border_mode='same',
name='conv2_1')(net['pool1'])
net['conv2_2'] = Convolution2D(128, 3, 3,
activation='relu',
border_mode='same',
name='conv2_2')(net['conv2_1'])
net['pool2'] = MaxPooling2D((2, 2), strides=(2, 2), border_mode='same',
name='pool2')(net['conv2_2'])
# Block 3
net['conv3_1'] = Convolution2D(256, 3, 3,
activation='relu',
border_mode='same',
name='conv3_1')(net['pool2'])
net['conv3_2'] = Convolution2D(256, 3, 3,
activation='relu',
border_mode='same',
name='conv3_2')(net['conv3_1'])
net['conv3_3'] = Convolution2D(256, 3, 3,
activation='relu',
border_mode='same',
name='conv3_3')(net['conv3_2'])
net['pool3'] = MaxPooling2D((2, 2), strides=(2, 2), border_mode='same',
name='pool3')(net['conv3_3'])
# Block 4
net['conv4_1'] = Convolution2D(512, 3, 3,
activation='relu',
border_mode='same',
name='conv4_1')(net['pool3'])
net['conv4_2'] = Convolution2D(512, 3, 3,
activation='relu',
border_mode='same',
name='conv4_2')(net['conv4_1'])
net['conv4_3'] = Convolution2D(512, 3, 3,
activation='relu',
border_mode='same',
name='conv4_3')(net['conv4_2'])
net['pool4'] = MaxPooling2D((2, 2), strides=(2, 2), border_mode='same',
name='pool4')(net['conv4_3'])
# Block 5
net['conv5_1'] = Convolution2D(512, 3, 3,
activation='relu',
border_mode='same',
name='conv5_1')(net['pool4'])
net['conv5_2'] = Convolution2D(512, 3, 3,
activation='relu',
border_mode='same',
name='conv5_2')(net['conv5_1'])
net['conv5_3'] = Convolution2D(512, 3, 3,
activation='relu',
border_mode='same',
name='conv5_3')(net['conv5_2'])
net['pool5'] = MaxPooling2D((3, 3), strides=(1, 1), border_mode='same',
name='pool5')(net['conv5_3'])
# FC6
net['fc6'] = AtrousConvolution2D(1024, 3, 3, atrous_rate=(6, 6),
activation='relu', border_mode='same',
name='fc6')(net['pool5'])
# x = Dropout(0.5, name='drop6')(x)
# FC7
net['fc7'] = Convolution2D(1024, 1, 1, activation='relu',
border_mode='same', name='fc7')(net['fc6'])
# x = Dropout(0.5, name='drop7')(x)
# Block 6
# deleted
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm')(net['conv4_3'])
num_priors = 3
x = Convolution2D(num_priors * 4, 3, 3, border_mode='same',
name='conv4_3_norm_mbox_loc')(net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc'] = x
flatten = Flatten(name='conv4_3_norm_mbox_loc_flat')
net['conv4_3_norm_mbox_loc_flat'] = flatten(net['conv4_3_norm_mbox_loc'])
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes, 3, 3, border_mode='same',
name=name)(net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf'] = x
flatten = Flatten(name='conv4_3_norm_mbox_conf_flat')
net['conv4_3_norm_mbox_conf_flat'] = flatten(net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size, 30.0, aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv4_3_norm_mbox_priorbox')
net['conv4_3_norm_mbox_priorbox'] = priorbox(net['conv4_3_norm'])
# Prediction from fc7
num_priors = 6
net['fc7_mbox_loc'] = Convolution2D(num_priors * 4, 3, 3,
border_mode='same',
name='fc7_mbox_loc')(net['fc7'])
flatten = Flatten(name='fc7_mbox_loc_flat')
net['fc7_mbox_loc_flat'] = flatten(net['fc7_mbox_loc'])
name = 'fc7_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
net['fc7_mbox_conf'] = Convolution2D(num_priors * num_classes, 3, 3,
border_mode='same',
name=name)(net['fc7'])
flatten = Flatten(name='fc7_mbox_conf_flat')
net['fc7_mbox_conf_flat'] = flatten(net['fc7_mbox_conf'])
priorbox = PriorBox(img_size, 60.0, max_size=114.0, aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='fc7_mbox_priorbox')
net['fc7_mbox_priorbox'] = priorbox(net['fc7'])
# Gather all predictions
net['mbox_loc'] = merge([net['conv4_3_norm_mbox_loc_flat'],
net['fc7_mbox_loc_flat']],
mode='concat', concat_axis=1, name='mbox_loc')
net['mbox_conf'] = merge([net['conv4_3_norm_mbox_conf_flat'],
net['fc7_mbox_conf_flat']],
mode='concat', concat_axis=1, name='mbox_conf')
if hasattr(net['mbox_loc'], '_keras_shape'):
num_boxes = net['mbox_loc']._keras_shape[-1] // 4
elif hasattr(net['mbox_loc'], 'int_shape'):
num_boxes = K.int_shape(net['mbox_loc'])[-1] // 4
net['mbox_loc'] = Reshape((num_boxes, 4),
name='mbox_loc_final')(net['mbox_loc'])
net['mbox_conf'] = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(net['mbox_conf'])
net['mbox_conf'] = Activation('softmax',
name='mbox_conf_final')(net['mbox_conf'])
net['mbox_priorbox'] = merge([net['conv4_3_norm_mbox_priorbox'],
net['fc7_mbox_priorbox']],
mode='concat', concat_axis=1,
name='mbox_priorbox')
net['predictions'] = merge([net['mbox_loc'],
net['mbox_conf'],
net['mbox_priorbox']],
mode='concat', concat_axis=2,
name='predictions')
model = Model(net['input'], net['predictions'])
return model
def feature_learner(name, images, DIM, dropout=None, is_training=None):
output = images
output = Conv2D('{}.conv.Init'.format(name),
output,
DIM,
initializer='glorot_uniform')
output = ResidualLayer('{}.conv.1.1'.format(name),
output,
DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.1.2'.format(name),
output,
DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.1.3'.format(name),
output,
DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.1.4'.format(name),
output,
2 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.2.1'.format(name),
output,
2 * DIM,
stride=2,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.2.2'.format(name),
output,
2 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.2.3'.format(name),
output,
2 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.2.4'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.3.1'.format(name),
output,
4 * DIM,
stride=2,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.3.2'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.3.3'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.3.4'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.4.1'.format(name),
output,
4 * DIM,
stride=2,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.4.2'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.4.3'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = ResidualLayer('{}.conv.4.4'.format(name),
output,
4 * DIM,
is_training=is_training,
dropout=dropout)
output = Normalize('{}.convout.3.NORM'.format(name),
output,
bn_is_training=is_training)
output = tf.nn.relu(output)
output = tf.reduce_mean(output, [2, 3])
return output
def SSD300(input_shape=(300, 300, 3),
num_classes=21,
weights_path=None,
frozen_layers=None):
input_layer = Input(shape=input_shape)
# Block 1 -----------------------------------------------
conv1_1 = Conv2D(64, (3, 3),
name='conv1_1',
padding='same',
activation='relu')(input_layer)
conv1_2 = Conv2D(64, (3, 3),
name='conv1_2',
padding='same',
activation='relu')(conv1_1)
pool1 = MaxPooling2D(
name='pool1',
pool_size=(2, 2),
strides=(2, 2),
padding='same',
)(conv1_2)
# Block 2 ----------------------------------------------
conv2_1 = Conv2D(128, (3, 3),
name='conv2_1',
padding='same',
activation='relu')(pool1)
conv2_2 = Conv2D(128, (3, 3),
name='conv2_2',
padding='same',
activation='relu')(conv2_1)
pool2 = MaxPooling2D(name='pool2',
pool_size=(2, 2),
strides=(2, 2),
padding='same')(conv2_2)
# Block 3 ----------------------------------------------
conv3_1 = Conv2D(256, (3, 3),
name='conv3_1',
padding='same',
activation='relu')(pool2)
conv3_2 = Conv2D(256, (3, 3),
name='conv3_2',
padding='same',
activation='relu')(conv3_1)
conv3_3 = Conv2D(256, (3, 3),
name='conv3_3',
padding='same',
activation='relu')(conv3_2)
pool3 = MaxPooling2D(name='pool3',
pool_size=(2, 2),
strides=(2, 2),
padding='same')(conv3_3)
# Block 4 ---------------------------------------------
conv4_1 = Conv2D(512, (3, 3),
name='conv4_1',
padding='same',
activation='relu')(pool3)
conv4_2 = Conv2D(512, (3, 3),
name='conv4_2',
padding='same',
activation='relu')(conv4_1)
conv4_3 = Conv2D(512, (3, 3),
name='conv4_3',
padding='same',
activation='relu')(conv4_2)
pool4 = MaxPooling2D(name='pool4',
pool_size=(2, 2),
strides=(2, 2),
padding='same')(conv4_3)
# Block 5 --------------------------------------------
conv5_1 = Conv2D(512, (3, 3),
name='conv5_1',
padding='same',
activation='relu')(pool4)
conv5_2 = Conv2D(512, (3, 3),
name='conv5_2',
padding='same',
activation='relu')(conv5_1)
conv5_3 = Conv2D(512, (3, 3),
name='conv5_3',
padding='same',
activation='relu')(conv5_2)
pool5 = MaxPooling2D(name='pool5',
pool_size=(3, 3),
strides=(1, 1),
padding='same')(conv5_3)
# FC6 ------------------------------------------------
fc6 = Conv2D(1024, (3, 3),
name='fc6',
dilation_rate=(6, 6),
padding='same',
activation='relu')(pool5)
# FC7 ------------------------------------------------
fc7 = Conv2D(1024, (1, 1), name='fc7', padding='same',
activation='relu')(fc6)
# EXTRAS
# Block 6 --------------------------------------------
conv6_1 = Conv2D(256, (1, 1),
name='conv6_1',
padding='same',
activation='relu')(fc7)
conv6_2 = Conv2D(512, (3, 3),
name='conv6_2',
strides=(2, 2),
padding='same',
activation='relu')(conv6_1)
# Block 7 --------------------------------------------
conv7_1 = Conv2D(128, (1, 1),
name='conv7_1',
padding='same',
activation='relu')(conv6_2)
#conv7_1z = ZeroPadding2D(name='conv7_1z')(conv7_1)
conv7_2 = Conv2D(256, (3, 3),
name='conv7_2',
padding='same',
strides=(2, 2),
activation='relu')(conv7_1)
# Block 8 --------------------------------------------
conv8_1 = Conv2D(128, (1, 1),
name='conv8_1',
padding='same',
activation='relu')(conv7_2)
conv8_2 = Conv2D(256, (3, 3),
name='conv8_2',
padding='valid',
strides=(1, 1),
activation='relu')(conv8_1)
# Block 9 -------------------------------------------
conv9_1 = Conv2D(128, (1, 1),
name='conv9_1',
padding='same',
activation='relu')(conv8_2)
conv9_2 = Conv2D(256, (3, 3),
name='conv9_2',
padding='valid',
strides=(1, 1),
activation='relu')(conv9_1)
# Last Pool ------------------------------------------
# pool6 = GlobalAveragePooling2D(name='pool6')(conv8_2)
# Prediction from conv4_3 ----------------------------
num_priors = 4
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
conv4_3_norm = Normalize(20, name='conv4_3_norm')(conv4_3)
conv4_3_norm_mbox_loc = Conv2D(num_priors * 4, (3, 3),
name='conv4_3_norm_mbox_loc',
padding='same')(conv4_3_norm)
conv4_3_norm_mbox_loc_flat = Flatten(
name='conv4_3_norm_mbox_loc_flat')(conv4_3_norm_mbox_loc)
conv4_3_norm_mbox_conf = Conv2D(num_priors * num_classes, (3, 3),
name=name,
padding='same')(conv4_3_norm)
conv4_3_norm_mbox_conf_flat = Flatten(
name='conv4_3_norm_mbox_conf_flat')(conv4_3_norm_mbox_conf)
# Prediction from fc7 ---------------------------------
num_priors = 6
name = 'fc7_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
fc7_mbox_conf = Conv2D(num_priors * num_classes, (3, 3),
padding='same',
name=name)(fc7)
fc7_mbox_conf_flat = Flatten(name='fc7_mbox_conf_flat')(fc7_mbox_conf)
fc7_mbox_loc = Conv2D(num_priors * 4, (3, 3),
name='fc7_mbox_loc',
padding='same')(fc7)
fc7_mbox_loc_flat = Flatten(name='fc7_mbox_loc_flat')(fc7_mbox_loc)
# Prediction from conv6_2 ------------------------------
num_priors = 6
name = 'conv6_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
conv6_2_mbox_conf = Conv2D(num_priors * num_classes, (3, 3),
padding='same',
name=name)(conv6_2)
conv6_2_mbox_conf_flat = Flatten(
name='conv6_2_mbox_conf_flat')(conv6_2_mbox_conf)
conv6_2_mbox_loc = Conv2D(num_priors * 4, (
3,
3,
),
name='conv6_2_mbox_loc',
padding='same')(conv6_2)
conv6_2_mbox_loc_flat = Flatten(
name='conv6_2_mbox_loc_flat')(conv6_2_mbox_loc)
# Prediction from conv7_2 --------------------------------
num_priors = 6
name = 'conv7_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
conv7_2_mbox_conf = Conv2D(num_priors * num_classes, (3, 3),
padding='same',
name=name)(conv7_2)
conv7_2_mbox_conf_flat = Flatten(
name='conv7_2_mbox_conf_flat')(conv7_2_mbox_conf)
conv7_2_mbox_loc = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='conv7_2_mbox_loc')(conv7_2)
conv7_2_mbox_loc_flat = Flatten(
name='conv7_2_mbox_loc_flat')(conv7_2_mbox_loc)
# Prediction from conv8_2 -------------------------------
num_priors = 4
name = 'conv8_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
conv8_2_mbox_conf = Conv2D(num_priors * num_classes, (3, 3),
padding='same',
name=name)(conv8_2)
conv8_2_mbox_conf_flat = Flatten(
name='conv8_2_mbox_conf_flat')(conv8_2_mbox_conf)
conv8_2_mbox_loc = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='conv8_2_mbox_loc')(conv8_2)
conv8_2_mbox_loc_flat = Flatten(
name='conv8_2_mbox_loc_flat')(conv8_2_mbox_loc)
# Prediction from pool6 --------------------------------------------
"""
num_priors = 4
name = 'pool6_mbox_conf_flat'
if num_classes != 21:
name += '_{}'.format(num_classes)
pool6_mbox_loc_flat = Dense(num_priors * 4, name='pool6_mbox_loc_flat')(pool6)
pool6_mbox_conf_flat = Dense(num_priors * num_classes, name=name)(pool6)
"""
num_priors = 4
name = 'pool6_mbox_conf_flat'
if num_classes != 21:
name += '_{}'.format(num_classes)
conv9_2_mbox_conf = Conv2D(num_priors * num_classes, (3, 3),
padding='same',
name=name)(conv9_2)
pool6_mbox_conf_flat = Flatten(
name='pool6_mbox_loc_flat')(conv9_2_mbox_conf)
conv9_2_mbox_loc = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='pool6_mbox_loc')(conv9_2)
pool6_mbox_loc_flat = Flatten(
name='pool6_2_mbox_loc_flat')(conv9_2_mbox_loc)
# Gather all predictions -------------------------------------------
mbox_loc = concatenate([
conv4_3_norm_mbox_loc_flat, fc7_mbox_loc_flat, conv6_2_mbox_loc_flat,
conv7_2_mbox_loc_flat, conv8_2_mbox_loc_flat, pool6_mbox_loc_flat
],
axis=1,
name='mbox_loc')
mbox_conf = concatenate([
conv4_3_norm_mbox_conf_flat, fc7_mbox_conf_flat,
conv6_2_mbox_conf_flat, conv7_2_mbox_conf_flat, conv8_2_mbox_conf_flat,
pool6_mbox_conf_flat
],
axis=1,
name='mbox_conf')
if hasattr(mbox_loc, '_keras_shape'):
num_boxes = mbox_loc._keras_shape[-1] // 4
elif hasattr(mbox_loc, 'int_shape'):
num_boxes = K.int_shape(mbox_loc)[-1] // 4
mbox_loc = Reshape((num_boxes, 4), name='mbox_loc_final')(mbox_loc)
mbox_conf = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(mbox_conf)
mbox_conf = Activation('softmax', name='mbox_conf_final')(mbox_conf)
predictions = concatenate([mbox_loc, mbox_conf],
axis=2,
name='predictions')
model = Model(inputs=input_layer, outputs=predictions)
if weights_path is not None:
model.load_weights(weights_path, by_name=True)
if frozen_layers is not None:
for layer in model.layers:
if layer.name in frozen_layers:
layer.trainable = False
return model
def simple_SSD(input_shape, num_classes, min_size, num_priors, max_size,
aspect_ratios, variances):
input_tensor = Input(shape=input_shape)
body = Convolution2D(16, 7, 7)(input_tensor)
body = Activation('relu')(body)
body = MaxPooling2D(2, 2, border_mode='valid')(body)
body = Convolution2D(32, 5, 5)(body)
body = Activation('relu')(body)
branch_1 = MaxPooling2D(2, 2, border_mode='valid')(body)
body = Convolution2D(64, 3, 3)(branch_1)
body = Activation('relu')(body)
branch_2 = MaxPooling2D(2, 2, border_mode='valid')(body)
# first branch
norm_1 = Normalize(20)(branch_1)
localization_1 = Convolution2D(num_priors * 4, 3, 3,
border_mode='same')(norm_1)
localization_1 = Flatten(localization_1)
classification_1 = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same')(norm_1)
classification_1 = Flatten()(classification_1)
prior_boxes_1 = PriorBox(input_shape[0:2], min_size, max_size,
aspect_ratios)
# second branch
norm_2 = Normalize(20)(branch_2)
localization_2 = Convolution2D(num_priors * 4, 3, 3,
border_mode='same')(norm_2)
localization_2 = Flatten(localization_2)
classification_2 = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same')(norm_2)
classification_2 = Flatten()(classification_2)
prior_boxes_2 = PriorBox(input_shape[0:2], min_size, max_size,
aspect_ratios)
localization_head = Merge([localization_1, localization_2],
mode='concat',
concat_axis=1)
classification_head = Merge([classification_1, classification_2],
mode='concat',
concat_axis=1)
prior_boxes_head = Merge([prior_boxes_1, prior_boxes_2],
mode='concat',
concat_axis=1)
if hasattr(localization_head, '_keras_shape'):
num_boxes = localization_head._keras_shape[-1] // 4
elif hasattr(localization_head, 'int_shape'):
num_boxes = K.int_shape(localization_head)[-1] // 4
localization_head = Reshape((num_boxes, 4))(localization_head)
classification_head = Reshape(
(num_boxes, num_classes))(classification_head)
classification_head = Activation('softmax')(classification_head)
predictions = Merge(localization_head,
classification_head,
prior_boxes_head,
mode='concat',
concat_axis=2)
model = Model(input_tensor, predictions)
return model
def mini_SSD300(input_shape=(300, 300, 3), num_classes=21):
net = {}
# Block 1
input_tensor = input_tensor = Input(shape=input_shape)
img_size = (input_shape[1], input_shape[0])
net['input'] = input_tensor
net['conv1_1'] = Convolution2D(64, 3, 3,
activation='relu',
border_mode='same',
name='conv1_1')(net['input'])
net['conv1_2'] = Convolution2D(64, 3, 3,
activation='relu',
border_mode='same',
name='conv1_2')(net['conv1_1'])
net['pool1'] = MaxPooling2D((2, 2), strides=(2, 2), border_mode='same',
name='pool1')(net['conv1_2'])
# Block 2
net['conv2_1'] = Convolution2D(128, 3, 3,
activation='relu',
border_mode='same',
name='conv2_1')(net['pool1'])
net['conv2_2'] = Convolution2D(128, 3, 3,
activation='relu',
border_mode='same',
name='conv2_2')(net['conv2_1'])
net['pool2'] = MaxPooling2D((2, 2), strides=(2, 2), border_mode='same',
name='pool2')(net['conv2_2'])
# Block 3
net['conv3_1'] = Convolution2D(256, 3, 3,
activation='relu',
border_mode='same',
name='conv3_1')(net['pool2'])
net['conv3_2'] = Convolution2D(256, 3, 3,
activation='relu',
border_mode='same',
name='conv3_2')(net['conv3_1'])
net['conv3_3'] = Convolution2D(256, 3, 3,
activation='relu',
border_mode='same',
name='conv3_3')(net['conv3_2'])
net['pool3'] = MaxPooling2D((2, 2), strides=(2, 2), border_mode='same',
name='pool3')(net['conv3_3'])
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm')(net['conv3_3'])
num_priors = 6
x = Convolution2D(num_priors * 4, 3, 3, border_mode='same',
name='conv4_3_norm_mbox_loc')(net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc'] = x
flatten = Flatten(name='conv4_3_norm_mbox_loc_flat')
net['conv4_3_norm_mbox_loc_flat'] = flatten(net['conv4_3_norm_mbox_loc'])
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes, 3, 3, border_mode='same',
name=name)(net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf'] = x
flatten = Flatten(name='conv4_3_norm_mbox_conf_flat')
net['conv4_3_norm_mbox_conf_flat'] = flatten(net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size, 30.0, max_size=60, aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv4_3_norm_mbox_priorbox')
net['conv4_3_norm_mbox_priorbox'] = priorbox(net['conv4_3_norm'])
# Prediction from fc7
if hasattr(net['conv4_3_norm_mbox_loc_flat'], '_keras_shape'):
num_boxes = net['conv4_3_norm_mbox_loc_flat']._keras_shape[-1] // 4
elif hasattr(net['mbox_loc'], 'int_shape'):
num_boxes = K.int_shape(net['conv4_3_norm_mbox_loc_flat'])[-1] // 4
net['mbox_loc'] = Reshape((num_boxes, 4),
name='mbox_loc_final')(net['conv4_3_norm_mbox_loc_flat'])
net['mbox_conf'] = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(net['conv4_3_norm_mbox_conf_flat'])
net['mbox_conf'] = Activation('softmax',
name='mbox_conf_final')(net['mbox_conf'])
net['predictions'] = merge([net['mbox_loc'],
net['mbox_conf'],
net['conv4_3_norm_mbox_priorbox']],
mode='concat', concat_axis=2,
name='predictions')
model = Model(net['input'], net['predictions'])
return model
