def build_model(self, data):
conv1_1 = self._conv2d(data, 64, (3, 3), name='conv1_1')
conv1_2 = self._conv2d(conv1_1, 64, (3, 3), name='conv1_2')
pool1 = self._pooling(conv1_2, (2, 2), (2, 2), name='pool1')
conv2_1 = self._conv2d(pool1, 128, (3, 3), name='conv2_1')
conv2_2 = self._conv2d(conv2_1, 128, (3, 3), name='conv2_2')
pool2 = self._pooling(conv2_2, (2, 2), (2, 2), name='pool2')
conv3_1 = self._conv2d(pool2, 256, (3, 3), name='conv3_1')
conv3_2 = self._conv2d(conv3_1, 256, (3, 3), name='conv3_2')
conv3_3 = self._conv2d(conv3_2, 256, (3, 3), name='conv3_3')
pool3 = self._pooling(conv3_3, (2, 2), (2, 2), name='pool3')
conv4_1 = self._conv2d(pool3, 512, (3, 3), name='conv4_1')
conv4_2 = self._conv2d(conv4_1, 512, (3, 3), name='conv4_2')
conv4_3 = self._conv2d(conv4_2, 512, (3, 3), name='conv4_3')
pool4 = self._pooling(conv4_3, (2, 2), (2, 2), name='pool4')
conv5_1 = self._conv2d(pool4, 512, (3, 3), name='conv5_1')
conv5_2 = self._conv2d(conv5_1, 512, (3, 3), name='conv5_2')
conv5_3 = self._conv2d(conv5_2, 512, (3, 3), name='conv5_3')
# SSD change pool5 from 2x2-s2 to 3x3-s1
pool5 = self._pooling(conv5_3, (3, 3), (1, 1), 'pool5')
fc6 = Conv2D(1024, (3, 3),
dilation_rate=(6, 6),
activation='relu',
padding='same',
kernel_initializer='glorot_uniform',
name='fc6')(pool5)
fc7 = Conv2D(1024, (1, 1),
activation='relu',
padding='same',
kernel_initializer='glorot_uniform',
name='fc7')(fc6)
# TODO: 比较 SSD 和 Textboxes 的 caffe model
conv6_1 = self._conv2d(fc7, 256, (1, 1), name='conv6_1')
conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv6_padding')(conv6_1)
conv6_2 = self._conv2d(conv6_1, 512, (3, 3), (2, 2), 'valid',
'conv6_2')
conv7_1 = self._conv2d(conv6_2, 128, (1, 1), name='conv7_1')
conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv7_padding')(conv7_1)
conv7_2 = self._conv2d(conv7_1, 256, (3, 3), (2, 2), 'valid',
'conv7_2')
conv8_1 = self._conv2d(conv7_2, 128, (1, 1), name='conv8_1')
conv8_2 = self._conv2d(conv8_1,
256, (3, 3), (1, 1),
'valid',
name='conv8_2')
conv9_1 = self._conv2d(conv8_2, 128, (1, 1), name='conv9_1')
conv9_2 = self._conv2d(conv9_1,
256, (3, 3), (1, 1),
'valid',
name='conv9_2')
def MobileNetV3(stack_fn, last_point_ch, input_shape=None, alpha=1.0, model_type='large',
minimalistic=False, include_top=True, weights='imagenet', input_tensor=None,
num_classes=1000, pooling=None, dropout_rate=0.2, **kwargs):
# Instantiate the MobileNetV3 architecture.
"""
# Arguments
stack_fn: a function that returns output tensor for the stacked residual blocks.
last_point_ch: number channels at the last layer (before top)
input_shape: optional tuple such as (224,224,3) or infer input_shape from
an input_tensor; If selecting include both input_tensor and input_shape,
input_shape will be used if matching or throwing an error.
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases filters # in each layer.
- If `alpha` > 1.0, proportionally increases filters # in each layer.
- If `alpha` = 1, default filters # from the paper used at each layer.
model_type: MobileNetV3 is defined as two models: large and small. .
minimalistic: Also contains the minimalistic models.
include_top: whether to include the FC layer at the top of the network.
weights: `None` (random initialization), 'imagenet' or the path to any weights.
input_tensor: optional Keras tensor (output of `layers.Input()`)
pooling: Optional mode for feature extraction when `include_top` is `False`.
- `None`: the output of model is the 4D tensor of the last conv layer
- `avg` means global average pooling and the output as a 2D tensor.
- `max` means global max pooling will be applied.
num_classes: specified if `include_top` is True
num_classes: specified if `include_top` is True
dropout_rate: fraction of the input units to drop on the last layer
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights` or invalid input shape.
RuntimeError: run the model with a backend without support separable conv
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and num_classes != 1000:
raise ValueError('If using `weights` as `"imagenet"` with `include_top` '
'as true, `num_classes` should be 1000')
# Determine the proper input shape/default size if both input_shape and input_tensor
# are used while being matched.
if input_shape is not None and input_tensor is not None:
try:
is_input_t_tensor = K.is_keras_tensor(input_tensor)
except ValueError:
try:
is_input_t_tensor = K.is_keras_tensor(
get_source_inputs(input_tensor))
except ValueError:
raise ValueError('input_tensor: ', input_tensor,
'is not type input_tensor')
if is_input_t_tensor:
if K.image_data_format == 'channels_last':
if K.int_shape(input_tensor)[2] != input_shape[1]:
raise ValueError('input_shape: ', input_shape,
'and input_tensor: ', input_tensor,
'do not meet the same shape requirements')
else:
if K.int_shape(input_tensor)[1] != input_shape[1]:
raise ValueError('input_shape: ', input_shape,
'and input_tensor: ', input_tensor,
'do not meet the same shape requirements')
else:
raise ValueError('input_tensor specified: ', input_tensor,
'is not a keras tensor')
# Infer the shape from input_tensor if input_shape as None
if input_shape is None and input_tensor is not None:
try:
K.is_keras_tensor(input_tensor)
except ValueError:
raise ValueError('input_tensor: ', input_tensor,
'is type: ', type(input_tensor),
'which is not a valid type')
if K.is_keras_tensor(input_tensor):
if K.image_data_format() == 'channels_last':
rows = K.int_shape(input_tensor)[1]
cols = K.int_shape(input_tensor)[2]
input_shape = (cols, rows, 3)
else:
rows = K.int_shape(input_tensor)[2]
cols = K.int_shape(input_tensor)[3]
input_shape = (3, cols, rows)
# If input_shape is None and input_tensor is None using standart shape
if input_shape is None and input_tensor is None:
input_shape = (None, None, 3)
if K.image_data_format() == 'channels_last':
row_axis, col_axis = (0, 1)
else:
row_axis, col_axis = (1, 2)
rows = input_shape[row_axis]
cols = input_shape[col_axis]
if rows and cols and (rows<32 or cols<32):
raise ValueError('Input size must be at least 32x32; got `input_shape=' +
str(input_shape) + '`')
if weights == 'imagenet':
if minimalistic is False and alpha not in [0.75, 1.0] \
or minimalistic is True and alpha != 1.0:
raise ValueError('If imagenet weights are being loaded, '
'alpha can be one of `0.75`, `1.0` for non minimalistic'
' or `1.0` for minimalistic only.')
if rows != cols or rows != 224:
warnings.warn('`input_shape` is undefined or non-square, '
'or `rows` is not 224.'
' Weights for input shape (224,224) will be'
' loaded as the default.')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
channel_axis = -1 if K.image_data_format() == 'channels_last' else 1
if minimalistic:
kernel = 3
activation = relu
se_ratio = None
else:
kernel = 5
activation = hard_swish
se_ratio = 0.25
x = ZeroPadding2D(padding=correct_pad(K,img_input,3), name='Conv_pad')(img_input)
x = Conv2D(16, kernel_size=3, strides=(2,2), padding='valid', use_bias=False, name='Conv')(x)
x = BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name='Conv/BatchNorm')(x)
x = Activation(activation)(x)
x = stack_fn(x, kernel, activation, se_ratio)
last_conv_ch = _depth(K.int_shape(x)[channel_axis]*6)
# Increase the output channels # if the width multiplier > 1.0
if alpha > 1.0: # No alpha applied to last conv as stated in the paper
last_point_ch = _depth(last_point_ch * alpha)
x = Conv2D(last_conv_ch, kernel_size=1, padding='same', use_bias=False, name='Conv_1')(x)
x = BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name='Conv_1/BatchNorm')(x)
x = Activation(activation)(x)
if include_top:
x = GlobalAveragePooling2D()(x)
if channel_axis == 1:
x = Reshape((last_conv_ch,1,1))(x)
else:
x = Reshape((1,1,last_conv_ch))(x)
x = Conv2D(last_point_ch, kernel_size=1, padding='same', name='Conv_2')(x)
x = Activation(activation)(x)
if dropout_rate > 0:
x = Dropout(dropout_rate)(x)
x = Conv2D(num_classes, kernel_size=1, padding='same', name='Logits')(x)
x = Flatten()(x)
x = Softmax(name='Predictions/Softmax')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D(name='avg_pool')(x)
elif pooling == 'max':
x = GlobalMaxPooling2D(name='max_pool')(x)
# Ensure the model considers any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Build the model.
model = Model(inputs, x, name='MobilenetV3'+model_type)
# Load the weights.
if weights == 'imagenet':
model_name = "{}{}_224_{}_float".format(
model_type, '_minimalistic' if minimalistic else '', str(alpha))
if include_top:
file_name = 'weights_mobilenet_v3_' + model_name + '.h5'
file_hash = WEIGHTS_HASHES[model_name][0]
else:
file_name = 'weights_mobilenet_v3_' + model_name + '_no_top.h5'
file_hash = WEIGHTS_HASHES[model_name][1]
weights_path = get_file(file_name, BASE_WEIGHT_PATH+file_name,
cache_subdir='models', file_hash=file_hash)
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def ssd_300(input_image,
classes,
mode='training',
l2_regularization=0.0005,
min_scale=None,
max_scale=None,
scales=None,
aspect_ratios_global=None,
aspect_ratios_per_layers=[[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5], [1.0, 2.0, 0.5]],
two_anchor_box=True,
steps=[8, 16, 32, 64, 100, 300],
offset=None,
clip_boxes=False,
variance=[0.1, 0.1, 0.2, 0.2],
coords='centroids',
normalize_coords=True,
subtract_mean=[123, 117, 104],
divides_by_stddev=None,
swap_channels=[2, 1, 0],
confidence_thresh=0.01,
iou_threshold=0.45,
top_k=200,
nms_max_output_size=400,
return_predictor_sizes=False):
n_pred_layers = 6
classes += 1
l2_reg = l2_regularization
img_h, img_w, img_ch = input_image[0], input_image[1], input_image[2]
############################################################################
# Compute the anchor box parameters.
############################################################################
# Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers.
if aspect_ratios_per_layers:
aspect_ratios = aspect_ratios_per_layers
else:
aspect_ratios = [aspect_ratios_global] * n_pred_layers
if aspect_ratios_per_layers:
n_boxes = []
for ar in aspect_ratios_per_layers:
if (1 in ar) & two_anchor_box:
n_boxes.append(len(ar) + 1)
else:
n_boxes.append(len(ar))
else:
if (1 in aspect_ratios_global) & two_anchor_box:
n_boxes = len(aspect_ratios_global) + 1
else:
n_boxes = len(aspect_ratios_global)
n_boxes = [n_boxes] * n_pred_layers
if steps is steps is None:
steps = [None] * n_pred_layers
if offset is None:
offset = [None] * n_pred_layers
############################################################################
# Define functions for the Lambda layers below.
############################################################################
def identify_layer(tensor):
return tensor
def input_mean_normalization(tensor):
return tensor - np.array(subtract_mean)
def input_stddev_normalization(tensor):
return tensor / np.array(divide_by_stddev)
def input_channel_swap(tensor):
if len(swap_channels) == 3:
return K.stack([
tensor[..., swap_channels[0]], tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]]
],
axis=-1)
elif len(swap_channels) == 4:
return K.stack([
tensor[..., swap_channels[0]], tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]], tensor[..., swap_channels[3]]
],
axis=-1)
############################################################################
# Build the network.
############################################################################
x = Input(shape=(img_h, img_w, img_ch))
'''
x1 = Lambda(identity_layer, output_shape=(img_h, img_w, img_ch), name='identity_layer')(x)
if not (subtract_mean is None):
x1 = Lambda(input_mean_normalization, output_shape=(img_h, img_w, img_ch), name='input_mean_normalization')(x1)
if not (divide_by_stddev is None):
x1 = Lambda(input_stddev_normalization, output_shape=(img_h, img_w, img_ch), name='input_stddev_normalization')(x1)
if swap_channels:
x1 = Lambda(input_channel_swap, output_shape=(img_h, img_w, img_ch), name='input_channel_swap')(x1)
'''
conv1_1 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv1_1')(x)
conv1_2 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv1_2')(conv1_1)
pool1 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool1')(conv1_2)
conv2_1 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv2_1')(pool1)
conv2_2 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv2_2')(conv2_1)
pool2 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool2')(conv2_2)
conv3_1 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_1')(pool2)
conv3_2 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_2')(conv3_1)
conv3_3 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_3')(conv3_2)
pool3 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool3')(conv3_3)
conv4_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_1')(pool3)
conv4_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_2')(conv4_1)
conv4_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_3')(conv4_2)
pool4 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool4')(conv4_3)
conv5_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_1')(pool4)
conv5_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_2')(conv5_1)
conv5_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_3')(conv5_2)
pool5 = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
padding='same',
name='pool5')(conv5_3)
fc6 = Conv2D(1024, (3, 3),
dilation_rate=(6, 6),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc6')(pool5)
fc7 = Conv2D(1024, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc7')(fc6)
conv6_1 = Conv2D(256, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_1')(fc7)
conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv6_padding')(conv6_1)
conv6_2 = Conv2D(512, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_2')(conv6_1)
conv7_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_1')(conv6_2)
conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv7_padding')(conv7_1)
conv7_2 = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_2')(conv7_1)
conv8_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_1')(conv7_2)
conv8_2 = Conv2D(256, (3, 3),
strides=(1, 1),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_2')(conv8_1)
conv9_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_1')(conv8_2)
conv9_2 = Conv2D(256, (3, 3),
strides=(1, 1),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_2')(conv9_1)
#Yashwant Chouhan Date 25-6-2019
# Feed conv4_3 into the L2 normalization layer
conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3)
### Build the convolutional predictor layers on top of the base network
# We precidt `classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * classes`
# Output shape of the confidence layers: `(batch, height, width, n_boxes * classes)`
conv4_3_norm_mbox_conf = Conv2D(
n_boxes[0] * classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_3_norm_mbox_conf')(conv4_3_norm)
fc7_mbox_conf = Conv2D(n_boxes[1] * classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc7_mbox_conf')(fc7)
conv6_2_mbox_conf = Conv2D(n_boxes[2] * classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_2_mbox_conf')(conv6_2)
conv7_2_mbox_conf = Conv2D(n_boxes[3] * classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_2_mbox_conf')(conv7_2)
conv8_2_mbox_conf = Conv2D(n_boxes[4] * classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_2_mbox_conf')(conv8_2)
conv9_2_mbox_conf = Conv2D(n_boxes[5] * classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_2_mbox_conf')(conv9_2)
# We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4`
# Output shape of the localization layers: `(batch, height, width, n_boxes * 4)`
conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_3_norm_mbox_loc')(conv4_3_norm)
fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc7_mbox_loc')(fc7)
conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_2_mbox_loc')(conv6_2)
conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_2_mbox_loc')(conv7_2)
conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_2_mbox_loc')(conv8_2)
conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_2_mbox_loc')(conv9_2)
### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names)
# Output shape of anchors: `(batch, height, width, n_boxes, 8)`
conv4_3_norm_mbox_priorbox = AnchorBoxes(
img_h,
img_w,
this_scale=scales[0],
next_scale=scales[1],
aspect_ratios=aspect_ratios[0],
two_anchor_box=two_anchor_box,
this_steps=steps[0],
this_offsets=offsets[0],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc)
fc7_mbox_priorbox = AnchorBoxes(img_h,
img_w,
this_scale=scales[1],
next_scale=scales[2],
aspect_ratios=aspect_ratios[1],
two_anchor_box=two_anchor_box,
this_steps=steps[1],
this_offsets=offsets[1],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='fc7_mbox_priorbox')(fc7_mbox_loc)
conv6_2_mbox_priorbox = AnchorBoxes(
img_h,
img_w,
this_scale=scales[2],
next_scale=scales[3],
aspect_ratios=aspect_ratios[2],
two_anchor_box=two_anchor_box,
this_steps=steps[2],
this_offsets=offsets[2],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc)
conv7_2_mbox_priorbox = AnchorBoxes(
img_h,
img_w,
this_scale=scales[3],
next_scale=scales[4],
aspect_ratios=aspect_ratios[3],
two_anchor_box=two_anchor_box,
this_steps=steps[3],
this_offsets=offsets[3],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc)
conv8_2_mbox_priorbox = AnchorBoxes(
img_h,
img_w,
this_scale=scales[4],
next_scale=scales[5],
aspect_ratios=aspect_ratios[4],
two_anchor_box=two_anchor_box,
this_steps=steps[4],
this_offsets=offsets[4],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc)
conv9_2_mbox_priorbox = AnchorBoxes(
img_h,
img_w,
this_scale=scales[5],
next_scale=scales[6],
aspect_ratios=aspect_ratios[5],
two_anchor_box=two_anchor_box,
this_steps=steps[5],
this_offsets=offsets[5],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc)
### Reshape
# Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, classes)`
# We want the classes isolated in the last axis to perform softmax on them
conv4_3_norm_mbox_conf_reshape = Reshape(
(-1, classes),
name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf)
fc7_mbox_conf_reshape = Reshape(
(-1, classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf)
conv6_2_mbox_conf_reshape = Reshape(
(-1, classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf)
conv7_2_mbox_conf_reshape = Reshape(
(-1, classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf)
conv8_2_mbox_conf_reshape = Reshape(
(-1, classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf)
conv9_2_mbox_conf_reshape = Reshape(
(-1, classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf)
# Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)`
# We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
conv4_3_norm_mbox_loc_reshape = Reshape(
(-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc)
fc7_mbox_loc_reshape = Reshape((-1, 4),
name='fc7_mbox_loc_reshape')(fc7_mbox_loc)
conv6_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc)
conv7_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc)
conv8_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc)
conv9_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc)
# Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)`
conv4_3_norm_mbox_priorbox_reshape = Reshape(
(-1, 8),
name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox)
fc7_mbox_priorbox_reshape = Reshape(
(-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox)
conv6_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox)
conv7_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox)
conv8_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox)
conv9_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox)
### Concatenate the predictions from the different layers
# Axis 0 (batch) and axis 2 (classes or 4, respectively) are identical for all layer predictions,
# so we want to concatenate along axis 1, the number of boxes per layer
# Output shape of `mbox_conf`: (batch, n_boxes_total, classes)
mbox_conf = Concatenate(axis=1, name='mbox_conf')([
conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape,
conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape,
conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape
])
# Output shape of `mbox_loc`: (batch, n_boxes_total, 4)
mbox_loc = Concatenate(axis=1, name='mbox_loc')([
conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape,
conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape,
conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape
])
# Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8)
mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([
conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape,
conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape,
conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape
])
# The box coordinate predictions will go into the loss function just the way they are,
# but for the class predictions, we'll apply a softmax activation layer first
mbox_conf_softmax = Activation('softmax',
name='mbox_conf_softmax')(mbox_conf)
# Concatenate the class and box predictions and the anchors to one large predictions vector
# Output shape of `predictions`: (batch, n_boxes_total, classes + 4 + 8)
predictions = Concatenate(axis=2, name='predictions')(
[mbox_conf_softmax, mbox_loc, mbox_priorbox])
if mode == 'training':
model = Model(inputs=x, outputs=predictions)
elif mode == 'inference':
decoded_predictions = DecodeDetections(
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_h=img_h,
img_w=img_w,
name='decoded_predictions')(predictions)
model = Model(inputs=x, outputs=decoded_predictions)
elif mode == 'inference_fast':
decoded_predictions = DecodeDetectionsFast(
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_h=img_h,
img_w=img_w,
name='decoded_predictions')(predictions)
model = Model(inputs=x, outputs=decoded_predictions)
else:
raise ValueError(
"`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'."
.format(mode))
if return_predictor_sizes:
predictor_sizes = np.array([
conv4_3_norm_mbox_conf._keras_shape[1:3],
fc7_mbox_conf._keras_shape[1:3],
conv6_2_mbox_conf._keras_shape[1:3],
conv7_2_mbox_conf._keras_shape[1:3],
conv8_2_mbox_conf._keras_shape[1:3],
conv9_2_mbox_conf._keras_shape[1:3]
])
return model, predictor_sizes
else:
return model
img_data /= 255.0
img_data = np.expand_dims(img_data, axis=4)
Y = np_utils.to_categorical(labels, num_classes)
x, y = shuffle(img_data, Y, random_state=2)
X_train, X_test, y_train, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=2)
input_shape = img_data[0].shape
print(input_shape)
model.add(ZeroPadding2D((1, 1), input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu', kernel_constraint=maxnorm(3)))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(64, (3, 3), activation='relu', kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(128, (3, 3), activation='relu', kernel_constraint=maxnorm(3)))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(128, (3, 3), activation='relu', kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
def get_resnet(img_rows=224, img_cols=224, separately=False):
"""
Resnet 152 Model for Keras
Model Schema and layer naming follow that of the original Caffe implementation
https://github.com/KaimingHe/deep-residual-networks
ImageNet Pretrained Weights
Theano: https://drive.google.com/file/d/0Byy2AcGyEVxfZHhUT3lWVWxRN28/view?usp=sharing
TensorFlow: https://drive.google.com/file/d/0Byy2AcGyEVxfeXExMzNNOHpEODg/view?usp=sharing
Parameters:
img_rows, img_cols - resolution of inputs
channel - 1 for grayscale, 3 for color
"""
img_input = Input(shape=(img_rows, img_cols, 3), name='data')
eps = 1.1e-5
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Convolution2D(64, (7, 7), strides=(2, 2), name='conv1',
use_bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=3, name='bn_conv1')(x)
x = Scale(axis=3, name='scale_conv1')(x)
x = Activation('relu', name='conv1_relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
for i in range(1, 8):
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b' + str(i))
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
for i in range(1, 36):
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b' + str(i))
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x_fc = AveragePooling2D((7, 7), name='avg_pool')(x)
x_fc = Flatten()(x_fc)
x_fc = Dense(1000, activation='softmax', name='fc1000')(x_fc)
model = Model(img_input, x_fc)
# Use pre-trained weights for Tensorflow backend
weights_path = 'utils/weights/resnet152_weights_tf.h5'
assert (os.path.exists(weights_path))
model.load_weights(weights_path, by_name=True)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
x_newfc = AveragePooling2D((7, 7), name='avg_pool')(x)
x_newfc = Flatten()(x_newfc)
x_newfc = Dense(256, name='fc8')(x_newfc)
model = Model(img_input, x_newfc)
return model
def SSD512(input_shape, num_classes=21):
"""SSD512 architecture.
# Arguments
input_shape: Shape of the input image,
expected to be either (512, 512, 3) or (3, 512, 512)(not tested).
num_classes: Number of classes including background.
# References
https://arxiv.org/abs/1512.02325
"""
net = {}
# Block 1
input_tensor = input_tensor = Input(shape=input_shape)
img_size = (input_shape[1], input_shape[0])
net['input'] = input_tensor
net['conv1_1'] = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='conv1_1')(net['input'])
net['conv1_2'] = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='conv1_2')(net['conv1_1'])
net['pool1'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool1')(net['conv1_2'])
# Block 2
net['conv2_1'] = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='conv2_1')(net['pool1'])
net['conv2_2'] = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='conv2_2')(net['conv2_1'])
net['pool2'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool2')(net['conv2_2'])
# Block 3
net['conv3_1'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_1')(net['pool2'])
net['conv3_2'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_2')(net['conv3_1'])
net['conv3_3'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_3')(net['conv3_2'])
net['pool3'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool3')(net['conv3_3'])
# Block 4
net['conv4_1'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_1')(net['pool3'])
net['conv4_2'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_2')(net['conv4_1'])
net['conv4_3'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_3')(net['conv4_2'])
net['pool4'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool4')(net['conv4_3'])
# Block 5
net['conv5_1'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_1')(net['pool4'])
net['conv5_2'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_2')(net['conv5_1'])
net['conv5_3'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_3')(net['conv5_2'])
net['pool5'] = MaxPooling2D((3, 3),
strides=(1, 1),
border_mode='same',
name='pool5')(net['conv5_3'])
# FC6
net['fc6'] = AtrousConvolution2D(1024,
3,
3,
atrous_rate=(6, 6),
activation='relu',
border_mode='same',
name='fc6')(net['pool5'])
# x = Dropout(0.5, name='drop6')(x)
# FC7
net['fc7'] = Convolution2D(1024,
1,
1,
activation='relu',
border_mode='same',
name='fc7')(net['fc6'])
# x = Dropout(0.5, name='drop7')(x)
# Block 6
net['conv6_1'] = Convolution2D(256,
1,
1,
activation='relu',
border_mode='same',
name='conv6_1')(net['fc7'])
net['conv6_2'] = Convolution2D(512,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='same',
name='conv6_2')(net['conv6_1'])
# Block 7
net['conv7_1'] = Convolution2D(128,
1,
1,
activation='relu',
border_mode='same',
name='conv7_1')(net['conv6_2'])
net['conv7_2'] = ZeroPadding2D()(net['conv7_1'])
net['conv7_2'] = Convolution2D(256,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='valid',
name='conv7_2')(net['conv7_2'])
# Block 8
net['conv8_1'] = Convolution2D(128,
1,
1,
activation='relu',
border_mode='same',
name='conv8_1')(net['conv7_2'])
net['conv8_2'] = Convolution2D(256,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='same',
name='conv8_2')(net['conv8_1'])
# Block 9
net['conv9_1'] = Convolution2D(128,
1,
1,
activation='relu',
border_mode='same',
name='conv9_1')(net['conv8_2'])
net['conv9_2'] = Convolution2D(256,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='same',
name='conv9_2')(net['conv9_1'])
# Block 10
net['conv10_1'] = Convolution2D(128,
1,
1,
activation='relu',
border_mode='same',
name='conv10_1')(net['conv9_2'])
net['conv10_2'] = Convolution2D(256,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='same',
name='conv10_2')(net['conv10_1'])
# Last Pool
net['pool6'] = GlobalAveragePooling2D(name='pool6')(net['conv10_2'])
# Prediction from conv4_3
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm')(net['conv4_3'])
num_priors = 4
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv4_3_norm_mbox_loc')(net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc'] = x
flatten = Flatten(name='conv4_3_norm_mbox_loc_flat')
net['conv4_3_norm_mbox_loc_flat'] = flatten(net['conv4_3_norm_mbox_loc'])
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf'] = x
flatten = Flatten(name='conv4_3_norm_mbox_conf_flat')
net['conv4_3_norm_mbox_conf_flat'] = flatten(net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size,
35.84,
max_size=76.8,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv4_3_norm_mbox_priorbox')
net['conv4_3_norm_mbox_priorbox'] = priorbox(net['conv4_3_norm'])
# Prediction from fc7
num_priors = 6
net['fc7_mbox_loc'] = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='fc7_mbox_loc')(net['fc7'])
flatten = Flatten(name='fc7_mbox_loc_flat')
net['fc7_mbox_loc_flat'] = flatten(net['fc7_mbox_loc'])
name = 'fc7_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
net['fc7_mbox_conf'] = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['fc7'])
flatten = Flatten(name='fc7_mbox_conf_flat')
net['fc7_mbox_conf_flat'] = flatten(net['fc7_mbox_conf'])
priorbox = PriorBox(img_size,
76.8,
max_size=153.6,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='fc7_mbox_priorbox')
net['fc7_mbox_priorbox'] = priorbox(net['fc7'])
# Prediction from conv6_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv6_2_mbox_loc')(net['conv6_2'])
net['conv6_2_mbox_loc'] = x
flatten = Flatten(name='conv6_2_mbox_loc_flat')
net['conv6_2_mbox_loc_flat'] = flatten(net['conv6_2_mbox_loc'])
name = 'conv6_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv6_2'])
net['conv6_2_mbox_conf'] = x
flatten = Flatten(name='conv6_2_mbox_conf_flat')
net['conv6_2_mbox_conf_flat'] = flatten(net['conv6_2_mbox_conf'])
priorbox = PriorBox(img_size,
153.6,
max_size=230.4,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv6_2_mbox_priorbox')
net['conv6_2_mbox_priorbox'] = priorbox(net['conv6_2'])
# Prediction from conv7_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv7_2_mbox_loc')(net['conv7_2'])
net['conv7_2_mbox_loc'] = x
flatten = Flatten(name='conv7_2_mbox_loc_flat')
net['conv7_2_mbox_loc_flat'] = flatten(net['conv7_2_mbox_loc'])
name = 'conv7_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv7_2'])
net['conv7_2_mbox_conf'] = x
flatten = Flatten(name='conv7_2_mbox_conf_flat')
net['conv7_2_mbox_conf_flat'] = flatten(net['conv7_2_mbox_conf'])
priorbox = PriorBox(img_size,
230.4,
max_size=307.2,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv7_2_mbox_priorbox')
net['conv7_2_mbox_priorbox'] = priorbox(net['conv7_2'])
# Prediction from conv8_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv8_2_mbox_loc')(net['conv8_2'])
net['conv8_2_mbox_loc'] = x
flatten = Flatten(name='conv8_2_mbox_loc_flat')
net['conv8_2_mbox_loc_flat'] = flatten(net['conv8_2_mbox_loc'])
name = 'conv8_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv8_2'])
net['conv8_2_mbox_conf'] = x
flatten = Flatten(name='conv8_2_mbox_conf_flat')
net['conv8_2_mbox_conf_flat'] = flatten(net['conv8_2_mbox_conf'])
priorbox = PriorBox(img_size,
307.2,
max_size=384.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv8_2_mbox_priorbox')
net['conv8_2_mbox_priorbox'] = priorbox(net['conv8_2'])
# Prediction from conv9_2
num_priors = 4
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv9_2_mbox_loc')(net['conv9_2'])
net['conv9_2_mbox_loc'] = x
flatten = Flatten(name='conv9_2_mbox_loc_flat')
net['conv9_2_mbox_loc_flat'] = flatten(net['conv9_2_mbox_loc'])
name = 'conv9_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv9_2'])
net['conv9_2_mbox_conf'] = x
flatten = Flatten(name='conv9_2_mbox_conf_flat')
net['conv9_2_mbox_conf_flat'] = flatten(net['conv9_2_mbox_conf'])
priorbox = PriorBox(img_size,
384.0,
max_size=460.8,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv9_2_mbox_priorbox')
net['conv9_2_mbox_priorbox'] = priorbox(net['conv9_2'])
# Prediction from pool6
num_priors = 4
x = Dense(num_priors * 4, name='pool6_mbox_loc_flat')(net['pool6'])
net['pool6_mbox_loc_flat'] = x
name = 'pool6_mbox_conf_flat'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Dense(num_priors * num_classes, name=name)(net['pool6'])
net['pool6_mbox_conf_flat'] = x
priorbox = PriorBox(img_size,
460.8,
max_size=537.6,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='pool6_mbox_priorbox')
if K.image_dim_ordering() == 'tf':
target_shape = (1, 1, 256)
else:
target_shape = (256, 1, 1)
net['pool6_reshaped'] = Reshape(target_shape,
name='pool6_reshaped')(net['pool6'])
net['pool6_mbox_priorbox'] = priorbox(net['pool6_reshaped'])
# Gather all predictions
net['mbox_loc'] = merge([
net['conv4_3_norm_mbox_loc_flat'], net['fc7_mbox_loc_flat'],
net['conv6_2_mbox_loc_flat'], net['conv7_2_mbox_loc_flat'],
net['conv8_2_mbox_loc_flat'], net['conv9_2_mbox_loc_flat'],
net['pool6_mbox_loc_flat']
],
mode='concat',
concat_axis=1,
name='mbox_loc')
net['mbox_conf'] = merge([
net['conv4_3_norm_mbox_conf_flat'], net['fc7_mbox_conf_flat'],
net['conv6_2_mbox_conf_flat'], net['conv7_2_mbox_conf_flat'],
net['conv8_2_mbox_conf_flat'], net['conv9_2_mbox_conf_flat'],
net['pool6_mbox_conf_flat']
],
mode='concat',
concat_axis=1,
name='mbox_conf')
net['mbox_priorbox'] = merge([
net['conv4_3_norm_mbox_priorbox'], net['fc7_mbox_priorbox'],
net['conv6_2_mbox_priorbox'], net['conv7_2_mbox_priorbox'],
net['conv8_2_mbox_priorbox'], net['conv9_2_mbox_priorbox'],
net['pool6_mbox_priorbox']
],
mode='concat',
concat_axis=1,
name='mbox_priorbox')
if hasattr(net['mbox_loc'], '_keras_shape'):
num_boxes = net['mbox_loc']._keras_shape[-1] // 4
elif hasattr(net['mbox_loc'], 'int_shape'):
num_boxes = K.int_shape(net['mbox_loc'])[-1] // 4
net['mbox_loc'] = Reshape((num_boxes, 4),
name='mbox_loc_final')(net['mbox_loc'])
net['mbox_conf'] = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(net['mbox_conf'])
net['mbox_conf'] = Activation('softmax',
name='mbox_conf_final')(net['mbox_conf'])
net['predictions'] = merge(
[net['mbox_loc'], net['mbox_conf'], net['mbox_priorbox']],
mode='concat',
concat_axis=2,
name='predictions')
model = Model(net['input'], net['predictions'])
return model
"""SSD300 architecture.
# Arguments
input_shape: Shape of the input image,
expected to be either (300, 300, 3) or (3, 300, 300)(not tested).
num_classes: Number of classes including background.
# References
https://arxiv.org/abs/1512.02325
"""
net = {}
# Block 1
input_tensor = input_tensor = Input(shape=input_shape)
img_size = (input_shape[1], input_shape[0])
net['input'] = input_tensor
net['conv1_1'] = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='conv1_1')(net['input'])
net['conv1_2'] = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='conv1_2')(net['conv1_1'])
net['pool1'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool1')(net['conv1_2'])
# Block 2
net['conv2_1'] = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='conv2_1')(net['pool1'])
net['conv2_2'] = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='conv2_2')(net['conv2_1'])
net['pool2'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool2')(net['conv2_2'])
# Block 3
net['conv3_1'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_1')(net['pool2'])
net['conv3_2'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_2')(net['conv3_1'])
net['conv3_3'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_3')(net['conv3_2'])
net['pool3'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool3')(net['conv3_3'])
# Block 4
net['conv4_1'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_1')(net['pool3'])
net['conv4_2'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_2')(net['conv4_1'])
net['conv4_3'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_3')(net['conv4_2'])
net['pool4'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool4')(net['conv4_3'])
# Block 5
net['conv5_1'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_1')(net['pool4'])
net['conv5_2'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_2')(net['conv5_1'])
net['conv5_3'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_3')(net['conv5_2'])
net['pool5'] = MaxPooling2D((3, 3),
strides=(1, 1),
border_mode='same',
name='pool5')(net['conv5_3'])
# FC6
net['fc6'] = AtrousConvolution2D(1024,
3,
3,
atrous_rate=(6, 6),
activation='relu',
border_mode='same',
name='fc6')(net['pool5'])
# x = Dropout(0.5, name='drop6')(x)
# FC7
net['fc7'] = Convolution2D(1024,
1,
1,
activation='relu',
border_mode='same',
name='fc7')(net['fc6'])
# x = Dropout(0.5, name='drop7')(x)
# Block 6
net['conv6_1'] = Convolution2D(256,
1,
1,
activation='relu',
border_mode='same',
name='conv6_1')(net['fc7'])
net['conv6_2'] = Convolution2D(512,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='same',
name='conv6_2')(net['conv6_1'])
# Block 7
net['conv7_1'] = Convolution2D(128,
1,
1,
activation='relu',
border_mode='same',
name='conv7_1')(net['conv6_2'])
net['conv7_2'] = ZeroPadding2D()(net['conv7_1'])
net['conv7_2'] = Convolution2D(256,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='valid',
name='conv7_2')(net['conv7_2'])
# Block 8
net['conv8_1'] = Convolution2D(128,
1,
1,
activation='relu',
border_mode='same',
name='conv8_1')(net['conv7_2'])
net['conv8_2'] = Convolution2D(256,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='same',
name='conv8_2')(net['conv8_1'])
# Last Pool
net['pool6'] = GlobalAveragePooling2D(name='pool6')(net['conv8_2'])
# Prediction from conv4_3
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm')(net['conv4_3'])
num_priors = 3
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv4_3_norm_mbox_loc')(net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc'] = x
flatten = Flatten(name='conv4_3_norm_mbox_loc_flat')
net['conv4_3_norm_mbox_loc_flat'] = flatten(net['conv4_3_norm_mbox_loc'])
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf'] = x
flatten = Flatten(name='conv4_3_norm_mbox_conf_flat')
net['conv4_3_norm_mbox_conf_flat'] = flatten(net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size,
30.0,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv4_3_norm_mbox_priorbox')
net['conv4_3_norm_mbox_priorbox'] = priorbox(net['conv4_3_norm'])
# Prediction from fc7
num_priors = 6
net['fc7_mbox_loc'] = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='fc7_mbox_loc')(net['fc7'])
flatten = Flatten(name='fc7_mbox_loc_flat')
net['fc7_mbox_loc_flat'] = flatten(net['fc7_mbox_loc'])
name = 'fc7_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
net['fc7_mbox_conf'] = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['fc7'])
flatten = Flatten(name='fc7_mbox_conf_flat')
net['fc7_mbox_conf_flat'] = flatten(net['fc7_mbox_conf'])
priorbox = PriorBox(img_size,
60.0,
max_size=114.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='fc7_mbox_priorbox')
net['fc7_mbox_priorbox'] = priorbox(net['fc7'])
# Prediction from conv6_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv6_2_mbox_loc')(net['conv6_2'])
net['conv6_2_mbox_loc'] = x
flatten = Flatten(name='conv6_2_mbox_loc_flat')
net['conv6_2_mbox_loc_flat'] = flatten(net['conv6_2_mbox_loc'])
name = 'conv6_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv6_2'])
net['conv6_2_mbox_conf'] = x
flatten = Flatten(name='conv6_2_mbox_conf_flat')
net['conv6_2_mbox_conf_flat'] = flatten(net['conv6_2_mbox_conf'])
priorbox = PriorBox(img_size,
114.0,
max_size=168.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv6_2_mbox_priorbox')
net['conv6_2_mbox_priorbox'] = priorbox(net['conv6_2'])
# Prediction from conv7_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv7_2_mbox_loc')(net['conv7_2'])
net['conv7_2_mbox_loc'] = x
flatten = Flatten(name='conv7_2_mbox_loc_flat')
net['conv7_2_mbox_loc_flat'] = flatten(net['conv7_2_mbox_loc'])
name = 'conv7_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv7_2'])
net['conv7_2_mbox_conf'] = x
flatten = Flatten(name='conv7_2_mbox_conf_flat')
net['conv7_2_mbox_conf_flat'] = flatten(net['conv7_2_mbox_conf'])
priorbox = PriorBox(img_size,
168.0,
max_size=222.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv7_2_mbox_priorbox')
net['conv7_2_mbox_priorbox'] = priorbox(net['conv7_2'])
# Prediction from conv8_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv8_2_mbox_loc')(net['conv8_2'])
net['conv8_2_mbox_loc'] = x
flatten = Flatten(name='conv8_2_mbox_loc_flat')
net['conv8_2_mbox_loc_flat'] = flatten(net['conv8_2_mbox_loc'])
name = 'conv8_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv8_2'])
net['conv8_2_mbox_conf'] = x
flatten = Flatten(name='conv8_2_mbox_conf_flat')
net['conv8_2_mbox_conf_flat'] = flatten(net['conv8_2_mbox_conf'])
priorbox = PriorBox(img_size,
222.0,
max_size=276.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv8_2_mbox_priorbox')
net['conv8_2_mbox_priorbox'] = priorbox(net['conv8_2'])
# Prediction from pool6
num_priors = 6
x = Dense(num_priors * 4, name='pool6_mbox_loc_flat')(net['pool6'])
net['pool6_mbox_loc_flat'] = x
name = 'pool6_mbox_conf_flat'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Dense(num_priors * num_classes, name=name)(net['pool6'])
net['pool6_mbox_conf_flat'] = x
priorbox = PriorBox(img_size,
276.0,
max_size=330.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='pool6_mbox_priorbox')
if K.image_dim_ordering() == 'tf':
target_shape = (1, 1, 256)
else:
target_shape = (256, 1, 1)
net['pool6_reshaped'] = Reshape(target_shape,
name='pool6_reshaped')(net['pool6'])
net['pool6_mbox_priorbox'] = priorbox(net['pool6_reshaped'])
# Gather all predictions
net['mbox_loc'] = merge([
net['conv4_3_norm_mbox_loc_flat'], net['fc7_mbox_loc_flat'],
net['conv6_2_mbox_loc_flat'], net['conv7_2_mbox_loc_flat'],
net['conv8_2_mbox_loc_flat'], net['pool6_mbox_loc_flat']
],
mode='concat',
concat_axis=1,
name='mbox_loc')
net['mbox_conf'] = merge([
net['conv4_3_norm_mbox_conf_flat'], net['fc7_mbox_conf_flat'],
net['conv6_2_mbox_conf_flat'], net['conv7_2_mbox_conf_flat'],
net['conv8_2_mbox_conf_flat'], net['pool6_mbox_conf_flat']
],
mode='concat',
concat_axis=1,
name='mbox_conf')
net['mbox_priorbox'] = merge([
net['conv4_3_norm_mbox_priorbox'], net['fc7_mbox_priorbox'],
net['conv6_2_mbox_priorbox'], net['conv7_2_mbox_priorbox'],
net['conv8_2_mbox_priorbox'], net['pool6_mbox_priorbox']
],
mode='concat',
concat_axis=1,
name='mbox_priorbox')
if hasattr(net['mbox_loc'], '_keras_shape'):
num_boxes = net['mbox_loc']._keras_shape[-1] // 4
elif hasattr(net['mbox_loc'], 'int_shape'):
num_boxes = K.int_shape(net['mbox_loc'])[-1] // 4
net['mbox_loc'] = Reshape((num_boxes, 4),
name='mbox_loc_final')(net['mbox_loc'])
net['mbox_conf'] = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(net['mbox_conf'])
net['mbox_conf'] = Activation('softmax',
name='mbox_conf_final')(net['mbox_conf'])
net['predictions'] = merge(
[net['mbox_loc'], net['mbox_conf'], net['mbox_priorbox']],
mode='concat',
concat_axis=2,
name='predictions')
model = Model(net['input'], net['predictions'])
return model
def DenseNet(nb_dense_block=4, growth_rate=32, nb_filter=64, reduction=0.0, dropout_rate=0.0, weight_decay=1e-4,
classes=1000, weights_path=None):
'''Instantiate the DenseNet 121 architecture,
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.common.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(224, 224, 3), name='data')
else:
concat_axis = 1
img_input = Input(shape=(3, 224, 224), name='data')
# From architecture for ImageNet (Table 1 in the paper)
# nb_filter = 64
nb_layers = [6, 12, 24, 16] # For DenseNet-121
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Convolution2D(nb_filter, 7, 7, subsample=(2, 2), name='conv1', bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x, stage, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x, stage, nb_filter, compression=compression, dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x, final_stage, nb_layers[-1], nb_filter, growth_rate, dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis, name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis, name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x = Dense(classes, name='fc6')(x)
x = Activation('softmax', name='prob')(x)
model = Model(img_input, x, name='densenet')
if weights_path is not None:
model.load_weights(weights_path)
return model
def inception_net(input_img,
t0_f0=64,
t1_f0=96,
t1_f1=128,
t2_f0=16,
t2_f1=32,
t3_f1=32):
'''This is the base building block of the inception net'''
tower_0 = Conv2D(
t0_f0,
(1, 1), #padding='same',
use_bias=True,
activation='relu',
kernel_initializer='glorot_normal',
kernel_regularizer=regularizers.l2(0.01))(input_img)
tower_1 = Conv2D(
t1_f0,
(1, 1), #padding='same',
use_bias=True,
activation='relu',
kernel_initializer='glorot_normal',
kernel_regularizer=regularizers.l2(0.01))(input_img)
tower_1 = ZeroPadding2D((1, 1))(tower_1)
tower_1 = Conv2D(
t1_f1,
(3, 3), #padding='same',
use_bias=True,
activation='relu',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.04),
kernel_regularizer=regularizers.l2(0.01))(tower_1)
tower_2 = Conv2D(
t2_f0,
(1, 1), #padding='same',
use_bias=True,
activation='relu',
kernel_initializer='glorot_normal',
kernel_regularizer=regularizers.l2(0.01))(input_img)
tower_2 = ZeroPadding2D((2, 2))(tower_2)
tower_2 = Conv2D(
t2_f1,
(5, 5), #padding='same',
use_bias=True,
activation='relu',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.08),
kernel_regularizer=regularizers.l2(0.01))(tower_2)
tower_3 = ZeroPadding2D((1, 1))(input_img)
tower_3 = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1), #padding='same'
)(tower_3)
tower_3 = Conv2D(
t3_f1,
(1, 1), #padding='same',
use_bias=True,
activation='relu',
kernel_initializer='glorot_normal',
kernel_regularizer=regularizers.l2(0.01))(tower_3)
if (K.image_dim_ordering == 'th'):
bn_axis = 1
else:
bn_axis = 3
output = layers.concatenate([tower_0, tower_1, tower_2, tower_3],
axis=bn_axis)
return (output)
def build_model():
model = Sequential()
# Layer 1
model.add(
ZeroPadding2D(padding=(2, 2), input_shape=(
1, img_rows,
img_cols))) #theano (channels,w,h) ,tensorflow (w,h,channels)
model.add(Convolution2D(64, 5, 5))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
print("Layer 1: " + str(model.layers[-1].output_shape))
# Layer 2
model.add(ZeroPadding2D(padding=(2, 2)))
model.add(Convolution2D(128, 5, 5))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
print("Layer 2: " + str(model.layers[-1].output_shape))
# Layer 3
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(Convolution2D(128, 3, 3))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
print("Layer 3: " + str(model.layers[-1].output_shape))
# Layer 4
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(Convolution2D(128, 3, 3))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
print("Layer 4: " + str(model.layers[-1].output_shape))
# Layer 5
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(Convolution2D(256, 3, 3))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
print("Layer 5: " + str(model.layers[-1].output_shape))
# Layer 6
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(Convolution2D(256, 3, 3))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
print("Layer 6: " + str(model.layers[-1].output_shape))
# Layer 7
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(Convolution2D(512, 3, 3))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
print("Layer 7: " + str(model.layers[-1].output_shape))
# Layer 8
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(Convolution2D(512, 3, 3))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
print("Layer 4: " + str(model.layers[-1].output_shape))
# Layer 9
model.add(ZeroPadding2D(padding=(2, 2)))
model.add(Convolution2D(512, 5, 5))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
print("Layer 9: " + str(model.layers[-1].output_shape))
# Layer 10
model.add(ZeroPadding2D(padding=(2, 2)))
model.add(Convolution2D(7, 5, 5))
model.add(BatchNormalization(axis=1))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
print("Layer 10: " + str(model.layers[-1].output_shape))
#sgd = SGD(lr = 10e-4, decay = 5e-4, momentum = 0.9, nesterov = False)
sgd = SGD(lr=10e-4, decay=5e-4, momentum=0.9, nesterov=False)
model.compile(loss=fcrn_loss, optimizer=sgd, metrics=['accuracy'])
#model.compile(loss='mean_squared_error', optimizer = sgd, metrics = ['accuracy'])
return model
def ResNet50(input_shape = (64, 64, 3), classes = 6):
"""
Implementation of the popular ResNet50 the following architecture:
CONV2D -> BATCHNORM -> RELU -> MAXPOOL -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3
-> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> TOPLAYER
Arguments:
input_shape -- shape of the images of the dataset
classes -- integer, number of classes
Returns:
model -- a Model() instance in Keras
"""
# Define the input as a tensor with shape input_shape
X_input = Input(input_shape)
# Zero-Padding
X = ZeroPadding2D((3, 3))(X_input)
# Stage 1
X = Conv2D(64, (7, 7), strides = (2, 2), name = 'conv1', kernel_initializer = glorot_uniform(seed=0))(X)
X = BatchNormalization(axis = 3, name = 'bn_conv1')(X)
X = Activation('relu')(X)
X = MaxPooling2D((3, 3), strides=(2, 2))(X)
# Stage 2
X = convolutional_block(X, f = 3, filters = [64, 64, 256], stage = 2, block='a', s = 1)
X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')
X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')
### START CODE HERE ###
# Stage 3 (≈4 lines)
X = convolutional_block(X,f=3, filters=[128,128,512], stage=3,block='a', s=2)
X = identity_block(X, f=3, filters=[128,128,512], stage=3,block='b')
X = identity_block(X, f=3, filters=[128,128,512], stage=3,block='c')
X = identity_block(X, f=3, filters=[128,128,512], stage=3,block='d')
# Stage 4 (≈6 lines)
X = convolutional_block(X,f=3, filters=[256,256,1024], stage=4,block='a', s=2)
X = identity_block(X, f=3, filters=[256,256,1024], stage=4,block='b')
X = identity_block(X, f=3, filters=[256,256,1024], stage=4,block='c')
X = identity_block(X, f=3, filters=[256,256,1024], stage=4,block='d')
X = identity_block(X, f=3, filters=[256,256,1024], stage=4,block='e')
X = identity_block(X, f=3, filters=[256,256,1024], stage=4,block='f')
# Stage 5 (≈3 lines)
X = convolutional_block(X,f=3, filters=[512,512,2048], stage=5,block='a', s=2)
X = identity_block(X, f=3, filters=[512,512,2048], stage=5,block='b')
X = identity_block(X, f=3, filters=[512,512,2048], stage=5,block='c')
# AVGPOOL (≈1 line). Use "X = AveragePooling2D(...)(X)"
X = AveragePooling2D(pool_size=(2,2),name="avg_pool")(X)
### END CODE HERE ###
# output layer
X = Flatten()(X)
X = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer = glorot_uniform(seed=0))(X)
# Create model
model = Model(inputs = X_input, outputs = X, name='ResNet50')
return model
def center_normalize(x):
return (x - K.mean(x)) / K.std(x)
print('1')
model = Sequential()
model.add(
Activation(activation=center_normalize,
input_shape=(ROWS, COLS, CHANNELS)))
model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3, border_mode='valid'))
model.add(Activation('relu'))
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.25))
model.add(Convolution2D(96, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(96, 3, 3, border_mode='valid'))
model.add(Activation('relu'))
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.25))
model.add(Convolution2D(128, 2, 2, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(128, 2, 2, border_mode='same'))
model.add(Activation('relu'))
def nn_base(input_tensor=None, trainable=False):
# Determine proper input shape
if K.image_dim_ordering() == 'th':
input_shape = (3, None, None)
else:
input_shape = (None, None, 3)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D((3, 3))(img_input)
x = Convolution2D(64, (7, 7), strides=(2, 2), name='conv1', trainable = trainable)(x)
x = FixedBatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1), trainable = trainable)
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b', trainable = trainable)
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c', trainable = trainable)
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a', trainable = trainable)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b', trainable = trainable)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c', trainable = trainable)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d', trainable = trainable)
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='g', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='h', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='i', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='j', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='k', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='l', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='m', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='n', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='o', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='p', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='q', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='r', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='s', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='t', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='u', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='v', trainable = trainable)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='w', trainable = trainable)
return x
def save_bottleneck_features():
"""builds the pretrained vgg16 model and runs it on our training and validation datasets"""
datagen = ImageDataGenerator(rescale=1./255)
# match the vgg16 architecture so we can load the pretrained weights into this model
model = Sequential()
model.add(ZeroPadding2D((1, 1), input_shape=(3, img_width, img_height)))
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
# load VGG16 weights
f = h5py.File(weights_path)
for k in range(f.attrs['nb_layers']):
if k >= len(model.layers):
break
g = f['layer_{}'.format(k)]
weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
model.layers[k].set_weights(weights)
f.close()
print 'Model loaded.'
generator = datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
batch_size=32,
class_mode=None,
shuffle=False)
bottleneck_features_train = model.predict_generator(generator, nb_train_samples)
np.save(open('bottleneck_features_train.npy', 'wb'), bottleneck_features_train)
generator = datagen.flow_from_directory(
validation_data_dir,
target_size=(img_width, img_height),
batch_size=32,
class_mode=None,
shuffle=False)
bottleneck_features_validation = model.predict_generator(generator, nb_validation_samples)
np.save(open('bottleneck_features_validation.npy', 'wb'), bottleneck_features_validation)
def fine_tune():
"""recreates top model architecture/weights and fine tunes with image augmentation and optimizations"""
X_test, y_test = load_data('valid')
print('X_test shape:', X_test.shape)
print(X_test.shape[0], 'test samples')
X_test /= 255
# reconstruct vgg16 model
model = Sequential()
model.add(ZeroPadding2D((1, 1), input_shape=(3, img_width, img_height)))
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
# load vgg16 weights
f = h5py.File(weights_path)
for k in range(f.attrs['nb_layers']):
if k >= len(model.layers):
break
g = f['layer_{}'.format(k)]
weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
model.layers[k].set_weights(weights)
f.close()
# add the classification layers
top_model = Sequential()
top_model.add(Flatten(input_shape=model.output_shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.5))
top_model.add(Dense(1, activation='sigmoid'))
top_model.load_weights(top_model_weights_path)
# add the model on top of the convolutional base
model.add(top_model)
# set the first 25 layers (up to the last conv block)
# to non-trainable (weights will not be updated)
for layer in model.layers[:25]:
layer.trainable = False
# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
model.compile(loss='binary_crossentropy',
optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy','precision', 'recall'])
# prepare data augmentation configuration
train_datagen = ImageDataGenerator(
rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_height, img_width),
batch_size=32,
class_mode='binary')
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_height, img_width),
batch_size=32,
class_mode='binary')
# fine-tune the model
i = 1
for epoch in range(1,nb_epoch+1):
scores = model.fit_generator(
train_generator,
samples_per_epoch=nb_train_samples,
nb_epoch=1,
validation_data=validation_generator,
nb_val_samples=nb_validation_samples,
callbacks = save_best_only)
y_pred = model.predict_classes(X_test, batch_size=64)
file_name = 'y_pred_'+str(i)+'.txt'
np.savetxt(file_name, y_pred)
y_score = model.predict_proba(X_test, batch_size=64)
file_name = 'y_score_'+str(i)+'.txt'
np.savetxt(file_name, y_score)
i=i+1
#f_hist_2.write(str(scores.history))
#f_hist_2.close()
# save the model
json_string = model.to_json()
with open('final_model_architecture.json', 'w') as f:
f.write(json_string)
model.save_weights('final_weights.h5')
# Predictions
predictions = model.predict_generator(validation_generator, nb_validation_samples)
np.savetxt('y_pred.txt', predictions)
return model
def faceRecoModel(input_shape):
"""
Implementation of the Inception model used for FaceNet
Arguments:
input_shape -- shape of the images of the dataset
Returns:
model -- a Model() instance in Keras
"""
# Define the input as a tensor with shape input_shape
X_input = Input(input_shape)
# Zero-Padding
X = ZeroPadding2D((3, 3))(X_input)
# First Block
X = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(X)
X = BatchNormalization(axis=1, name='bn1')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
X = MaxPooling2D((3, 3), strides=2)(X)
# Second Block
X = Conv2D(64, (1, 1), strides=(1, 1), name='conv2')(X)
X = BatchNormalization(axis=1, epsilon=0.00001, name='bn2')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
# Second Block
X = Conv2D(192, (3, 3), strides=(1, 1), name='conv3')(X)
X = BatchNormalization(axis=1, epsilon=0.00001, name='bn3')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
X = MaxPooling2D(pool_size=3, strides=2)(X)
# Inception 1: a/b/c
X = inception_block_1a(X)
X = inception_block_1b(X)
X = inception_block_1c(X)
# Inception 2: a/b
X = inception_block_2a(X)
X = inception_block_2b(X)
# Inception 3: a/b
X = inception_block_3a(X)
X = inception_block_3b(X)
# Top layer
X = AveragePooling2D(pool_size=(3, 3),
strides=(1, 1),
data_format='channels_first')(X)
X = Flatten()(X)
X = Dense(128, name='dense_layer')(X)
# L2 normalization
X = Lambda(lambda x: K.l2_normalize(x, axis=1))(X)
# Create model instance
model = Model(inputs=X_input, outputs=X, name='FaceRecoModel')
return model
def block3(x,
filters,
kernel_size=3,
stride=1,
groups=32,
conv_shortcut=True,
name=None):
# A residual block.
"""
# Arguments
x: input tensor.
filters: integer, filters of the bottleneck layer.
kernel_size: default 3, kernel size of the bottleneck layer.
stride: default 1, stride of the first layer.
groups: default 32, group size for grouped convolution.
conv_shortcut: default True, use convolution shortcut if True,
otherwise identity shortcut.
name: string, block label.
# Return
Output tensor for the residual block.
"""
bn_axis = 3 if K.image_data_format() == 'channels_last' else 1
if conv_shortcut is True:
shortcut = Conv2D((64 // groups) * filters,
1,
strides=stride,
use_bias=False,
name=name + '_0_conv')(x)
shortcut = BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + '_0_bn')(shortcut)
else:
shortcut = x
x = Conv2D(filters, 1, use_bias=False, name=name + '_1_conv')(x)
x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name=name + '_1_bn')(x)
x = Activation('relu', name=name + '_1_relu')(x)
c = filters // groups
x = ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x)
x = DepthwiseConv2D(kernel_size,
strides=stride,
depth_multiplier=c,
use_bias=False,
name=name + '_2_conv')(x)
kernel = np.zeros((1, 1, filters * c, filters), dtype=np.float32)
for i in range(filters):
start = (i // c) * c * c + i % c
end = start + c * c
kernel[:, :, start:end:c, i] = 1.
x = Conv2D(filters,
1,
use_bias=False,
trainable=False,
kernel_initializer={
'class_name': 'Constant',
'config': {
'value': kernel
}
},
name=name + '_2_gconv')(x)
x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name=name + '_2_bn')(x)
x = Activation('relu', name=name + '_2_relu')(x)
x = Conv2D((64 // groups) * filters,
1,
use_bias=False,
name=name + '_3_conv')(x)
x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name=name + '_3_bn')(x)
x = Add(name=name + '_add')([shortcut, x])
x = Activation('relu', name=name + '_out')(x)
return x
def ResNet50(input_tensor=None,
input_shape=None,
include_top=False,
n_classes=1000):
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = ZeroPadding2D((3, 3))(img_input)
x = Conv2D(filters=64, kernel_size=(7, 7), strides=(2, 2))(x)
x = BatchNormalization(axis=3)(x)
x = Activation('relu')(x)
x = MaxPool2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, [64, 64, 256], strides=(1, 1), kernel_size=3)
x = identity_block(x, [64, 64, 256], kernel_size=3)
x = identity_block(x, [64, 64, 256], kernel_size=3)
x = conv_block(x, [128, 128, 512], kernel_size=3)
x = identity_block(x, [128, 128, 512], kernel_size=3)
x = identity_block(x, [128, 128, 512], kernel_size=3)
x = identity_block(x, [128, 128, 512], kernel_size=3)
x = conv_block(x, [256, 256, 1024], kernel_size=3)
x = identity_block(x, [256, 256, 1024], kernel_size=3)
x = identity_block(x, [256, 256, 1024], kernel_size=3)
x = identity_block(x, [256, 256, 1024], kernel_size=3)
x = identity_block(x, [256, 256, 1024], kernel_size=3)
x = identity_block(x, [256, 256, 1024], kernel_size=3)
x = conv_block(x, [512, 512, 2048], kernel_size=3)
x = identity_block(x, [512, 512, 2048], kernel_size=3)
x = identity_block(x, [512, 512, 2048], kernel_size=3)
x = AveragePooling2D((7, 7))(x)
if include_top:
x = Flatten()(x)
x = Dense(n_classes, activation='softmax')(x)
#else:
# x = GlobalMaxPooling2D()(x)
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
model = Model(inputs=inputs, outputs=x)
# load weights
if include_top:
print('Using weights with the top layers...')
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='a7b3fe01876f51b976af0dea6bc144eb')
else:
print('Using weights without the top layers...')
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='a268eb855778b3df3c7506639542a6af')
model.load_weights(weights_path)
for layer in model.layers:
layer.trainable = False
# Adding additional layers when not using for imagenet.
# if not include_top:
# x = Flatten()(model.output)
# predictions = Dense(n_classes, activation='softmax')(x)
# resnet = Model(inputs=model.input, outputs=predictions)
# return resnet
# else:
return model
def ResNet(stack_fn,
preact,
use_bias,
model_name='resnet',
include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
num_classes=1000,
**kwargs):
# Instantiates the ResNet, ResNetV2, and ResNeXt architecture.
"""
# Arguments
stack_fn: a function that returns output tensor for the stacked residual blocks.
preact: whether to use pre-activation or not (True for ResNetV2, False for ResNet and ResNeXt).
use_bias: whether use biases for conv layers or not (True for ResNet/ResNetV2, False for ResNeXt).
model_name: string, model name.
include_top: whether to include the FC layer at the top of the network.
weights: `None` (random initialization), 'imagenet' or the path to any weights.
input_tensor: optional Keras tensor (output of `layers.Input()`)
input_shape: tuple, only to be specified if `include_top` is False.
pooling: Optional mode for feature extraction when `include_top` is `False`.
- `None`: the output of model is the 4D tensor of the last conv layer
- `avg` means global average pooling and the output as a 2D tensor.
- `max` means global max pooling will be applied.
num_classes: specified if `include_top` is True
num_classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights` or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and num_classes != 1000:
raise ValueError(
'If using `weights` as `"imagenet"` with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=32,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
bn_axis = 3 if K.image_data_format() == 'channels_last' else 1
x = ZeroPadding2D(padding=((3, 3), (3, 3)), name='conv1_pad')(img_input)
x = Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv')(x)
if preact is False:
x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name='conv1_bn')(x)
x = Activation('relu', name='conv1_relu')(x)
x = ZeroPadding2D(padding=((1, 1), (1, 1)), name='pool1_pad')(x)
x = MaxPooling2D(3, strides=2, name='pool1_pool')(x)
x = stack_fn(x)
if preact is True:
x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name='post_bn')(x)
x = Activation('relu', name='post_relu')(x)
if include_top:
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(num_classes, activation='softmax', name='probs')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D(name='avg_pool')(x)
elif pooling == 'max':
x = GlobalMaxPooling2D(name='max_pool')(x)
# Ensure the model considers any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Build the model.
model = Model(inputs, x, name=model_name)
# Load weights.
if (weights == 'imagenet') and (model_name in WEIGHTS_HASHES):
if include_top:
file_name = model_name + '_weights_tf_dim_ordering_tf_kernels.h5'
file_hash = WEIGHTS_HASHES[model_name][0]
else:
file_name = model_name + '_weights_tf_dim_ordering_tf_kernels_notop.h5'
file_hash = WEIGHTS_HASHES[model_name][1]
weights_path = get_file(file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir='models',
file_hash=file_hash)
by_name = True if 'resnext' in model_name else False
model.load_weights(weights_path, by_name=True)
elif weights is not None:
model.load_weights(weights)
return model
def ssd_512(image_size,
n_classes,
l2_regularization=0.0005,
min_scale=None,
max_scale=None,
scales=None,
aspect_ratios_global=None,
aspect_ratios_per_layer=[[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5], [1.0, 2.0, 0.5]],
two_boxes_for_ar1=True,
steps=[8, 16, 32, 64, 128, 256, 512],
offsets=None,
limit_boxes=False,
variances=[0.1, 0.1, 0.2, 0.2],
coords='centroids',
normalize_coords=False,
subtract_mean=[123, 117, 104],
divide_by_stddev=None,
swap_channels=True,
return_predictor_sizes=False):
'''
Build a Keras model with SSD512 architecture, see references.
The base network is a reduced atrous VGG-16, extended by the SSD architecture,
as described in the paper.
In case you're wondering why this function has so many arguments: All arguments except
the first two (`image_size` and `n_classes`) are only needed so that the anchor box
layers can produce the correct anchor boxes. In case you're training the network, the
parameters passed here must be the same as the ones used to set up `SSDBoxEncoder`.
In case you're loading trained weights, the parameters passed here must be the same
as the ones used to produce the trained weights.
Some of these arguments are explained in more detail in the documentation of the
`SSDBoxEncoder` class.
Note: Requires Keras v2.0 or later. Currently works only with the
TensorFlow backend (v1.0 or later).
Arguments:
image_size (tuple): The input image size in the format `(height, width, channels)`.
n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO.
l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers.
Set to zero to deactivate L2-regularization.
min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images.
max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. All scaling factors between the smallest and the
largest will be linearly interpolated. Note that the second to last of the linearly interpolated
scaling factors will actually be the scaling factor for the last predictor layer, while the last
scaling factor is used for the second box for aspect ratio 1 in the last predictor layer
if `two_boxes_for_ar1` is `True`.
scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer.
This list must be one element longer than the number of predictor layers. The first `k` elements are the
scaling factors for the `k` predictor layers, while the last element is used for the second box
for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional
last scaling factor must be passed either way, even if it is not being used.
If a list is passed, this argument overrides `min_scale` and `max_scale`. All scaling factors
must be greater than zero.
aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be
generated. This list is valid for all prediction layers.
aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each prediction layer.
This allows you to set the aspect ratios for each predictor layer individually, which is the case for the
original SSD512 implementation. If a list is passed, it overrides `aspect_ratios_global`.
Defaults to the aspect ratios used in the original SSD512 architecture, i.e.:
[[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5, 3.0, 1.0/3.0],
[1.0, 2.0, 0.5, 3.0, 1.0/3.0],
[1.0, 2.0, 0.5, 3.0, 1.0/3.0],
[1.0, 2.0, 0.5, 3.0, 1.0/3.0],
[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5]]
two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise.
If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated
using the scaling factor for the respective layer, the second one will be generated using
geometric mean of said scaling factor and next bigger scaling factor. Defaults to `True`, following the original
implementation.
steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many
pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over
the image. If the list contains ints/floats, then that value will be used for both spatial dimensions.
If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`.
If no steps are provided, then they will be computed such that the anchor box center points will form an
equidistant grid within the image dimensions.
offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either floats or tuples of two floats. These numbers represent for each predictor layer how many
pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be
as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions
of the step size specified in the `steps` argument. If the list contains floats, then that value will
be used for both spatial dimensions. If the list contains tuples of two floats, then they represent
`(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size.
limit_boxes (bool, optional): If `True`, limits box coordinates to stay within image boundaries.
This would normally be set to `True`, but here it defaults to `False`, following the original
implementation.
variances (list, optional): A list of 4 floats >0 with scaling factors (actually it's not factors but divisors
to be precise) for the encoded predicted box coordinates. A variance value of 1.0 would apply
no scaling at all to the predictions, while values in (0,1) upscale the encoded predictions and values greater
than 1.0 downscale the encoded predictions. Defaults to `[0.1, 0.1, 0.2, 0.2]`, following the original implementation.
The coordinate format must be 'centroids'.
coords (str, optional): The box coordinate format to be used. Can be either 'centroids' for the format
`(cx, cy, w, h)` (box center coordinates, width, and height) or 'minmax' for the format
`(xmin, xmax, ymin, ymax)`. Defaults to 'centroids', following the original implementation.
normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates,
i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates.
subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values
of any shape that is broadcast-compatible with the image shape. The elements of this array will be
subtracted from the image pixel intensity values. For example, pass a list of three integers
to perform per-channel mean normalization for color images.
divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or
floating point values of any shape that is broadcast-compatible with the image shape. The image pixel
intensity values will be divided by the elements of this array. For example, pass a list
of three integers to perform per-channel standard deviation normalization for color images.
swap_channels (bool, optional): If `True`, the color channel order of the input images will be reversed,
i.e. if the input color channel order is RGB, the color channels will be swapped to BGR.
return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also
a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since
you can always get their sizes easily via the Keras API, but it's convenient and less error-prone
to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the
spatial dimensions of the predictor layers), for inference you don't need them.
Returns:
model: The Keras SSD512 model.
predictor_sizes (optional): A Numpy array containing the `(height, width)` portion
of the output tensor shape for each convolutional predictor layer. During
training, the generator function needs this in order to transform
the ground truth labels into tensors of identical structure as the
output tensors of the model, which is in turn needed for the cost
function.
References:
https://arxiv.org/abs/1512.02325v5
'''
n_predictor_layers = 7 # The number of predictor conv layers in the network is 7 for the original SSD512
n_classes += 1 # Account for the background class.
# Get a few exceptions out of the way first.
if aspect_ratios_global is None and aspect_ratios_per_layer is None:
raise ValueError(
"`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified."
)
if aspect_ratios_per_layer:
if len(aspect_ratios_per_layer) != n_predictor_layers:
raise ValueError(
"It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}."
.format(n_predictor_layers, len(aspect_ratios_per_layer)))
if (min_scale is None or max_scale is None) and scales is None:
raise ValueError(
"Either `min_scale` and `max_scale` or `scales` need to be specified."
)
if scales:
if len(scales) != n_predictor_layers + 1:
raise ValueError(
"It must be either scales is None or len(scales) == {}, but len(scales) == {}."
.format(n_predictor_layers + 1, len(scales)))
else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale`
scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1)
if len(variances) != 4:
raise ValueError(
"4 variance values must be pased, but {} values were received.".
format(len(variances)))
variances = np.array(variances)
if np.any(variances <= 0):
raise ValueError(
"All variances must be >0, but the variances given are {}".format(
variances))
if (not (steps is None)) and (len(steps) != n_predictor_layers):
raise ValueError(
"You must provide at least one step value per predictor layer.")
if (not (offsets is None)) and (len(offsets) != n_predictor_layers):
raise ValueError(
"You must provide at least one offset value per predictor layer.")
# Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers.
if aspect_ratios_per_layer:
aspect_ratios = aspect_ratios_per_layer
else:
aspect_ratios = [aspect_ratios_global] * n_predictor_layers
# Compute the number of boxes to be predicted per cell for each predictor layer.
# We need this so that we know how many channels the predictor layers need to have.
if aspect_ratios_per_layer:
n_boxes = []
for ar in aspect_ratios_per_layer:
if (1 in ar) & two_boxes_for_ar1:
n_boxes.append(len(ar) +
1) # +1 for the second box for aspect ratio 1
else:
n_boxes.append(len(ar))
else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer
if (1 in aspect_ratios_global) & two_boxes_for_ar1:
n_boxes = len(aspect_ratios_global) + 1
else:
n_boxes = len(aspect_ratios_global)
n_boxes = [n_boxes] * n_predictor_layers
if steps is None:
steps = [None] * n_predictor_layers
if offsets is None:
offsets = [None] * n_predictor_layers
l2_reg = l2_regularization
# Input image format
img_height, img_width, img_channels = image_size[0], image_size[
1], image_size[2]
### Build the actual network.
x = Input(shape=(img_height, img_width, img_channels))
# The following identity layer is only needed so that subsequent lambda layers can be optional.
x1 = Lambda(lambda z: z,
output_shape=(img_height, img_width, img_channels),
name='idendity_layer')(x)
if not (subtract_mean is None):
x1 = Lambda(lambda z: z - np.array(subtract_mean),
output_shape=(img_height, img_width, img_channels),
name='input_mean_normalization')(x1)
if not (divide_by_stddev is None):
x1 = Lambda(lambda z: z / np.array(divide_by_stddev),
output_shape=(img_height, img_width, img_channels),
name='input_stddev_normalization')(x1)
if swap_channels and (img_channels == 3):
x1 = Lambda(lambda z: z[..., ::-1],
output_shape=(img_height, img_width, img_channels),
name='input_channel_swap')(x1)
conv1_1 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv1_1')(x1)
conv1_2 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv1_2')(conv1_1)
pool1 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool1')(conv1_2)
conv2_1 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv2_1')(pool1)
conv2_2 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv2_2')(conv2_1)
pool2 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool2')(conv2_2)
conv3_1 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_1')(pool2)
conv3_2 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_2')(conv3_1)
conv3_3 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_3')(conv3_2)
pool3 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool3')(conv3_3)
conv4_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_1')(pool3)
conv4_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_2')(conv4_1)
conv4_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_3')(conv4_2)
pool4 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool4')(conv4_3)
conv5_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_1')(pool4)
conv5_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_2')(conv5_1)
conv5_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_3')(conv5_2)
pool5 = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
padding='same',
name='pool5')(conv5_3)
fc6 = Conv2D(1024, (3, 3),
dilation_rate=(6, 6),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc6')(pool5)
fc7 = Conv2D(1024, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc7')(fc6)
conv6_1 = Conv2D(256, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_1')(fc7)
conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv6_padding')(conv6_1)
conv6_2 = Conv2D(512, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_2')(conv6_1)
conv7_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_1')(conv6_2)
conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv7_padding')(conv7_1)
conv7_2 = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_2')(conv7_1)
conv8_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_1')(conv7_2)
conv8_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv8_padding')(conv8_1)
conv8_2 = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_2')(conv8_1)
conv9_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_1')(conv8_2)
conv9_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv9_padding')(conv9_1)
conv9_2 = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_2')(conv9_1)
conv10_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv10_1')(conv9_2)
conv10_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv10_padding')(conv10_1)
conv10_2 = Conv2D(256, (4, 4),
strides=(1, 1),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv10_2')(conv10_1)
# Feed conv4_3 into the L2 normalization layer
conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3)
### Build the convolutional predictor layers on top of the base network
# We precidt `n_classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * n_classes`
# Output shape of the confidence layers: `(batch, height, width, n_boxes * n_classes)`
conv4_3_norm_mbox_conf = Conv2D(
n_boxes[0] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_3_norm_mbox_conf')(conv4_3_norm)
fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc7_mbox_conf')(fc7)
conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_2_mbox_conf')(conv6_2)
conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_2_mbox_conf')(conv7_2)
conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_2_mbox_conf')(conv8_2)
conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_2_mbox_conf')(conv9_2)
conv10_2_mbox_conf = Conv2D(n_boxes[6] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv10_2_mbox_conf')(conv10_2)
# We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4`
# Output shape of the localization layers: `(batch, height, width, n_boxes * 4)`
conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_3_norm_mbox_loc')(conv4_3_norm)
fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc7_mbox_loc')(fc7)
conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_2_mbox_loc')(conv6_2)
conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_2_mbox_loc')(conv7_2)
conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_2_mbox_loc')(conv8_2)
conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_2_mbox_loc')(conv9_2)
conv10_2_mbox_loc = Conv2D(n_boxes[6] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv10_2_mbox_loc')(conv10_2)
### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names)
# Output shape of anchors: `(batch, height, width, n_boxes, 8)`
conv4_3_norm_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[0],
next_scale=scales[1],
aspect_ratios=aspect_ratios[0],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[0],
this_offsets=offsets[0],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc)
fc7_mbox_priorbox = AnchorBoxes(img_height,
img_width,
this_scale=scales[1],
next_scale=scales[2],
aspect_ratios=aspect_ratios[1],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[1],
this_offsets=offsets[1],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='fc7_mbox_priorbox')(fc7_mbox_loc)
conv6_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[2],
next_scale=scales[3],
aspect_ratios=aspect_ratios[2],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[2],
this_offsets=offsets[2],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc)
conv7_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[3],
next_scale=scales[4],
aspect_ratios=aspect_ratios[3],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[3],
this_offsets=offsets[3],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc)
conv8_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[4],
next_scale=scales[5],
aspect_ratios=aspect_ratios[4],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[4],
this_offsets=offsets[4],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc)
conv9_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[5],
next_scale=scales[6],
aspect_ratios=aspect_ratios[5],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[5],
this_offsets=offsets[5],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc)
conv10_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[6],
next_scale=scales[7],
aspect_ratios=aspect_ratios[6],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[6],
this_offsets=offsets[6],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv10_2_mbox_priorbox')(conv10_2_mbox_loc)
### Reshape
# Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)`
# We want the classes isolated in the last axis to perform softmax on them
conv4_3_norm_mbox_conf_reshape = Reshape(
(-1, n_classes),
name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf)
fc7_mbox_conf_reshape = Reshape(
(-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf)
conv6_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf)
conv7_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf)
conv8_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf)
conv9_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf)
conv10_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv10_2_mbox_conf_reshape')(conv10_2_mbox_conf)
# Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)`
# We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
conv4_3_norm_mbox_loc_reshape = Reshape(
(-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc)
fc7_mbox_loc_reshape = Reshape((-1, 4),
name='fc7_mbox_loc_reshape')(fc7_mbox_loc)
conv6_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc)
conv7_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc)
conv8_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc)
conv9_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc)
conv10_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv10_2_mbox_loc_reshape')(conv10_2_mbox_loc)
# Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)`
conv4_3_norm_mbox_priorbox_reshape = Reshape(
(-1, 8),
name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox)
fc7_mbox_priorbox_reshape = Reshape(
(-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox)
conv6_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox)
conv7_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox)
conv8_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox)
conv9_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox)
conv10_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv10_2_mbox_priorbox_reshape')(conv10_2_mbox_priorbox)
### Concatenate the predictions from the different layers
# Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions,
# so we want to concatenate along axis 1, the number of boxes per layer
# Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes)
mbox_conf = Concatenate(axis=1, name='mbox_conf')([
conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape,
conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape,
conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape,
conv10_2_mbox_conf_reshape
])
# Output shape of `mbox_loc`: (batch, n_boxes_total, 4)
mbox_loc = Concatenate(axis=1, name='mbox_loc')([
conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape,
conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape,
conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape,
conv10_2_mbox_loc_reshape
])
# Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8)
mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([
conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape,
conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape,
conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape,
conv10_2_mbox_priorbox_reshape
])
# The box coordinate predictions will go into the loss function just the way they are,
# but for the class predictions, we'll apply a softmax activation layer first
mbox_conf_softmax = Activation('softmax',
name='mbox_conf_softmax')(mbox_conf)
# Concatenate the class and box predictions and the anchors to one large predictions vector
# Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8)
predictions = Concatenate(axis=2, name='predictions')(
[mbox_conf_softmax, mbox_loc, mbox_priorbox])
model = Model(inputs=x, outputs=predictions)
if return_predictor_sizes:
# Get the spatial dimensions (height, width) of the predictor conv layers, we need them to
# be able to generate the default boxes for the matching process outside of the model during training.
# Note that the original implementation performs anchor box matching inside the loss function. We don't do that.
# Instead, we'll do it in the batch generator function.
# The spatial dimensions are the same for the confidence and localization predictors, so we just take those of the conf layers.
predictor_sizes = np.array([
conv4_3_norm_mbox_conf._keras_shape[1:3],
fc7_mbox_conf._keras_shape[1:3],
conv6_2_mbox_conf._keras_shape[1:3],
conv7_2_mbox_conf._keras_shape[1:3],
conv8_2_mbox_conf._keras_shape[1:3],
conv9_2_mbox_conf._keras_shape[1:3],
conv10_2_mbox_conf._keras_shape[1:3]
])
return model, predictor_sizes
else:
return model
def _main(args):
config_path = os.path.expanduser(args.config_path)
weights_path = os.path.expanduser(args.weights_path)
assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format(
config_path)
assert weights_path.endswith(
'.weights'), '{} is not a .weights file'.format(weights_path)
output_path = os.path.expanduser(args.output_path)
assert output_path.endswith(
'.h5'), 'output path {} is not a .h5 file'.format(output_path)
output_root = os.path.splitext(output_path)[0]
# Load weights and config.
print('Loading weights.')
weights_file = open(weights_path, 'rb')
major, minor, revision = np.ndarray(shape=(3, ),
dtype='int32',
buffer=weights_file.read(12))
if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000:
seen = np.ndarray(shape=(1, ),
dtype='int64',
buffer=weights_file.read(8))
else:
seen = np.ndarray(shape=(1, ),
dtype='int32',
buffer=weights_file.read(4))
print('Weights Header: ', major, minor, revision, seen)
print('Parsing Darknet config.')
unique_config_file = unique_config_sections(config_path)
cfg_parser = configparser.ConfigParser()
cfg_parser.read_file(unique_config_file)
print('Creating Keras model.')
input_layer = Input(shape=(None, None, 3))
prev_layer = input_layer
all_layers = []
weight_decay = float(cfg_parser['net_0']['decay']
) if 'net_0' in cfg_parser.sections() else 5e-4
count = 0
out_index = []
for section in cfg_parser.sections():
print('Parsing section {}'.format(section))
if section.startswith('convolutional'):
filters = int(cfg_parser[section]['filters'])
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
pad = int(cfg_parser[section]['pad'])
activation = cfg_parser[section]['activation']
batch_normalize = 'batch_normalize' in cfg_parser[section]
padding = 'same' if pad == 1 and stride == 1 else 'valid'
# Setting weights.
# Darknet serializes convolutional weights as:
# [bias/beta, [gamma, mean, variance], conv_weights]
prev_layer_shape = K.int_shape(prev_layer)
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
print('conv2d', 'bn' if batch_normalize else ' ', activation,
weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype='float32',
buffer=weights_file.read(filters * 4))
count += filters
if batch_normalize:
bn_weights = np.ndarray(shape=(3, filters),
dtype='float32',
buffer=weights_file.read(filters * 12))
count += 3 * filters
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2] # running var
]
conv_weights = np.ndarray(shape=darknet_w_shape,
dtype='float32',
buffer=weights_file.read(weights_size *
4))
count += weights_size
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
conv_weights = [conv_weights] if batch_normalize else [
conv_weights, conv_bias
]
# Handle activation.
act_fn = None
if activation == 'leaky':
pass # Add advanced activation later.
elif activation != 'linear':
raise ValueError(
'Unknown activation function `{}` in section {}'.format(
activation, section))
# Create Conv2D layer
if stride > 1:
# Darknet uses left and top padding instead of 'same' mode
prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer)
conv_layer = (Conv2D(filters, (size, size),
strides=(stride, stride),
kernel_regularizer=l2(weight_decay),
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
padding=padding))(prev_layer)
if batch_normalize:
conv_layer = (BatchNormalization(
weights=bn_weight_list))(conv_layer)
prev_layer = conv_layer
if activation == 'linear':
all_layers.append(prev_layer)
elif activation == 'leaky':
act_layer = LeakyReLU(alpha=0.1)(prev_layer)
prev_layer = act_layer
all_layers.append(act_layer)
elif section.startswith('route'):
ids = [int(i) for i in cfg_parser[section]['layers'].split(',')]
layers = [all_layers[i] for i in ids]
if len(layers) > 1:
print('Concatenating route layers:', layers)
concatenate_layer = Concatenate()(layers)
all_layers.append(concatenate_layer)
prev_layer = concatenate_layer
else:
skip_layer = layers[0] # only one layer to route
all_layers.append(skip_layer)
prev_layer = skip_layer
elif section.startswith('maxpool'):
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
all_layers.append(
MaxPooling2D(pool_size=(size, size),
strides=(stride, stride),
padding='same')(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('shortcut'):
index = int(cfg_parser[section]['from'])
activation = cfg_parser[section]['activation']
assert activation == 'linear', 'Only linear activation supported.'
all_layers.append(Add()([all_layers[index], prev_layer]))
prev_layer = all_layers[-1]
elif section.startswith('upsample'):
stride = int(cfg_parser[section]['stride'])
assert stride == 2, 'Only stride=2 supported.'
all_layers.append(UpSampling2D(stride)(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('yolo'):
out_index.append(len(all_layers) - 1)
all_layers.append(None)
prev_layer = all_layers[-1]
elif section.startswith('net'):
pass
else:
raise ValueError(
'Unsupported section header type: {}'.format(section))
# Create and save model.
if len(out_index) == 0: out_index.append(len(all_layers) - 1)
model = Model(inputs=input_layer,
outputs=[all_layers[i] for i in out_index])
print(model.summary())
if args.weights_only:
model.save_weights('{}'.format(output_path))
print('Saved Keras weights to {}'.format(output_path))
else:
model.save('{}'.format(output_path))
print('Saved Keras model to {}'.format(output_path))
# Check to see if all weights have been read.
remaining_weights = len(weights_file.read()) / 4
weights_file.close()
print('Read {} of {} from Darknet weights.'.format(
count, count + remaining_weights))
if remaining_weights > 0:
print('Warning: {} unused weights'.format(remaining_weights))
if args.plot_model:
plot(model, to_file='{}.png'.format(output_root), show_shapes=True)
print('Saved model plot to {}.png'.format(output_root))
def AlexNet():
inputs = Input(shape=(3, 200, 200))
conv_1 = Convolution2D(96,
11,
11,
subsample=(4, 4),
activation='relu',
name='conv_1')(inputs)
conv_2 = LRN2D(name='convpool_1')(conv_1)
conv_2 = MaxPooling2D((3, 3), strides=(2, 2))(conv_2)
conv_2 = ZeroPadding2D((2, 2))(conv_2)
conv_2 = merge([
Convolution2D(128,
5,
5,
activation='relu',
init='glorot_uniform',
bias=True,
name='conv_2_' + str(i + 1))(splittensor(
ratio_split=2, id_split=i)(conv_2)) for i in range(2)
],
mode='concat',
concat_axis=1,
name='conv_2')
conv_3 = LRN2D()(conv_2)
conv_3 = MaxPooling2D((3, 3), strides=(2, 2))(conv_3)
conv_3 = ZeroPadding2D((1, 1))(conv_3)
conv_3 = Convolution2D(384,
3,
3,
activation='relu',
init='glorot_uniform',
bias=True,
name='conv_3')(conv_3)
conv_4 = ZeroPadding2D((1, 1))(conv_3)
conv_4 = merge([
Convolution2D(192,
3,
3,
activation='relu',
bias=True,
init='glorot_uniform',
name='conv_4_' + str(i + 1))(splittensor(
ratio_split=2, id_split=i)(conv_4)) for i in range(2)
],
mode='concat',
concat_axis=1,
name='conv_4')
conv_5 = ZeroPadding2D((1, 1))(conv_4)
conv_5 = merge([
Convolution2D(128,
3,
3,
activation='relu',
init='glorot_uniform',
bias=True,
name='conv_5_' + str(i + 1))(splittensor(
ratio_split=2, id_split=i)(conv_5)) for i in range(2)
],
mode='concat',
concat_axis=1,
name='conv_5')
dense_1 = MaxPooling2D((3, 3), strides=(2, 2), name='convpool_5')(conv_5)
dense_1 = Flatten(name='flatten')(dense_1)
dense_1 = Dense(4096, activation='relu', name='dense_1')(dense_1)
dense_2 = Dropout(0.5)(dense_1)
dense_2 = Dense(4096, activation='relu', name='dense_2')(dense_2)
dense_3 = Dropout(0.5)(dense_2)
dense_3 = Dense(16000, name='dense_3')(dense_3)
reshape = Reshape((400, 40))(dense_3)
prediction = Activation('softmax', name='softmax')(reshape)
model = Model(input=inputs, output=prediction)
return model
def get_resnet(f=16, bn_axis=3, classes=1):
input = Input((img_rows, img_cols, 1))
x = ZeroPadding2D((4, 4))(input)
x = Conv2D(f, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [f, f, f * 2], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [f, f, f * 2], stage=2, block='b')
x2 = identity_block(x, 3, [f, f, f * 2], stage=2, block='c')
x = conv_block(x2, 3, [f * 2, f * 2, f * 4], stage=3, block='a')
x = identity_block(x, 3, [f * 2, f * 2, f * 4], stage=3, block='b')
x3 = identity_block(x, 3, [f * 2, f * 2, f * 4], stage=3, block='d')
x = conv_block(x3, 3, [f * 4, f * 4, f * 8], stage=4, block='a')
x = identity_block(x, 3, [f * 4, f * 4, f * 8], stage=4, block='b')
x4 = identity_block(x, 3, [f * 4, f * 4, f * 8], stage=4, block='f')
x = conv_block(x4, 3, [f * 8, f * 8, f * 16], stage=5, block='a')
x = identity_block(x, 3, [f * 8, f * 8, f * 16], stage=5, block='b')
x = identity_block(x, 3, [f * 8, f * 8, f * 16], stage=5, block='c')
x = up_conv_block(x, 3, [f * 16, f * 8, f * 8], stage=6, block='a')
x = identity_block(x, 3, [f * 16, f * 8, f * 8], stage=6, block='b')
x = identity_block(x, 3, [f * 16, f * 8, f * 8], stage=6, block='c')
x = concatenate([x, x4], axis=bn_axis)
x = up_conv_block(x, 3, [f * 16, f * 4, f * 4], stage=7, block='a')
x = identity_block(x, 3, [f * 16, f * 4, f * 4], stage=7, block='b')
x = identity_block(x, 3, [f * 16, f * 4, f * 4], stage=7, block='f')
x = concatenate([x, x3], axis=bn_axis)
x = up_conv_block(x, 3, [f * 8, f * 2, f * 2], stage=8, block='a')
x = identity_block(x, 3, [f * 8, f * 2, f * 2], stage=8, block='b')
x = identity_block(x, 3, [f * 8, f * 2, f * 2], stage=8, block='d')
x = concatenate([x, x2], axis=bn_axis)
x = up_conv_block(x, 3, [f * 4, f, f], stage=10, block='a', strides=(1, 1))
x = identity_block(x, 3, [f * 4, f, f], stage=10, block='b')
x = identity_block(x, 3, [f * 4, f, f], stage=10, block='c')
x = UpSampling2D(size=(2, 2))(x)
x = Conv2D(classes, (3, 3),
padding='same',
activation='sigmoid',
name='convLast')(x)
model = Model(input, x, name='resnetUnet')
model.compile(
optimizer=Adam(lr=3e-4),
loss=dice_coef_loss,
metrics=[dice_coef, defect_accuracy, precision, recall, f1score])
model.summary()
return model
def get_resnet():
# In order to make things less confusing, all layers have been declared first, and then used
# declaration of layers
input_img = Input((1, 28, 28), name='input_layer')
zeroPad1 = ZeroPadding2D((1, 1), name='zeroPad1', dim_ordering='th')
zeroPad1_2 = ZeroPadding2D((1, 1), name='zeroPad1_2', dim_ordering='th')
layer1 = Convolution2D(6,
3,
3,
subsample=(2, 2),
init='he_uniform',
name='major_conv',
dim_ordering='th')
layer1_2 = Convolution2D(16,
3,
3,
subsample=(2, 2),
init='he_uniform',
name='major_conv2',
dim_ordering='th')
zeroPad2 = ZeroPadding2D((1, 1), name='zeroPad2', dim_ordering='th')
zeroPad2_2 = ZeroPadding2D((1, 1), name='zeroPad2_2', dim_ordering='th')
layer2 = Convolution2D(6,
3,
3,
subsample=(1, 1),
init='he_uniform',
name='l1_conv',
dim_ordering='th')
layer2_2 = Convolution2D(16,
3,
3,
subsample=(1, 1),
init='he_uniform',
name='l1_conv2',
dim_ordering='th')
zeroPad3 = ZeroPadding2D((1, 1), name='zeroPad3', dim_ordering='th')
zeroPad3_2 = ZeroPadding2D((1, 1), name='zeroPad3_2', dim_ordering='th')
layer3 = Convolution2D(6,
3,
3,
subsample=(1, 1),
init='he_uniform',
name='l2_conv',
dim_ordering='th')
layer3_2 = Convolution2D(16,
3,
3,
subsample=(1, 1),
init='he_uniform',
name='l2_conv2',
dim_ordering='th')
layer4 = Dense(64, activation='relu', init='he_uniform', name='dense1')
layer5 = Dense(16, activation='relu', init='he_uniform', name='dense2')
final = Dense(10,
activation='softmax',
init='he_uniform',
name='classifier')
# declaration completed
first = zeroPad1(input_img)
second = layer1(first)
second = BatchNormalization(0, axis=1, name='major_bn')(second)
second = Activation('relu', name='major_act')(second)
third = zeroPad2(second)
third = layer2(third)
third = BatchNormalization(0, axis=1, name='l1_bn')(third)
third = Activation('relu', name='l1_act')(third)
third = zeroPad3(third)
third = layer3(third)
third = BatchNormalization(0, axis=1, name='l1_bn2')(third)
third = Activation('relu', name='l1_act2')(third)
res = merge([third, second], mode='sum', name='res')
first2 = zeroPad1_2(res)
second2 = layer1_2(first2)
second2 = BatchNormalization(0, axis=1, name='major_bn2')(second2)
second2 = Activation('relu', name='major_act2')(second2)
third2 = zeroPad2_2(second2)
third2 = layer2_2(third2)
third2 = BatchNormalization(0, axis=1, name='l2_bn')(third2)
third2 = Activation('relu', name='l2_act')(third2)
third2 = zeroPad3_2(third2)
third2 = layer3_2(third2)
third2 = BatchNormalization(0, axis=1, name='l2_bn2')(third2)
third2 = Activation('relu', name='l2_act2')(third2)
res2 = merge([third2, second2], mode='sum', name='res2')
res2 = Flatten()(res2)
res2 = layer4(res2)
res2 = Dropout(0.4, name='dropout1')(res2)
res2 = layer5(res2)
res2 = Dropout(0.4, name='dropout2')(res2)
res2 = final(res2)
model = Model(input=input_img, output=res2)
sgd = SGD(decay=0., lr=0.01, momentum=0.9, nesterov=True)
model.compile(loss=scc, optimizer=sgd, metrics=['accuracy'])
return model
def root_relative_mean_squared_error_lasso(y_true, y_pred):
return K.sqrt(K.mean(K.square(
(y_pred - y_true) / y_true))) + 1 / np.linalg.norm(y_pred)
#Neural Network
keras.backend.set_floatx('float64')
""" encoder"""
NN1 = Sequential()
NN1.add(InputLayer(input_shape=(
Nparameters,
1,
1,
)))
NN1.add(ZeroPadding2D(padding=(2, 2)))
NN1.add(Conv2D(32, (3, 1), padding='valid', strides=(1, 1),
activation='elu')) #X_train_trafo.shape[1:],activation='elu'))
NN1.add(ZeroPadding2D(padding=(1, 1)))
NN1.add(Conv2D(32, (2, 2), padding='valid', strides=(1, 1), activation='elu'))
NN1.add(Conv2D(32, (2, 2), padding='valid', strides=(2, 1), activation='elu'))
NN1.add(ZeroPadding2D(padding=(1, 1)))
NN1.add(Conv2D(32, (3, 3), padding='valid', strides=(2, 1), activation='elu'))
NN1.add(ZeroPadding2D(padding=(1, 1)))
NN1.add(Conv2D(32, (2, 2), padding='valid', strides=(2, 1), activation='elu'))
NN1.add(ZeroPadding2D(padding=(1, 1)))
NN1.add(Conv2D(32, (2, 2), padding='valid', strides=(2, 1), activation='elu'))
#NN1.add(MaxPooling2D(pool_size=(2, 1)))
#NN1.add(Dropout(0.25))
#NN1.add(ZeroPadding2D(padding=(0,1)))
NN1.add(
# %load_ext autoreload
# %autoreload 2
# np.set_printoptions(threshold=np.nan)
# # Constructing the neural network model
# The model here constructed is based on FaceNet's Inception model.
# The implementation of model is available at: https://github.com/iwantooxxoox/Keras-OpenFace
# In[2]:
myInput = Input(shape=(96, 96, 3))
x = ZeroPadding2D(padding=(3, 3), input_shape=(96, 96, 3))(myInput)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn1')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = MaxPooling2D(pool_size=3, strides=2)(x)
x = Lambda(LRN2D, name='lrn_1')(x)
x = Conv2D(64, (1, 1), name='conv2')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn2')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = Conv2D(192, (3, 3), name='conv3')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn3')(x)
x = Activation('relu')(x)
x = Lambda(LRN2D, name='lrn_2')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
def ResNet50(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the ResNet50 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format="channels_last"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or "imagenet" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 244)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 197.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=197,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D((3, 3))(img_input)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = AveragePooling2D((7, 7), name='avg_pool')(x)
if include_top:
x = Flatten()(x)
x = Dense(classes, activation='softmax', name='fc1000')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='resnet50')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='a7b3fe01876f51b976af0dea6bc144eb')
else:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='a268eb855778b3df3c7506639542a6af')
model.load_weights(weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first':
if include_top:
maxpool = model.get_layer(name='avg_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='fc1000')
layer_utils.convert_dense_weights_data_format(
dense, shape, 'channels_first')
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
return model
def get_alexNet_model_wo_softmax(x, include_top=True):
def convLayer(x,
name,
filters,
kernel_size=(3, 3),
strides=(1, 1),
padding="valid",
groups=1):
"""(grouped) convolution"""
if groups == 1:
y = Conv2D(filters=filters,
kernel_size=kernel_size,
strides=(strides),
padding=padding)(x)
y = Activation('relu')(y)
return y
x_new = []
channels = int(x.get_shape()[-1])
split = int(channels / groups)
split_upper = split
split_lower = 0
for i in range(groups):
x_new.append(
Lambda(lambda x: x[:, :, :, split_lower:split_upper])(
x))
split_lower = split_lower + split_upper
split_upper = split_upper + split_upper
featureMaps = [
Conv2D(filters=int(filters / groups),
kernel_size=kernel_size,
strides=(strides),
padding=padding,
name=str(name) + '_' + str(i + 1))(x_n)
for i, x_n in enumerate(x_new)
]
# y = tf.concat(axis=3, values=featureMaps)
y = Concatenate(axis=-1, name=name)(featureMaps)
y = Activation('relu')(y)
return y
def crosschannelnormalization(alpha=1e-4,
k=2,
beta=0.75,
n=5,
**kwargs):
"""
This is the function used for cross channel normalization in the original
Alexnet
"""
def f(X):
b, ch, r, c = X.shape
half = n // 2
square = K.square(X)
extra_channels = K.spatial_2d_padding(
K.permute_dimensions(square, (0, 2, 3, 1)), (0, half))
extra_channels = K.permute_dimensions(
extra_channels, (0, 3, 1, 2))
scale = k
for i in range(n):
scale += alpha * extra_channels[:, i:i + ch, :, :]
scale = scale**beta
return X / scale
return Lambda(f,
output_shape=lambda input_shape: input_shape,
**kwargs)
conv_1 = Conv2D(96,
11,
11,
subsample=(4, 4),
activation='relu',
name='conv_1')(x)
conv_2 = MaxPooling2D((3, 3), strides=(2, 2))(conv_1)
conv_2 = crosschannelnormalization()(conv_2)
conv_2 = ZeroPadding2D((2, 2))(conv_2)
conv_2 = convLayer(conv_2,
name='conv_2',
filters=128,
kernel_size=(5, 5),
padding="same",
groups=2)
conv_3 = MaxPooling2D((3, 3), strides=(2, 2))(conv_2)
conv_3 = BatchNormalization()(conv_3)
conv_3 = ZeroPadding2D((1, 1))(conv_3)
conv_3 = Conv2D(384, 3, 3, activation='relu',
name='conv_3')(conv_3)
conv_4 = ZeroPadding2D((1, 1))(conv_3)
conv_4 = convLayer(conv_4,
name='conv_4',
filters=192,
kernel_size=(3, 3),
padding="same",
groups=2)
conv_5 = ZeroPadding2D((1, 1))(conv_4)
conv_5 = convLayer(conv_5,
name='conv_5',
filters=128,
kernel_size=(3, 3),
padding="same",
groups=2)
y = MaxPooling2D((3, 3), strides=(2, 2), name='convpool_5')(conv_5)
if include_top:
dense_1 = Flatten(name='flatten')(y)
dense_1 = Dense(4096, activation='relu',
name='dense_1')(dense_1)
dense_2 = Dropout(0.5)(dense_1)
dense_2 = Dense(4096, activation='relu',
name='dense_2')(dense_2)
dense_3 = Dropout(0.5)(dense_2)
dense_3 = Dense(1000, name='dense_3')(dense_3)
y = Activation('softmax', name='softmax')(dense_3)
#model = Model(inputs=[x], outputs=[y])
return y
def inception_block_1a(X):
"""
Implementation of an inception block
"""
X_3x3 = Conv2D(96, (1, 1),
data_format='channels_first',
name='inception_3a_3x3_conv1')(X)
X_3x3 = BatchNormalization(axis=1,
epsilon=0.00001,
name='inception_3a_3x3_bn1')(X_3x3)
X_3x3 = Activation('relu')(X_3x3)
X_3x3 = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_3x3)
X_3x3 = Conv2D(128, (3, 3),
data_format='channels_first',
name='inception_3a_3x3_conv2')(X_3x3)
X_3x3 = BatchNormalization(axis=1,
epsilon=0.00001,
name='inception_3a_3x3_bn2')(X_3x3)
X_3x3 = Activation('relu')(X_3x3)
X_5x5 = Conv2D(16, (1, 1),
data_format='channels_first',
name='inception_3a_5x5_conv1')(X)
X_5x5 = BatchNormalization(axis=1,
epsilon=0.00001,
name='inception_3a_5x5_bn1')(X_5x5)
X_5x5 = Activation('relu')(X_5x5)
X_5x5 = ZeroPadding2D(padding=(2, 2), data_format='channels_first')(X_5x5)
X_5x5 = Conv2D(32, (5, 5),
data_format='channels_first',
name='inception_3a_5x5_conv2')(X_5x5)
X_5x5 = BatchNormalization(axis=1,
epsilon=0.00001,
name='inception_3a_5x5_bn2')(X_5x5)
X_5x5 = Activation('relu')(X_5x5)
X_pool = MaxPooling2D(pool_size=3, strides=2,
data_format='channels_first')(X)
X_pool = Conv2D(32, (1, 1),
data_format='channels_first',
name='inception_3a_pool_conv')(X_pool)
X_pool = BatchNormalization(axis=1,
epsilon=0.00001,
name='inception_3a_pool_bn')(X_pool)
X_pool = Activation('relu')(X_pool)
X_pool = ZeroPadding2D(padding=((3, 4), (3, 4)),
data_format='channels_first')(X_pool)
X_1x1 = Conv2D(64, (1, 1),
data_format='channels_first',
name='inception_3a_1x1_conv')(X)
X_1x1 = BatchNormalization(axis=1,
epsilon=0.00001,
name='inception_3a_1x1_bn')(X_1x1)
X_1x1 = Activation('relu')(X_1x1)
# CONCAT
inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1)
return inception
spatial_model = Model(inputs=model1.input, outputs=x)
if train & (not retrain) & server:
spatial_model.load_weights(
'weights/mobilenet_spatial_{}e.h5'.format(spa_epochs))
print 'Loaded spatial weights.'
# Temporal
depth = 20
input_shape = (224, 224, depth)
model2 = keras.applications.mobilenet.MobileNet(include_top=True, dropout=0.5)
layers = [l for l in model2.layers]
input_opt = Input(shape=input_shape)
y = ZeroPadding2D(padding=(1, 1))(input_opt)
y = Conv2D(filters=32,
kernel_size=(3, 3),
padding='valid',
use_bias=False,
strides=(2, 2))(y)
for i in range(3, len(layers) - 3):
layers[i].name = 'temporal_' + layers[i].name
y = layers[i](y)
y = Flatten()(y)
y = Dense(classes, activation='softmax')(y)
temporal_model = Model(inputs=input_opt, outputs=y)
if train & (not retrain) & server:
temporal_model.load_weights('weights/mobilenet_temporal{}_{}e.h5'.format(
def CreateResNet(dataSize, learningRate, kernelInitializer):
# Define the input as a tensor with shape input_shape
X_input = Input(dataSize)
# Zero-Padding
X = ZeroPadding2D((3, 3))(X_input)
# Stage 1
X = Conv2D(64, (7, 7),
strides=(2, 2),
name='conv1',
kernel_initializer=kernelInitializer)(X)
X = BatchNormalization(axis=3, name='bn_conv1')(X)
X = Activation('relu')(X)
X = MaxPooling2D((3, 3), strides=(2, 2))(X)
# Stage 2
X = convolutional_block(X,
f=3,
filters=[64, 64, 256],
stage=2,
block='a',
s=1,
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [64, 64, 256],
stage=2,
block='b',
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [64, 64, 256],
stage=2,
block='c',
kernelInitializer=kernelInitializer)
### START CODE HERE ###
# Stage 3 (≈4 lines)
X = convolutional_block(X,
f=3,
filters=[128, 128, 512],
stage=3,
block='a',
s=2,
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [128, 128, 512],
stage=3,
block='b',
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [128, 128, 512],
stage=3,
block='c',
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [128, 128, 512],
stage=3,
block='d',
kernelInitializer=kernelInitializer)
# Stage 4 (≈6 lines)
X = convolutional_block(X,
f=3,
filters=[256, 256, 1024],
stage=4,
block='a',
s=2,
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [256, 256, 1024],
stage=4,
block='b',
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [256, 256, 1024],
stage=4,
block='c',
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [256, 256, 1024],
stage=4,
block='d',
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [256, 256, 1024],
stage=4,
block='e',
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [256, 256, 1024],
stage=4,
block='f',
kernelInitializer=kernelInitializer)
# Stage 5 (≈3 lines)
X = convolutional_block(X,
f=3,
filters=[512, 512, 2048],
stage=5,
block='a',
s=2,
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [512, 512, 2048],
stage=5,
block='b',
kernelInitializer=kernelInitializer)
X = identity_block(X,
3, [512, 512, 2048],
stage=5,
block='c',
kernelInitializer=kernelInitializer)
# AVGPOOL (≈1 line). Use "X = AveragePooling2D(...)(X)"
X = AveragePooling2D((2, 2), name="avg_pool")(X)
### END CODE HERE ###
# output layer
X = Flatten()(X)
X = Dense(1, activation='sigmoid', kernel_initializer=kernelInitializer)(X)
# Create model
model = Model(inputs=X_input, outputs=X, name='ResNet50')
optimizer = keras.optimizers.Adam(lr=learningRate)
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['accuracy'])
return model
