def __call__(self, bboxes): default_anchors = [] for i, bbox in enumerate(bboxes): temp_anchors = AnchorBoxes( self.img_height, self.img_width, this_scale=self.scales[i], next_scale=self.scales[i + 1], aspect_ratios=self.aspect_ratios[i], two_boxes_for_ar1=self.two_boxes_for_ar1, this_steps=self.steps[i], this_offsets=self.offsets[i], clip_boxes=self.clip_boxes, variances=self.variances, coords=self.coords, normalize_coords=self.normalize_coords, name='anchors' + str(i))(bbox) default_anchors.append(temp_anchors) return default_anchors
def build_model(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=0.1, max_scale=0.9, scales=None, aspect_ratios_global=[0.5, 1.0, 2.0], aspect_ratios_per_layer=None, two_boxes_for_ar1=True, steps=None, offsets=None, clip_boxes=False, variances=[1.0, 1.0, 1.0, 1.0], coords='centroids', normalize_coords=False, subtract_mean=None, divide_by_stddev=None, swap_channels=False, confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False, n_predictor_layers=2): n_predictor_layers = n_predictor_layers # The number of predictor conv layers in the network n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len( variances ) != 4: # We need one variance value for each of the four box coordinates raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # Build the network. ############################################################################ input_index = 0 model_input = Input(shape=(img_height, img_width, img_channels), name=f'input_{input_index}') # x = Input(shape=(img_height, img_width, img_channels), name=f'input_{input_index}') # model_input = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) # if not (subtract_mean is None): # model_input = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) # if not (divide_by_stddev is None): # model_input = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) # if swap_channels: # model_input = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) """ Feature Extraction """ input_conv = Conv2D(32, (3, 3), (2, 2), padding='same', kernel_regularizer=l2(l2_reg), name=f"input_{input_index}_conv")(model_input) input_relu = ReLU(name=f"input_{input_index}_relu")(input_conv) block_1 = invResBlock(input_relu, index=1, expand=32, dw_mult=3, squeeze=32, strides=2) block_2 = invResBlock(block_1, index=2, expand=32, dw_mult=3, squeeze=64, strides=2) block_3 = invResBlock(block_2, index=3, expand=48, dw_mult=3, squeeze=128, strides=2) block_4 = invResBlock(block_3, index=4, expand=96, dw_mult=3, squeeze=256, strides=2) """ End of Feature Extraction """ """ SSD Layers """ classes3 = Conv2D(n_boxes[0] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes4')(block_3) classes4 = Conv2D(n_boxes[1] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes5')(block_4) # classes6 = Conv2D(n_boxes[2] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes6')(conv6) # classes7 = Conv2D(n_boxes[3] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes7')(conv7) boxes3 = Conv2D(n_boxes[0] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes4')(block_3) boxes4 = Conv2D(n_boxes[1] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes5')(block_4) anchors3 = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors4')(boxes3) anchors4 = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors5')(boxes4) classes3_reshaped = Reshape((-1, n_classes), name='classes4_reshape')(classes3) classes4_reshaped = Reshape((-1, n_classes), name='classes5_reshape')(classes4) boxes3_reshaped = Reshape((-1, 4), name='boxes4_reshape')(boxes3) boxes4_reshaped = Reshape((-1, 4), name='boxes5_reshape')(boxes4) anchors3_reshaped = Reshape((-1, 8), name='anchors4_reshape')(anchors3) anchors4_reshaped = Reshape((-1, 8), name='anchors5_reshape')(anchors4) classes_concat = Concatenate(axis=1, name='classes_concat')([ classes3_reshaped, classes4_reshaped, ]) # Output shape of `boxes_concat`: (batch, n_boxes_total, 4) boxes_concat = Concatenate(axis=1, name='boxes_concat')([ boxes3_reshaped, boxes4_reshaped, ]) # Output shape of `anchors_concat`: (batch, n_boxes_total, 8) anchors_concat = Concatenate(axis=1, name='anchors_concat')([ anchors3_reshaped, anchors4_reshaped, ]) classes_softmax = Softmax(name='classes_softmax')(classes_concat) # Concatenate the class and box coordinate predictions and the anchors to one large predictions tensor # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [classes_softmax, boxes_concat, anchors_concat]) """ End of SSD Layers """ model = Model(model_input, predictions) if mode == 'training': model = Model(inputs=model_input, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=model_input, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=model_input, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: # The spatial dimensions are the same for the `classes` and `boxes` predictor layers. predictor_sizes = np.array( [classes3.shape.as_list()[1:3], classes4.shape.as_list()[1:3]]) return model, predictor_sizes else: return model
def build_model(image_size, n_classes, mode='training', l2_regularization=0.0, min_scale=0.1, max_scale=0.9, scales=None, aspect_ratios_global=[0.5, 1.0, 2.0], aspect_ratios_per_layer=None, two_boxes_for_ar1=True, steps=None, offsets=None, clip_boxes=False, variances=[1.0, 1.0, 1.0, 1.0], coords='centroids', normalize_coords=False, subtract_mean=None, divide_by_stddev=None, swap_channels=False, confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): n_predictor_layers = 4 # The number of predictor conv layers in the network n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError("`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified.") if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError("It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}.".format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError("Either `min_scale` and `max_scale` or `scales` need to be specified.") if scales: if len(scales) != n_predictor_layers+1: raise ValueError("It must be either scales is None or len(scales) == {}, but len(scales) == {}.".format(n_predictor_layers+1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers+1) if len(variances) != 4: # We need one variance value for each of the four box coordinates raise ValueError("4 variance values must be pased, but {} values were received.".format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError("All variances must be >0, but the variances given are {}".format(variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError("You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError("You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]]], axis=-1) elif len(swap_channels) == 4: return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]], tensor[...,swap_channels[3]]], axis=-1) ############################################################################ # Build the network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) if mode == 'hardware': x1 = x conv1 = Conv2D(32, (5, 5), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1')(x1) conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')(conv1) # Tensorflow uses filter format [filter_height, filter_width, in_channels, out_channels], hence axis = 3 # conv1 = ELU(name='elu1')(conv1) conv1 = LeakyReLU(0.125)(conv1) pool1 = MaxPooling2D(pool_size=(2, 2), name='pool1')(conv1) #48 conv2 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2')(pool1) conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn2')(conv2) conv2 = LeakyReLU(0.125)(conv2) conv2 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_2')(conv2) conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn2_2')(conv2) conv2 = LeakyReLU(0.125)(conv2) pool2 = MaxPooling2D(pool_size=(2, 2), name='pool2')(conv2) # 64 conv3 = Conv2D(128, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3')(pool2) conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn3')(conv3) conv3 = LeakyReLU(0.125)(conv3) conv3 = Conv2D(128, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_2')(conv3) conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn3_2')(conv3) conv3 = LeakyReLU(0.125)(conv3) pool3 = MaxPooling2D(pool_size=(2, 2), name='pool3')(conv3) # 64 conv4 = Conv2D(256, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4')(pool3) conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn4')(conv4) conv4 = LeakyReLU(0.125)(conv4) conv4 = Conv2D(256, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_2')(conv4) conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn4_2')(conv4) conv4 = LeakyReLU(0.125)(conv4) pool4 = MaxPooling2D(pool_size=(2, 2), name='pool4')(conv4) # 48 conv5 = Conv2D(128, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5')(pool4) conv5 = BatchNormalization(axis=3, momentum=0.99, name='bn5')(conv5) conv5 = LeakyReLU(0.125)(conv5) pool5 = MaxPooling2D(pool_size=(2, 2), name='pool5')(conv5) # 48 conv6 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6')(pool5) conv6 = BatchNormalization(axis=3, momentum=0.99, name='bn6')(conv6) conv6 = LeakyReLU(0.125)(conv6) # pool6 = MaxPooling2D(pool_size=(2, 2), name='pool6')(conv6) pool6 = conv6 # 32 conv7 = Conv2D(32, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7')(pool6) conv7 = BatchNormalization(axis=3, momentum=0.99, name='bn7')(conv7) conv7 = LeakyReLU(0.125)(conv7) # The next part is to add the convolutional predictor layers on top of the base network # that we defined above. Note that I use the term "base network" differently than the paper does. # To me, the base network is everything that is not convolutional predictor layers or anchor # box layers. In this case we'll have four predictor layers, but of course you could # easily rewrite this into an arbitrarily deep base network and add an arbitrary number of # predictor layers on top of the base network by simply following the pattern shown here. # Build the convolutional predictor layers on top of conv layers 4, 5, 6, and 7. # We build two predictor layers on top of each of these layers: One for class prediction (classification), one for box coordinate prediction (localization) # We precidt `n_classes` confidence values for each box, hence the `classes` predictors have depth `n_boxes * n_classes` # We predict 4 box coordinates for each box, hence the `boxes` predictors have depth `n_boxes * 4` # Output shape of `classes`: `(batch, height, width, n_boxes * n_classes)` conv4 = Conv2D(256, (1,1))(conv4) conv5 = Conv2D(192, (1,1))(conv5) conv6 = Conv2D(128, (1,1))(conv6) conv7 = Conv2D(96, (1,1))(conv7) classes4 = Conv2D(n_boxes[0] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes4')(conv4) classes5 = Conv2D(n_boxes[1] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes5')(conv5) classes6 = Conv2D(n_boxes[2] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes6')(conv6) classes7 = Conv2D(n_boxes[3] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes7')(conv7) # Output shape of `boxes`: `(batch, height, width, n_boxes * 4)` boxes4 = Conv2D(n_boxes[0] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes4')(conv4) boxes5 = Conv2D(n_boxes[1] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes5')(conv5) boxes6 = Conv2D(n_boxes[2] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes6')(conv6) boxes7 = Conv2D(n_boxes[3] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes7')(conv7) # Generate the anchor boxes # Output shape of `anchors`: `(batch, height, width, n_boxes, 8)` anchors4 = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors4')(boxes4) anchors5 = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors5')(boxes5) anchors6 = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors6')(boxes6) anchors7 = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors7')(boxes7) # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them classes4_reshaped = Reshape((-1, n_classes), name='classes4_reshape')(classes4) classes5_reshaped = Reshape((-1, n_classes), name='classes5_reshape')(classes5) classes6_reshaped = Reshape((-1, n_classes), name='classes6_reshape')(classes6) classes7_reshaped = Reshape((-1, n_classes), name='classes7_reshape')(classes7) # Reshape the box coordinate predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss boxes4_reshaped = Reshape((-1, 4), name='boxes4_reshape')(boxes4) boxes5_reshaped = Reshape((-1, 4), name='boxes5_reshape')(boxes5) boxes6_reshaped = Reshape((-1, 4), name='boxes6_reshape')(boxes6) boxes7_reshaped = Reshape((-1, 4), name='boxes7_reshape')(boxes7) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` anchors4_reshaped = Reshape((-1, 8), name='anchors4_reshape')(anchors4) anchors5_reshaped = Reshape((-1, 8), name='anchors5_reshape')(anchors5) anchors6_reshaped = Reshape((-1, 8), name='anchors6_reshape')(anchors6) anchors7_reshaped = Reshape((-1, 8), name='anchors7_reshape')(anchors7) # Concatenate the predictions from the different layers and the assosciated anchor box tensors # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1 # Output shape of `classes_concat`: (batch, n_boxes_total, n_classes) classes_concat = Concatenate(axis=1, name='classes_concat')([classes4_reshaped, classes5_reshaped, classes6_reshaped, classes7_reshaped]) # Output shape of `boxes_concat`: (batch, n_boxes_total, 4) boxes_concat = Concatenate(axis=1, name='boxes_concat')([boxes4_reshaped, boxes5_reshaped, boxes6_reshaped, boxes7_reshaped]) # Output shape of `anchors_concat`: (batch, n_boxes_total, 8) anchors_concat = Concatenate(axis=1, name='anchors_concat')([anchors4_reshaped, anchors5_reshaped, anchors6_reshaped, anchors7_reshaped]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first classes_softmax = Activation('softmax', name='classes_softmax')(classes_concat) # Concatenate the class and box coordinate predictions and the anchors to one large predictions tensor # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')([classes_softmax, boxes_concat, anchors_concat]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections(confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast(confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'hardware': model = Model(inputs=x, outputs=[classes4,classes5,classes6,classes7,boxes4,boxes5,boxes6, boxes7]) elif mode == 'anchor': model = Model(inputs=x, outputs=[anchors4, anchors5, anchors6, anchors7]) else: raise ValueError("`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'.".format(mode)) if return_predictor_sizes: # The spatial dimensions are the same for the `classes` and `boxes` predictor layers. predictor_sizes = np.array([classes4._keras_shape[1:3], classes5._keras_shape[1:3], classes6._keras_shape[1:3], classes7._keras_shape[1:3]]) return model, predictor_sizes else: return model
def ssd_300(image_size, n_classes, input_tensor=None, mode='training', groups=3, scale_factor=1.0, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): ''' Build a Keras model with SSD300 architecture, see references. The base network is a reduced atrous VGG-16, extended by the SSD architecture, as described in the paper. Most of the arguments that this function takes are only needed for the anchor box layers. In case you're training the network, the parameters passed here must be the same as the ones used to set up `SSDBoxEncoder`. In case you're loading trained weights, the parameters passed here must be the same as the ones used to produce the trained weights. Some of these arguments are explained in more detail in the documentation of the `SSDBoxEncoder` class. Note: Requires Keras v2.0 or later. Currently works only with the TensorFlow backend (v1.0 or later). Arguments: image_size (tuple): The input image size in the format `(height, width, channels)`. input_tensor: Tensor with shape (batch, height, width, channels) n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO. mode (str, optional): One of 'training', 'inference' and 'inference_fast'. In 'training' mode, the model outputs the raw prediction tensor, while in 'inference' and 'inference_fast' modes, the raw predictions are decoded into absolute coordinates and filtered via confidence thresholding, non-maximum suppression, and top-k filtering. The difference between latter two modes is that 'inference' follows the exact procedure of the original Caffe implementation, while 'inference_fast' uses a faster prediction decoding procedure. min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. All scaling factors between the smallest and the largest will be linearly interpolated. Note that the second to last of the linearly interpolated scaling factors will actually be the scaling factor for the last predictor layer, while the last scaling factor is used for the second box for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer. This list must be one element longer than the number of predictor layers. The first `k` elements are the scaling factors for the `k` predictor layers, while the last element is used for the second box for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional last scaling factor must be passed either way, even if it is not being used. If a list is passed, this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero. aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be generated. This list is valid for all prediction layers. aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each prediction layer. This allows you to set the aspect ratios for each predictor layer individually, which is the case for the original SSD300 implementation. If a list is passed, it overrides `aspect_ratios_global`. two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise. If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated using the scaling factor for the respective layer, the second one will be generated using geometric mean of said scaling factor and next bigger scaling factor. steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over the image. If the list contains ints/floats, then that value will be used for both spatial dimensions. If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`. If no steps are provided, then they will be computed such that the anchor box center points will form an equidistant grid within the image dimensions. offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either floats or tuples of two floats. These numbers represent for each predictor layer how many pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions of the step size specified in the `steps` argument. If the list contains floats, then that value will be used for both spatial dimensions. If the list contains tuples of two floats, then they represent `(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size. clip_boxes (bool, optional): If `True`, clips the anchor box coordinates to stay within image boundaries. variances (list, optional): A list of 4 floats >0. The anchor box offset for each coordinate will be divided by its respective variance value. coords (str, optional): The box coordinate format to be used internally by the model (i.e. this is not the input format of the ground truth labels). Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width, and height), 'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`. normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates, i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates. subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values of any shape that is broadcast-compatible with the image shape. The elements of this array will be subtracted from the image pixel intensity values. For example, pass a list of three integers to perform per-channel mean normalization for color images. divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or floating point values of any shape that is broadcast-compatible with the image shape. The image pixel intensity values will be divided by the elements of this array. For example, pass a list of three integers to perform per-channel standard deviation normalization for color images. swap_channels (list, optional): Either `False` or a list of integers representing the desired order in which the input image channels should be swapped. confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific positive class in order to be considered for the non-maximum suppression stage for the respective class. A lower value will result in a larger part of the selection process being done by the non-maximum suppression stage, while a larger value will result in a larger part of the selection process happening in the confidence thresholding stage. iou_threshold (float, optional): A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold` with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers to the box's confidence score. top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the non-maximum suppression stage. nms_max_output_size (int, optional): The maximal number of predictions that will be left over after the NMS stage. return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since you can always get their sizes easily via the Keras API, but it's convenient and less error-prone to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the spatial dimensions of the predictor layers), for inference you don't need them. Returns: model: The Keras SSD300 model. predictor_sizes (optional): A Numpy array containing the `(height, width)` portion of the output tensor shape for each convolutional predictor layer. During training, the generator function needs this in order to transform the ground truth labels into tensors of identical structure as the output tensors of the model, which is in turn needed for the cost function. References: https://arxiv.org/abs/1512.02325v5 ''' n_predictor_layers = 6 # The number of predictor conv layers in the network is 6 for the original SSD300. n_classes += 1 # Account for the background class. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################# # Functions for Shufflenetv1 architeture ############################################################################# def _shuffle_unit(inputs, in_channels, out_channels, groups, bottleneck_ratio, strides=2, stage=1, block=1, linear=False): """ creates a shuffleunit Parameters ---------- inputs: Input tensor of with `channels_last` data format in_channels: number of input channels out_channels: number of output channels strides: An integer or tuple/list of 2 integers, specifying the strides of the convolution along the width and height. groups: int(1) number of groups per channel bottleneck_ratio: float bottleneck ratio implies the ratio of bottleneck channels to output channels. For example, bottleneck ratio = 1 : 4 means the output feature map is 4 times the width of the bottleneck feature map. stage: int(1) stage number block: int(1) block number Returns ------- """ if K.image_data_format() == 'channels_last': bn_axis = -1 else: bn_axis = 1 prefix = 'stage%d/block%d' % (stage, block) # default: 1/4 of the output channel of a ShuffleNet Unit bottleneck_channels = int(out_channels * bottleneck_ratio) groups = (1 if stage == 2 and block == 1 else groups) x = _group_conv(inputs, in_channels, out_channels=bottleneck_channels, groups=(1 if stage == 2 and block == 1 else groups), name='%s/1x1_gconv_1' % prefix) x = BatchNormalization(axis=bn_axis, name='%s/bn_gconv_1' % prefix)(x) x = Activation('relu', name='%s/relu_gconv_1' % prefix)(x) x = Lambda(channel_shuffle, arguments={'groups': groups}, name='%s/channel_shuffle' % prefix)(x) x = DepthwiseConv2D(kernel_size=(3, 3), padding="same", use_bias=False, strides=strides, name='%s/1x1_dwconv_1' % prefix)(x) x = BatchNormalization(axis=bn_axis, name='%s/bn_dwconv_1' % prefix)(x) x = _group_conv( x, bottleneck_channels, out_channels=out_channels if strides == 1 else out_channels - in_channels, groups=groups, name='%s/1x1_gconv_2' % prefix) x = BatchNormalization(axis=bn_axis, name='%s/bn_gconv_2' % prefix)(x) if strides < 2: ret = Add(name='%s/add' % prefix)([x, inputs]) else: avg = AveragePooling2D(pool_size=3, strides=2, padding='same', name='%s/avg_pool' % prefix)(inputs) ret = Concatenate(bn_axis, name='%s/concat' % prefix)([x, avg]) if linear: return Activation('relu', name='%s/relu_out' % prefix)(ret), ret ret = Activation('relu', name='%s/relu_out' % prefix)(ret) return ret def _group_conv(x, in_channels, out_channels, groups, kernel=1, stride=1, name=''): """ grouped convolution Parameters ---------- x: Input tensor of with `channels_last` data format in_channels: number of input channels out_channels: number of output channels groups: number of groups per channel kernel: int(1) An integer or tuple/list of 2 integers, specifying the width and height of the 2D convolution window. Can be a single integer to specify the same value for all spatial dimensions. stride: int(1) An integer or tuple/list of 2 integers, specifying the strides of the convolution along the width and height. Can be a single integer to specify the same value for all spatial dimensions. name: str A string to specifies the layer name Returns ------- """ if groups == 1: return Conv2D(filters=out_channels, kernel_size=kernel, padding='same', use_bias=False, strides=stride, name=name)(x) # number of intput channels per group ig = in_channels // groups group_list = [] assert out_channels % groups == 0 for i in range(groups): offset = i * ig group = Lambda(lambda z: z[:, :, :, offset:offset + ig], name='%s/g%d_slice' % (name, i))(x) group_list.append( Conv2D(int(0.5 + out_channels / groups), kernel_size=kernel, strides=stride, use_bias=False, padding='same', name='%s_/g%d' % (name, i))(group)) return Concatenate(name='%s/concat' % name)(group_list) def channel_shuffle(x, groups): """ Parameters ---------- x: Input tensor of with `channels_last` data format groups: int number of groups per channel Returns ------- channel shuffled output tensor Examples -------- Example for a 1D Array with 3 groups >>> d = np.array([0,1,2,3,4,5,6,7,8]) >>> x = np.reshape(d, (3,3)) >>> x = np.transpose(x, [1,0]) >>> x = np.reshape(x, (9,)) '[0 1 2 3 4 5 6 7 8] --> [0 3 6 1 4 7 2 5 8]' """ height, width, in_channels = x.shape.as_list()[1:] channels_per_group = in_channels // groups x = K.reshape(x, [-1, height, width, groups, channels_per_group]) x = K.permute_dimensions(x, (0, 1, 2, 4, 3)) # transpose x = K.reshape(x, [-1, height, width, in_channels]) return x def _conv_blockSSD(inputs, filters, block_id=11): channel_axis = -1 x = ZeroPadding2D(padding=(1, 1), name='conv_pad_%d_1' % block_id)(inputs) x = Conv2D(filters, (1, 1), padding='valid', use_bias=False, strides=(1, 1), name='conv__%d_1' % block_id)(x) x = BatchNormalization(axis=channel_axis, name='conv_%d_bn_1' % block_id)(x) x = Activation('relu', name='conv_%d_relu_1' % block_id)(x) Conv = Conv2D(filters * 2, (3, 3), padding='valid', use_bias=False, strides=(2, 2), name='conv__%d_2' % block_id)(x) x = BatchNormalization(axis=channel_axis, name='conv_%d_bn_2' % block_id)(Conv) x = Activation('relu', name='conv_%d_relu_2' % block_id)(x) return x, Conv ############################################################################ # Build the network. ############################################################################ if input_tensor != None: x = Input(tensor=input_tensor, shape=(img_height, img_width, img_channels)) else: x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) num_shuffle_units = list([3, 7, 3]) out_dim_stage_two = {1: 144, 2: 200, 3: 240, 4: 272, 8: 384} exp = np.insert(np.arange(0, len(num_shuffle_units), dtype=np.float32), 0, 0) out_channels_in_stage = 2**exp out_channels_in_stage *= out_dim_stage_two[ groups] # calculate output channels for each stage out_channels_in_stage *= scale_factor out_channels_in_stage[0] = 24 # first stage has always 24 output channels out_channels_in_stage = out_channels_in_stage.astype(int) # Get shufflenet architecture shufflenetv1 = shufflenet.ShuffleNet(groups=groups, scale_factor=scale_factor, input_shape=(img_height, img_width, img_channels), include_top=False) FeatureExtractor = Model( inputs=shufflenetv1.input, outputs=shufflenetv1.get_layer('stage3/block8/add').output) # Stage 3 last block unit shuffle_unit13 = FeatureExtractor(x1) layer = Activation('relu', name='stage3/block8/relu_out')(shuffle_unit13) layer = _shuffle_unit(layer, in_channels=out_channels_in_stage[4 - 2], out_channels=out_channels_in_stage[4 - 1], strides=2, groups=groups, bottleneck_ratio=0.25, stage=4, block=1) layer = _shuffle_unit(layer, in_channels=out_channels_in_stage[4 - 1], out_channels=out_channels_in_stage[4 - 1], strides=1, groups=groups, bottleneck_ratio=0.25, stage=4, block=2) layer = _shuffle_unit(layer, in_channels=out_channels_in_stage[4 - 1], out_channels=out_channels_in_stage[4 - 1], strides=1, groups=groups, bottleneck_ratio=0.25, stage=4, block=3) layer, shuffle_unit17 = _shuffle_unit( layer, in_channels=out_channels_in_stage[4 - 1], out_channels=out_channels_in_stage[4 - 1], strides=1, groups=groups, bottleneck_ratio=0.25, stage=4, block=4, linear=True) layer, conv18_2 = _conv_blockSSD(layer, 256, block_id=18) layer, conv19_2 = _conv_blockSSD(layer, 128, block_id=19) layer, conv20_2 = _conv_blockSSD(layer, 128, block_id=20) layer, conv21_2 = _conv_blockSSD(layer, 64, block_id=21) ### Build the convolutional predictor layers on top of the base network # We precidt `n_classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * n_classes` # Output shape of the confidence layers: `(batch, height, width, n_boxes * n_classes)` conv13_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', name='conv13_mbox_conf')(shuffle_unit13) conv17_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', name='conv17_mbox_conf')(shuffle_unit17) conv18_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', name='conv18_2_mbox_conf')(conv18_2) conv19_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', name='conv19_2_mbox_conf')(conv19_2) conv20_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', name='conv20_2_mbox_conf')(conv20_2) conv21_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', name='conv21_2_mbox_conf')(conv21_2) # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4` # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)` conv13_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', name='conv13_mbox_loc')(shuffle_unit13) conv17_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', name='conv17_mbox_loc')(shuffle_unit17) conv18_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', name='conv18_2_mbox_loc')(conv18_2) conv19_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', name='conv19_2_mbox_loc')(conv19_2) conv20_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', name='conv20_2_mbox_loc')(conv20_2) conv21_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', name='conv21_2_mbox_loc')(conv21_2) ### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) # Output shape of anchors: `(batch, height, width, n_boxes, 8)` conv13_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv13_mbox_priorbox')(conv13_mbox_loc) conv17_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv17_mbox_priorbox')(conv17_mbox_loc) conv18_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv18_2_mbox_priorbox')(conv18_2_mbox_loc) conv19_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv19_2_mbox_priorbox')(conv19_2_mbox_loc) conv20_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv20_2_mbox_priorbox')(conv20_2_mbox_loc) conv21_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv21_2_mbox_priorbox')(conv21_2_mbox_loc) ### Reshape # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv13_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv13_mbox_conf_reshape')(conv13_mbox_conf) conv17_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv17_mbox_conf_reshape')(conv17_mbox_conf) conv18_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv18_2_mbox_conf_reshape')(conv18_2_mbox_conf) conv19_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv19_2_mbox_conf_reshape')(conv19_2_mbox_conf) conv20_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv20_2_mbox_conf_reshape')(conv20_2_mbox_conf) conv21_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv21_2_mbox_conf_reshape')(conv21_2_mbox_conf) # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss conv13_mbox_loc_reshape = Reshape( (-1, 4), name='conv13_mbox_loc_reshape')(conv13_mbox_loc) conv17_mbox_loc_reshape = Reshape( (-1, 4), name='conv17_mbox_loc_reshape')(conv17_mbox_loc) conv18_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv18_2_mbox_loc_reshape')(conv18_2_mbox_loc) conv19_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv19_2_mbox_loc_reshape')(conv19_2_mbox_loc) conv20_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv20_2_mbox_loc_reshape')(conv20_2_mbox_loc) conv21_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv21_2_mbox_loc_reshape')(conv21_2_mbox_loc) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` conv13_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv13_mbox_priorbox_reshape')(conv13_mbox_priorbox) conv17_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv17_mbox_priorbox_reshape')(conv17_mbox_priorbox) conv18_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv18_2_mbox_priorbox_reshape')(conv18_2_mbox_priorbox) conv19_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv19_2_mbox_priorbox_reshape')(conv19_2_mbox_priorbox) conv20_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv20_2_mbox_priorbox_reshape')(conv20_2_mbox_priorbox) conv21_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv21_2_mbox_priorbox_reshape')(conv21_2_mbox_priorbox) ### Concatenate the predictions from the different layers # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1, the number of boxes per layer # Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([ conv13_mbox_conf_reshape, conv17_mbox_conf_reshape, conv18_2_mbox_conf_reshape, conv19_2_mbox_conf_reshape, conv20_2_mbox_conf_reshape, conv21_2_mbox_conf_reshape ]) # Output shape of `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([ conv13_mbox_loc_reshape, conv17_mbox_loc_reshape, conv18_2_mbox_loc_reshape, conv19_2_mbox_loc_reshape, conv20_2_mbox_loc_reshape, conv21_2_mbox_loc_reshape ]) # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([ conv13_mbox_priorbox_reshape, conv17_mbox_priorbox_reshape, conv18_2_mbox_priorbox_reshape, conv19_2_mbox_priorbox_reshape, conv20_2_mbox_priorbox_reshape, conv21_2_mbox_priorbox_reshape ]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate the class and box predictions and the anchors to one large predictions vector # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, #normalize_coords=normalize_coords, #change this parameter for inference normalize_coords=False, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) return model
def build_model(image_size, n_classes, mode='training', l2_regularization=0.0, min_scale=0.1, max_scale=0.9, scales=None, aspect_ratios_global=[0.5, 1.0, 2.0], aspect_ratios_per_layer=None, two_boxes_for_ar1=True, steps=None, offsets=None, clip_boxes=False, variances=[1.0, 1.0, 1.0, 1.0], coords='centroids', normalize_coords=False, subtract_mean=None, divide_by_stddev=None, swap_channels=False, confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): ''' Build a Keras model with SSD architecture, see references. The model consists of convolutional feature layers and a number of convolutional predictor layers that take their input from different feature layers. The model is fully convolutional. ''' n_predictor_layers = 4 # The number of predictor conv layers in the network n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len( variances ) != 4: # We need one variance value for each of the four box coordinates raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # Build the network. ############################################################################ base_model = MobileNet(input_shape=(img_height, img_width, img_channels), weights=None, include_top=False) base_model.load_weights("G:/keras_weights/mobilenet_1_0_224_tf_no_top.h5") #base_model.summary() x = base_model.input #base_model.layers[3].output conv4 = base_model.get_layer("conv_pw_4_relu").output conv5 = base_model.get_layer("conv_pw_6_relu").output conv6 = base_model.get_layer("conv_pw_12_relu").output conv7 = base_model.get_layer("conv_pw_13_relu").output # Build the convolutional predictor layers on top of conv layers 4, 5, 6, and 7. # We build two predictor layers on top of each of these layers: One for class prediction (classification), one for box coordinate prediction (localization) # We precidt `n_classes` confidence values for each box, hence the `classes` predictors have depth `n_boxes * n_classes` # We predict 4 box coordinates for each box, hence the `boxes` predictors have depth `n_boxes * 4` # Output shape of `classes`: `(batch, height, width, n_boxes * n_classes)` classes4 = Conv2D(n_boxes[0] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes4')(conv4) classes5 = Conv2D(n_boxes[1] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes5')(conv5) classes6 = Conv2D(n_boxes[2] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes6')(conv6) classes7 = Conv2D(n_boxes[3] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes7')(conv7) # Output shape of `boxes`: `(batch, height, width, n_boxes * 4)` boxes4 = Conv2D(n_boxes[0] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes4')(conv4) boxes5 = Conv2D(n_boxes[1] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes5')(conv5) boxes6 = Conv2D(n_boxes[2] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes6')(conv6) boxes7 = Conv2D(n_boxes[3] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes7')(conv7) # Generate the anchor boxes # Output shape of `anchors`: `(batch, height, width, n_boxes, 8)` anchors4 = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors4')(boxes4) anchors5 = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors5')(boxes5) anchors6 = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors6')(boxes6) anchors7 = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors7')(boxes7) # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them classes4_reshaped = Reshape((-1, n_classes), name='classes4_reshape')(classes4) classes5_reshaped = Reshape((-1, n_classes), name='classes5_reshape')(classes5) classes6_reshaped = Reshape((-1, n_classes), name='classes6_reshape')(classes6) classes7_reshaped = Reshape((-1, n_classes), name='classes7_reshape')(classes7) # Reshape the box coordinate predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss boxes4_reshaped = Reshape((-1, 4), name='boxes4_reshape')(boxes4) boxes5_reshaped = Reshape((-1, 4), name='boxes5_reshape')(boxes5) boxes6_reshaped = Reshape((-1, 4), name='boxes6_reshape')(boxes6) boxes7_reshaped = Reshape((-1, 4), name='boxes7_reshape')(boxes7) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` anchors4_reshaped = Reshape((-1, 8), name='anchors4_reshape')(anchors4) anchors5_reshaped = Reshape((-1, 8), name='anchors5_reshape')(anchors5) anchors6_reshaped = Reshape((-1, 8), name='anchors6_reshape')(anchors6) anchors7_reshaped = Reshape((-1, 8), name='anchors7_reshape')(anchors7) # Concatenate the predictions from the different layers and the assosciated anchor box tensors # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1 # Output shape of `classes_concat`: (batch, n_boxes_total, n_classes) classes_concat = Concatenate(axis=1, name='classes_concat')([ classes4_reshaped, classes5_reshaped, classes6_reshaped, classes7_reshaped ]) # Output shape of `boxes_concat`: (batch, n_boxes_total, 4) boxes_concat = Concatenate(axis=1, name='boxes_concat')( [boxes4_reshaped, boxes5_reshaped, boxes6_reshaped, boxes7_reshaped]) # Output shape of `anchors_concat`: (batch, n_boxes_total, 8) anchors_concat = Concatenate(axis=1, name='anchors_concat')([ anchors4_reshaped, anchors5_reshaped, anchors6_reshaped, anchors7_reshaped ]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first classes_softmax = Activation('softmax', name='classes_softmax')(classes_concat) # Concatenate the class and box coordinate predictions and the anchors to one large predictions tensor # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [classes_softmax, boxes_concat, anchors_concat]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: # The spatial dimensions are the same for the `classes` and `boxes` predictor layers. predictor_sizes = np.array([ classes4._keras_shape[1:3], classes5._keras_shape[1:3], classes6._keras_shape[1:3], classes7._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def build_model(image_size, n_classes, mode='training', l2_regularization=0.0, min_scale=0.1, max_scale=0.9, scales=None, aspect_ratios_global=[0.5, 1.0, 2.0], aspect_ratios_per_layer=None, two_boxes_for_ar1=True, steps=None, offsets=None, clip_boxes=False, variances=[1.0, 1.0, 1.0, 1.0], coords='centroids', normalize_coords=False, subtract_mean=None, divide_by_stddev=None, swap_channels=False, confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): ''' Build a Keras model with SSD architecture, see references. The model consists of convolutional feature layers and a number of convolutional predictor layers that take their input from different feature layers. The model is fully convolutional. The implementation found here is a smaller version of the original architecture used in the paper (where the base network consists of a modified VGG-16 extended by a few convolutional feature layers), but of course it could easily be changed to an arbitrarily large SSD architecture by following the general design pattern used here. This implementation has 7 convolutional layers and 4 convolutional predictor layers that take their input from layers 4, 5, 6, and 7, respectively. Most of the arguments that this function takes are only needed for the anchor box layers. In case you're training the network, the parameters passed here must be the same as the ones used to set up `SSDBoxEncoder`. In case you're loading trained weights, the parameters passed here must be the same as the ones used to produce the trained weights. Some of these arguments are explained in more detail in the documentation of the `SSDBoxEncoder` class. Note: Requires Keras v2.0 or later. Training currently works only with the TensorFlow backend (v1.0 or later). Arguments: image_size (tuple): The input image size in the format `(height, width, channels)`. n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO. mode (str, optional): One of 'training', 'inference' and 'inference_fast'. In 'training' mode, the model outputs the raw prediction tensor, while in 'inference' and 'inference_fast' modes, the raw predictions are decoded into absolute coordinates and filtered via confidence thresholding, non-maximum suppression, and top-k filtering. The difference between latter two modes is that 'inference' follows the exact procedure of the original Caffe implementation, while 'inference_fast' uses a faster prediction decoding procedure. l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers. min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. All scaling factors between the smallest and the largest will be linearly interpolated. Note that the second to last of the linearly interpolated scaling factors will actually be the scaling factor for the last predictor layer, while the last scaling factor is used for the second box for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer. This list must be one element longer than the number of predictor layers. The first `k` elements are the scaling factors for the `k` predictor layers, while the last element is used for the second box for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional last scaling factor must be passed either way, even if it is not being used. If a list is passed, this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero. aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be generated. This list is valid for all predictor layers. The original implementation uses more aspect ratios for some predictor layers and fewer for others. If you want to do that, too, then use the next argument instead. aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each predictor layer. This allows you to set the aspect ratios for each predictor layer individually. If a list is passed, it overrides `aspect_ratios_global`. two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise. If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated using the scaling factor for the respective layer, the second one will be generated using geometric mean of said scaling factor and next bigger scaling factor. steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over the image. If the list contains ints/floats, then that value will be used for both spatial dimensions. If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`. If no steps are provided, then they will be computed such that the anchor box center points will form an equidistant grid within the image dimensions. offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either floats or tuples of two floats. These numbers represent for each predictor layer how many pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions of the step size specified in the `steps` argument. If the list contains floats, then that value will be used for both spatial dimensions. If the list contains tuples of two floats, then they represent `(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size, which is also the recommended setting. clip_boxes (bool, optional): If `True`, clips the anchor box coordinates to stay within image boundaries. variances (list, optional): A list of 4 floats >0. The anchor box offset for each coordinate will be divided by its respective variance value. coords (str, optional): The box coordinate format to be used internally by the model (i.e. this is not the input format of the ground truth labels). Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width, and height), 'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`. normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates, i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates. subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values of any shape that is broadcast-compatible with the image shape. The elements of this array will be subtracted from the image pixel intensity values. For example, pass a list of three integers to perform per-channel mean normalization for color images. divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or floating point values of any shape that is broadcast-compatible with the image shape. The image pixel intensity values will be divided by the elements of this array. For example, pass a list of three integers to perform per-channel standard deviation normalization for color images. swap_channels (list, optional): Either `False` or a list of integers representing the desired order in which the input image channels should be swapped. confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific positive class in order to be considered for the non-maximum suppression stage for the respective class. A lower value will result in a larger part of the selection process being done by the non-maximum suppression stage, while a larger value will result in a larger part of the selection process happening in the confidence thresholding stage. iou_threshold (float, optional): A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold` with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers to the box's confidence score. top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the non-maximum suppression stage. nms_max_output_size (int, optional): The maximal number of predictions that will be left over after the NMS stage. return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since you can always get their sizes easily via the Keras API, but it's convenient and less error-prone to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the spatial dimensions of the predictor layers), for inference you don't need them. Returns: model: The Keras SSD model. predictor_sizes (optional): A Numpy array containing the `(height, width)` portion of the output tensor shape for each convolutional predictor layer. During training, the generator function needs this in order to transform the ground truth labels into tensors of identical structure as the output tensors of the model, which is in turn needed for the cost function. References: https://arxiv.org/abs/1512.02325v5 ''' n_predictor_layers = 4 # The number of predictor conv layers in the network n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len( variances ) != 4: # We need one variance value for each of the four box coordinates raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # Build the network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) conv1 = Conv2D(32, (5, 5), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1')(x1) conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')( conv1 ) # Tensorflow uses filter format [filter_height, filter_width, in_channels, out_channels], hence axis = 3 conv1 = ELU(name='elu1')(conv1) pool1 = MaxPooling2D(pool_size=(2, 2), name='pool1')(conv1) conv2 = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2')(pool1) conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn2')(conv2) conv2 = ELU(name='elu2')(conv2) pool2 = MaxPooling2D(pool_size=(2, 2), name='pool2')(conv2) conv3 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3')(pool2) conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn3')(conv3) conv3 = ELU(name='elu3')(conv3) pool3 = MaxPooling2D(pool_size=(2, 2), name='pool3')(conv3) conv4 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4')(pool3) conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn4')(conv4) conv4 = ELU(name='elu4')(conv4) pool4 = MaxPooling2D(pool_size=(2, 2), name='pool4')(conv4) conv5 = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5')(pool4) conv5 = BatchNormalization(axis=3, momentum=0.99, name='bn5')(conv5) conv5 = ELU(name='elu5')(conv5) pool5 = MaxPooling2D(pool_size=(2, 2), name='pool5')(conv5) conv6 = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6')(pool5) conv6 = BatchNormalization(axis=3, momentum=0.99, name='bn6')(conv6) conv6 = ELU(name='elu6')(conv6) pool6 = MaxPooling2D(pool_size=(2, 2), name='pool6')(conv6) conv7 = Conv2D(32, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7')(pool6) conv7 = BatchNormalization(axis=3, momentum=0.99, name='bn7')(conv7) conv7 = ELU(name='elu7')(conv7) # The next part is to add the convolutional predictor layers on top of the base network # that we defined above. Note that I use the term "base network" differently than the paper does. # To me, the base network is everything that is not convolutional predictor layers or anchor # box layers. In this case we'll have four predictor layers, but of course you could # easily rewrite this into an arbitrarily deep base network and add an arbitrary number of # predictor layers on top of the base network by simply following the pattern shown here. # Build the convolutional predictor layers on top of conv layers 4, 5, 6, and 7. # We build two predictor layers on top of each of these layers: One for class prediction (classification), one for box coordinate prediction (localization) # We precidt `n_classes` confidence values for each box, hence the `classes` predictors have depth `n_boxes * n_classes` # We predict 4 box coordinates for each box, hence the `boxes` predictors have depth `n_boxes * 4` # Output shape of `classes`: `(batch, height, width, n_boxes * n_classes)` classes4 = Conv2D(n_boxes[0] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes4')(conv4) classes5 = Conv2D(n_boxes[1] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes5')(conv5) classes6 = Conv2D(n_boxes[2] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes6')(conv6) classes7 = Conv2D(n_boxes[3] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes7')(conv7) # Output shape of `boxes`: `(batch, height, width, n_boxes * 4)` boxes4 = Conv2D(n_boxes[0] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes4')(conv4) boxes5 = Conv2D(n_boxes[1] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes5')(conv5) boxes6 = Conv2D(n_boxes[2] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes6')(conv6) boxes7 = Conv2D(n_boxes[3] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes7')(conv7) # Generate the anchor boxes # Output shape of `anchors`: `(batch, height, width, n_boxes, 8)` anchors4 = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors4')(boxes4) anchors5 = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors5')(boxes5) anchors6 = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors6')(boxes6) anchors7 = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors7')(boxes7) # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them classes4_reshaped = Reshape((-1, n_classes), name='classes4_reshape')(classes4) classes5_reshaped = Reshape((-1, n_classes), name='classes5_reshape')(classes5) classes6_reshaped = Reshape((-1, n_classes), name='classes6_reshape')(classes6) classes7_reshaped = Reshape((-1, n_classes), name='classes7_reshape')(classes7) # Reshape the box coordinate predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss boxes4_reshaped = Reshape((-1, 4), name='boxes4_reshape')(boxes4) boxes5_reshaped = Reshape((-1, 4), name='boxes5_reshape')(boxes5) boxes6_reshaped = Reshape((-1, 4), name='boxes6_reshape')(boxes6) boxes7_reshaped = Reshape((-1, 4), name='boxes7_reshape')(boxes7) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` anchors4_reshaped = Reshape((-1, 8), name='anchors4_reshape')(anchors4) anchors5_reshaped = Reshape((-1, 8), name='anchors5_reshape')(anchors5) anchors6_reshaped = Reshape((-1, 8), name='anchors6_reshape')(anchors6) anchors7_reshaped = Reshape((-1, 8), name='anchors7_reshape')(anchors7) # Concatenate the predictions from the different layers and the assosciated anchor box tensors # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1 # Output shape of `classes_concat`: (batch, n_boxes_total, n_classes) classes_concat = Concatenate(axis=1, name='classes_concat')([ classes4_reshaped, classes5_reshaped, classes6_reshaped, classes7_reshaped ]) # Output shape of `boxes_concat`: (batch, n_boxes_total, 4) boxes_concat = Concatenate(axis=1, name='boxes_concat')( [boxes4_reshaped, boxes5_reshaped, boxes6_reshaped, boxes7_reshaped]) # Output shape of `anchors_concat`: (batch, n_boxes_total, 8) anchors_concat = Concatenate(axis=1, name='anchors_concat')([ anchors4_reshaped, anchors5_reshaped, anchors6_reshaped, anchors7_reshaped ]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first classes_softmax = Activation('softmax', name='classes_softmax')(classes_concat) # Concatenate the class and box coordinate predictions and the anchors to one large predictions tensor # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [classes_softmax, boxes_concat, anchors_concat]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetections2( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: # The spatial dimensions are the same for the `classes` and `boxes` predictor layers. predictor_sizes = np.array([ classes4._keras_shape[1:3], classes5._keras_shape[1:3], classes6._keras_shape[1:3], classes7._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def body(features): with tf.variable_scope("SSDGraph"): with ipu.scopes.ipu_scope('/device:IPU:0'): # conv 1 block conv1_1 = layers.conv(features, ksize=3, stride=1, filters_out=64, name="conv1_1") conv1_1 = layers.relu(conv1_1) conv1_2 = layers.conv(conv1_1, ksize=3, stride=1, filters_out=64, name="conv1_2") conv1_2 = layers.relu(conv1_2) pool1 = layers.maxpool(conv1_2, size=2, stride=2) # conv 2 block conv2_1 = layers.conv(pool1, ksize=3, stride=1, filters_out=128, name="conv2_1") conv2_1 = layers.relu(conv2_1) conv2_2 = layers.conv(conv2_1, ksize=3, stride=1, filters_out=128, name="conv2_2") conv2_2 = layers.relu(conv2_2) pool2 = layers.maxpool(conv2_2, size=2, stride=2) # conv 3 block conv3_1 = layers.conv(pool2, ksize=3, stride=1, filters_out=256, name="conv3_1") conv3_1 = layers.relu(conv3_1) conv3_2 = layers.conv(conv3_1, ksize=3, stride=1, filters_out=256, name="conv3_2") conv3_2 = layers.relu(conv3_2) conv3_3 = layers.conv(conv3_2, ksize=3, stride=1, filters_out=256, name="conv3_3") conv3_3 = layers.relu(conv3_3) pool3 = layers.maxpool(conv3_3, size=2, stride=2) # conv 4 block conv4_1 = layers.conv(pool3, ksize=3, stride=1, filters_out=512, name="conv4_1") conv4_1 = layers.relu(conv4_1) conv4_2 = layers.conv(conv4_1, ksize=3, stride=1, filters_out=512, name="conv4_2") conv4_2 = layers.relu(conv4_2) conv4_3 = layers.conv(conv4_2, ksize=3, stride=1, filters_out=512, name="conv4_3") conv4_3 = layers.relu( conv4_3 ) # feature map to be used for object detection/classification pool4 = layers.maxpool(conv4_3, size=2, stride=2) # conv 5 block conv5_1 = layers.conv(pool4, ksize=3, stride=1, filters_out=512, name="conv5_1") conv5_1 = layers.relu(conv5_1) conv5_2 = layers.conv(conv5_1, ksize=3, stride=1, filters_out=512, name="conv5_2") conv5_2 = layers.relu(conv5_2) conv5_3 = layers.conv(conv5_2, ksize=3, stride=1, filters_out=512, name="conv5_3") conv5_3 = layers.relu(conv5_3) pool5 = layers.maxpool(conv5_3, size=3, stride=1) # END VGG # Extra feature layers # fc6 fc6 = layers.conv(pool5, ksize=3, dilation_rate=(6, 6), stride=1, filters_out=1024, name="fc6") fc6 = layers.relu(fc6) # fc7 fc7 = layers.conv(fc6, ksize=1, stride=1, filters_out=1024, name="fc7") fc7 = layers.relu( fc7 ) # feature map to be used for object detection/classification # conv 6 block conv6_1 = layers.conv(fc7, ksize=1, stride=1, filters_out=256, name="conv6_1") conv6_1 = layers.relu(conv6_1) conv6_1 = tf.pad(conv6_1, paddings=([[0, 0], [1, 1], [1, 1], [0, 0]]), name='conv6_padding') conv6_2 = layers.conv(conv6_1, ksize=3, stride=2, filters_out=512, padding='VALID', name="conv6_2") conv6_2 = layers.relu( conv6_2 ) # feature map to be used for object detection/classification # conv 7 block conv7_1 = layers.conv(conv6_2, ksize=1, stride=1, filters_out=128, name="conv7_1") conv7_1 = layers.relu(conv7_1) conv7_1 = tf.pad(conv7_1, paddings=([[0, 0], [1, 1], [1, 1], [0, 0]]), name='conv7_padding') conv7_2 = layers.conv(conv7_1, ksize=3, stride=2, filters_out=256, padding='VALID', name="conv7_2") conv7_2 = layers.relu( conv7_2 ) # feature map to be used for object detection/classification # conv 8 block conv8_1 = layers.conv(conv7_2, ksize=1, stride=1, filters_out=128, name="conv8_1") conv8_1 = layers.relu(conv8_1) conv8_2 = layers.conv(conv8_1, ksize=3, stride=1, filters_out=256, padding='VALID', name="conv8_2") conv8_2 = layers.relu( conv8_2 ) # feature map to be used for object detection/classification # conv 9 block conv9_1 = layers.conv(conv8_2, ksize=1, stride=1, filters_out=128, name="conv9_1") conv9_1 = layers.relu(conv9_1) conv9_2 = layers.conv(conv9_1, ksize=3, stride=1, filters_out=256, padding='VALID', name="conv9_2") conv9_2 = layers.relu( conv9_2 ) # feature map to be used for object detection/classification # Perform L2 normalization on conv4_3 conv4_3_norm = tf.math.l2_normalize(conv4_3, axis=3) # Conv confidence predictors have output depth N_BOXES * N_CLASSES conv4_3_norm_mbox_conf = layers.conv( conv4_3_norm, ksize=3, stride=1, filters_out=N_BOXES[0] * N_CLASSES, name='conv4_3_norm_mbox_conf') fc7_mbox_conf = layers.conv(fc7, ksize=3, stride=1, filters_out=N_BOXES[1] * N_CLASSES, name='fc7_mbox_conf') conv6_2_mbox_conf = layers.conv(conv6_2, ksize=3, stride=1, filters_out=N_BOXES[2] * N_CLASSES, name='conv6_2_mbox_conf') conv7_2_mbox_conf = layers.conv(conv7_2, ksize=3, stride=1, filters_out=N_BOXES[3] * N_CLASSES, name='conv7_2_mbox_conf') conv8_2_mbox_conf = layers.conv(conv8_2, ksize=3, stride=1, filters_out=N_BOXES[4] * N_CLASSES, name='conv8_2_mbox_conf') conv9_2_mbox_conf = layers.conv(conv9_2, ksize=3, stride=1, filters_out=N_BOXES[5] * N_CLASSES, name='conv9_2_mbox_conf') # Conv box location predictors have output depth N_BOXES * 4 (box coordinates) conv4_3_norm_mbox_loc = layers.conv( conv4_3_norm, ksize=3, stride=1, filters_out=N_BOXES[0] * 4, name='conv4_3_norm_mbox_loc') fc7_mbox_loc = layers.conv(fc7, ksize=3, stride=1, filters_out=N_BOXES[1] * 4, name='fc7_mbox_loc') conv6_2_mbox_loc = layers.conv(conv6_2, ksize=3, stride=1, filters_out=N_BOXES[2] * 4, name='conv6_2_mbox_loc') conv7_2_mbox_loc = layers.conv(conv7_2, ksize=3, stride=1, filters_out=N_BOXES[3] * 4, name='conv7_2_mbox_loc') conv8_2_mbox_loc = layers.conv(conv8_2, ksize=3, stride=1, filters_out=N_BOXES[4] * 4, name='conv8_2_mbox_loc') conv9_2_mbox_loc = layers.conv(conv9_2, ksize=3, stride=1, filters_out=N_BOXES[5] * 4, name='conv9_2_mbox_loc') # Generate the anchor boxes conv4_3_norm_mbox_priorbox = AnchorBoxes( HEIGHT, WIDTH, this_scale=SCALES[0], next_scale=SCALES[1], two_boxes_for_ar1=True, this_steps=STEPS[0], this_offsets=OFFSETS[0], clip_boxes=False, variances=VARIANCES, aspect_ratios=ASPECT_RATIOS_PER_LAYER[0], normalize_coords=True, name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc) fc7_mbox_priorbox = AnchorBoxes( HEIGHT, WIDTH, this_scale=SCALES[1], next_scale=SCALES[2], two_boxes_for_ar1=True, this_steps=STEPS[1], this_offsets=OFFSETS[1], clip_boxes=False, variances=VARIANCES, aspect_ratios=ASPECT_RATIOS_PER_LAYER[1], normalize_coords=True, name='fc7_mbox_priorbox')(fc7_mbox_loc) conv6_2_mbox_priorbox = AnchorBoxes( HEIGHT, WIDTH, this_scale=SCALES[2], next_scale=SCALES[3], two_boxes_for_ar1=True, this_steps=STEPS[2], this_offsets=OFFSETS[2], clip_boxes=False, variances=VARIANCES, aspect_ratios=ASPECT_RATIOS_PER_LAYER[2], normalize_coords=True, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes( HEIGHT, WIDTH, this_scale=SCALES[3], next_scale=SCALES[4], two_boxes_for_ar1=True, this_steps=STEPS[3], this_offsets=OFFSETS[3], clip_boxes=False, variances=VARIANCES, aspect_ratios=ASPECT_RATIOS_PER_LAYER[3], normalize_coords=True, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes( HEIGHT, WIDTH, this_scale=SCALES[4], next_scale=SCALES[5], two_boxes_for_ar1=True, this_steps=STEPS[4], this_offsets=OFFSETS[4], clip_boxes=False, variances=VARIANCES, aspect_ratios=ASPECT_RATIOS_PER_LAYER[4], normalize_coords=True, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes( HEIGHT, WIDTH, this_scale=SCALES[5], next_scale=SCALES[6], two_boxes_for_ar1=True, this_steps=STEPS[5], this_offsets=OFFSETS[5], clip_boxes=False, variances=VARIANCES, aspect_ratios=ASPECT_RATIOS_PER_LAYER[5], normalize_coords=True, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) # Reshape class predictions conv4_3_norm_mbox_conf_reshape = tf.reshape( conv4_3_norm_mbox_conf, shape=(-1, conv4_3_norm_mbox_conf.shape[1] * conv4_3_norm_mbox_conf.shape[2] * N_BOXES[0], N_CLASSES), name='conv4_3_norm_mbox_conf_reshape') fc7_mbox_conf_reshape = tf.reshape( fc7_mbox_conf, shape=(-1, fc7_mbox_conf.shape[1] * fc7_mbox_conf.shape[2] * N_BOXES[1], N_CLASSES), name='fc7_mbox_conf_reshape') conv6_2_mbox_conf_reshape = tf.reshape( conv6_2_mbox_conf, shape=(-1, conv6_2_mbox_conf.shape[1] * conv6_2_mbox_conf.shape[2] * N_BOXES[2], N_CLASSES), name='conv6_2_mbox_conf_reshape') conv7_2_mbox_conf_reshape = tf.reshape( conv7_2_mbox_conf, shape=(-1, conv7_2_mbox_conf.shape[1] * conv7_2_mbox_conf.shape[2] * N_BOXES[3], N_CLASSES), name='conv7_2_mbox_conf_reshape') conv8_2_mbox_conf_reshape = tf.reshape( conv8_2_mbox_conf, shape=(-1, conv8_2_mbox_conf.shape[1] * conv8_2_mbox_conf.shape[2] * N_BOXES[4], N_CLASSES), name='conv8_2_mbox_conf_reshape') conv9_2_mbox_conf_reshape = tf.reshape( conv9_2_mbox_conf, shape=(-1, conv9_2_mbox_conf.shape[1] * conv9_2_mbox_conf.shape[2] * N_BOXES[5], N_CLASSES), name='conv9_2_mbox_conf_reshape') # Reshape box location predictions conv4_3_norm_mbox_loc_reshape = tf.reshape( conv4_3_norm_mbox_loc, shape=(-1, conv4_3_norm_mbox_loc.shape[1] * conv4_3_norm_mbox_loc.shape[2] * N_BOXES[0], 4), name='conv4_3_norm_mbox_loc_reshape') fc7_mbox_loc_reshape = tf.reshape( fc7_mbox_loc, shape=(-1, fc7_mbox_loc.shape[1] * fc7_mbox_loc.shape[2] * N_BOXES[1], 4), name='fc7_mbox_loc_reshape') conv6_2_mbox_loc_reshape = tf.reshape( conv6_2_mbox_loc, shape=(-1, conv6_2_mbox_loc.shape[1] * conv6_2_mbox_loc.shape[2] * N_BOXES[2], 4), name='conv6_2_mbox_loc_reshape') conv7_2_mbox_loc_reshape = tf.reshape( conv7_2_mbox_loc, shape=(-1, conv7_2_mbox_loc.shape[1] * conv7_2_mbox_loc.shape[2] * N_BOXES[3], 4), name='conv7_2_mbox_loc_reshape') conv8_2_mbox_loc_reshape = tf.reshape( conv8_2_mbox_loc, shape=(-1, conv8_2_mbox_loc.shape[1] * conv8_2_mbox_loc.shape[2] * N_BOXES[4], 4), name='conv8_2_mbox_loc_reshape') conv9_2_mbox_loc_reshape = tf.reshape( conv9_2_mbox_loc, shape=(-1, conv9_2_mbox_loc.shape[1] * conv9_2_mbox_loc.shape[2] * N_BOXES[5], 4), name='conv9_2_mbox_loc_reshape') # Reshape anchor box tensors conv4_3_norm_mbox_priorbox_reshape = tf.reshape( conv4_3_norm_mbox_priorbox, shape=(-1, conv4_3_norm_mbox_priorbox.shape[1] * conv4_3_norm_mbox_priorbox.shape[2] * N_BOXES[0], 8), name='conv4_3_norm_mbox_priorbox_reshape') fc7_mbox_priorbox_reshape = tf.reshape( fc7_mbox_priorbox, shape=(-1, fc7_mbox_priorbox.shape[1] * fc7_mbox_priorbox.shape[2] * N_BOXES[1], 8), name='fc7_mbox_priorbox_reshape') conv6_2_mbox_priorbox_reshape = tf.reshape( conv6_2_mbox_priorbox, shape=(-1, conv6_2_mbox_priorbox.shape[1] * conv6_2_mbox_priorbox.shape[2] * N_BOXES[2], 8), name='conv6_2_mbox_priorbox_reshape') conv7_2_mbox_priorbox_reshape = tf.reshape( conv7_2_mbox_priorbox, shape=(-1, conv7_2_mbox_priorbox.shape[1] * conv7_2_mbox_priorbox.shape[2] * N_BOXES[3], 8), name='conv7_2_mbox_priorbox_reshape') conv8_2_mbox_priorbox_reshape = tf.reshape( conv8_2_mbox_priorbox, shape=(-1, conv8_2_mbox_priorbox.shape[1] * conv8_2_mbox_priorbox.shape[2] * N_BOXES[4], 8), name='conv8_2_mbox_priorbox_reshape') conv9_2_mbox_priorbox_reshape = tf.reshape( conv9_2_mbox_priorbox, shape=(-1, conv9_2_mbox_priorbox.shape[1] * conv9_2_mbox_priorbox.shape[2] * N_BOXES[5], 8), name='conv9_2_mbox_priorbox_reshape') # Concatenate predictions from different layers mbox_conf = tf.concat([ conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape ], axis=1, name='mbox_conf') mbox_loc = tf.concat([ conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape ], axis=1, name='mbox_loc') mbox_priorbox = tf.concat([ conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape ], axis=1, name='mbox_priorbox') # Softmax activation layer mbox_conf_softmax = tf.nn.softmax(mbox_conf, name='mbox_conf_softmax') predictions = tf.concat( [mbox_conf_softmax, mbox_loc, mbox_priorbox], axis=2, name='predictions') # Output outfeed = outfeed_queue.enqueue(predictions) return outfeed
def SSD(image_size, n_classes, l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[1.0], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.75, iou_threshold=0.1, top_k='all', nms_max_output_size=400, return_predictor_sizes=False): # Pre-processes n_predictor_layers = 6 n_classes += 1 l2_reg = l2_regularization img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2] if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError("`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified.") if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError("It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}.".format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError("Either `min_scale` and `max_scale` or `scales` need to be specified.") if scales: if len(scales) != n_predictor_layers+1: raise ValueError("It must be either scales is None or len(scales) == {}, but len(scales) == {}.".format(n_predictor_layers+1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers+1) if len(variances) != 4: raise ValueError("4 variance values must be pased, but {} values were received.".format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError("All variances must be >0, but the variances given are {}".format(variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError("You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError("You must provide at least one offset value per predictor layer.") # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]]], axis=-1) elif len(swap_channels) == 4: return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]], tensor[...,swap_channels[3]]], axis=-1) # Build the model x = Input(shape=(img_height, img_width, img_channels)) x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) L256 = Conv2D(64, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(x1) L256 = Conv2D(64, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L256) pool1 =MaxPooling2D(pool_size=(2,2))(L256) L128 = Conv2D(128, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(pool1) L128 = Conv2D(128, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L128) L128 = Conv2D(128, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L128) pool2 =MaxPooling2D(pool_size=(2,2))(L128) L64 = Conv2D(256, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(pool2) L64 = Conv2D(256, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L64) L64 = Conv2D(256, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L64) pool3 =MaxPooling2D(pool_size=(2,2))(L64) L32 = Conv2D(1024, (3,3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(pool3) L32 = Conv2D(1024, (1,1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L32) L16 = Conv2D(256, (1,1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L32) L16 = ZeroPadding2D(padding=((1,1), (1,1)))(L16) L16 = Conv2D(512, (3,3), strides=(2,2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L16) L8 = Conv2D(128, (1,1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L16) L8 = ZeroPadding2D(padding=((1,1), (1,1)))(L8) L8 = Conv2D(256, (3,3), strides=(2,2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L8) L4 = Conv2D(128, (1,1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L8) L4 = ZeroPadding2D(padding=((1,1), (1,1)))(L4) L4 = Conv2D(256, (3,3), strides=(2,2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L4) # L2 normalization L128_norm = L2Normalization(gamma_init=10)(L128) L64_norm = L2Normalization(gamma_init=5)(L64) # Prediction layers: confidence, location, and anchor boxes L128_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L128_mbox_conf')(L128_norm) L64_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L64_mbox_conf')(L64_norm) L32_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L32_mbox_conf')(L32) L16_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L16_mbox_conf')(L16) L8_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L8_mbox_conf')(L8) L4_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L4_mbox_conf')(L4) L128_mbox_loc = Conv2D(n_boxes[0] * 4, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L128_mbox_loc')(L128_norm) L64_mbox_loc = Conv2D(n_boxes[1] * 4, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L64_mbox_loc')(L64_norm) L32_mbox_loc = Conv2D(n_boxes[2] * 4, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L32_mbox_loc')(L32) L16_mbox_loc = Conv2D(n_boxes[3] * 4, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L16_mbox_loc')(L16) L8_mbox_loc = Conv2D(n_boxes[4] * 4, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L8_mbox_loc')(L8) L4_mbox_loc = Conv2D(n_boxes[5] * 4, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L4_mbox_loc')(L4) L128_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L128_mbox_priorbox')(L128_mbox_loc) L64_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L64_mbox_priorbox')(L64_mbox_loc) L32_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L32_mbox_priorbox')(L32_mbox_loc) L16_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L16_mbox_priorbox')(L16_mbox_loc) L8_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L8_mbox_priorbox')(L8_mbox_loc) L4_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L4_mbox_priorbox')(L4_mbox_loc) # Reshape and concatenate L128_mbox_conf_reshape = Reshape((-1, n_classes), name='L128_mbox_conf_reshape')(L128_mbox_conf) L64_mbox_conf_reshape = Reshape((-1, n_classes), name='L64_mbox_conf_reshape')(L64_mbox_conf) L32_mbox_conf_reshape = Reshape((-1, n_classes), name='L32_mbox_conf_reshape')(L32_mbox_conf) L16_mbox_conf_reshape = Reshape((-1, n_classes), name='L16_mbox_conf_reshape')(L16_mbox_conf) L8_mbox_conf_reshape = Reshape((-1, n_classes), name='L8_mbox_conf_reshape')(L8_mbox_conf) L4_mbox_conf_reshape = Reshape((-1, n_classes), name='L4_mbox_conf_reshape')(L4_mbox_conf) L128_mbox_loc_reshape = Reshape((-1, 4), name='L128_mbox_loc_reshape')(L128_mbox_loc) L64_mbox_loc_reshape = Reshape((-1, 4), name='L64_mbox_loc_reshape')(L64_mbox_loc) L32_mbox_loc_reshape = Reshape((-1, 4), name='L32_mbox_loc_reshape')(L32_mbox_loc) L16_mbox_loc_reshape = Reshape((-1, 4), name='L16_mbox_loc_reshape')(L16_mbox_loc) L8_mbox_loc_reshape = Reshape((-1, 4), name='L8_mbox_loc_reshape')(L8_mbox_loc) L4_mbox_loc_reshape = Reshape((-1, 4), name='L4_mbox_loc_reshape')(L4_mbox_loc) L128_mbox_priorbox_reshape = Reshape((-1, 8), name='L128_mbox_priorbox_reshape')(L128_mbox_priorbox) L64_mbox_priorbox_reshape = Reshape((-1, 8), name='L64_mbox_priorbox_reshape')(L64_mbox_priorbox) L32_mbox_priorbox_reshape = Reshape((-1, 8), name='L32_mbox_priorbox_reshape')(L32_mbox_priorbox) L16_mbox_priorbox_reshape = Reshape((-1, 8), name='L16_mbox_priorbox_reshape')(L16_mbox_priorbox) L8_mbox_priorbox_reshape = Reshape((-1, 8), name='L8_mbox_priorbox_reshape')(L8_mbox_priorbox) L4_mbox_priorbox_reshape = Reshape((-1, 8), name='L4_mbox_priorbox_reshape')(L4_mbox_priorbox) mbox_conf = Concatenate(axis=1, name='mbox_conf')([L128_mbox_conf_reshape, L64_mbox_conf_reshape, L32_mbox_conf_reshape, L16_mbox_conf_reshape, L8_mbox_conf_reshape, L4_mbox_conf_reshape]) mbox_loc = Concatenate(axis=1, name='mbox_loc')([L128_mbox_loc_reshape, L64_mbox_loc_reshape, L32_mbox_loc_reshape, L16_mbox_loc_reshape, L8_mbox_loc_reshape, L4_mbox_loc_reshape]) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([L128_mbox_priorbox_reshape, L64_mbox_priorbox_reshape, L32_mbox_priorbox_reshape, L16_mbox_priorbox_reshape, L8_mbox_priorbox_reshape, L4_mbox_priorbox_reshape]) # Softmax loss, concatenation and model return mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) predictions = Concatenate(axis=2, name='predictions')([mbox_conf_softmax, mbox_loc, mbox_priorbox]) model = Model(inputs=x, outputs=predictions) if return_predictor_sizes: predictor_sizes = np.array([L128_mbox_conf._keras_shape[1:3], L64_mbox_conf._keras_shape[1:3], L32_mbox_conf._keras_shape[1:3], L16_mbox_conf._keras_shape[1:3], L8_mbox_conf._keras_shape[1:3], L4_mbox_conf._keras_shape[1:3]]) return model, predictor_sizes else: return model
def ssd_300(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): ''' ############################################################################# # Build model Keras voi kien truc SSD300, xem paper de hieu ve kien truc ############################################################################# Base network la mang VGG-16 duoc luoc bo cac layer classification, va model duoc mo rong bang kien truc SSD, nhu duoc mo ta trong paper. Hau het cac doi so ma function nay can duoc su dung cho viec cau hinh cac anchor box layers. Trong truong hop ban thuc hien trainning mang, cac tham so duoc truyen o day phai giong voi cac tham so duoc su dung de thiet lap `SSDBoxEncoder`. Trong truong hop ban tai trong so da duoc dao tao truoc, cac tham so duoc truyen vao o day phai giong voi cac tham so da tao ra duoc cac trong so duoc dao tao truoc do. Mot so trong so cac doi so nay se duoc giai thich chi tiet hon trong tai lieu cua class `SSDBoxEncoder` Luu y: Yeu cau Keras tu v2.0 tro len, va hien tai chi hoat dong voi backend Tensorflow (v1.0) tro len. ############################################################################################ # Giai thich chi tiet cac doi so duoc truyen vao function ########################################################################################### Cac doi so: * image_size (tuple): - Kich thuoc hinh anh dau vao voi dinh dang `(height, width, channels)`. * n_classes (int): - So luong cac class positives (cac class object, ko phai class background) vi du: 20 cho Pascal VOC, 80 cho MS COCO. * mode (str, optional): - Mot trong cac gia tri 'training', 'inference' va 'inference_fast'. - Trong mode 'trainning', output cua model se la raw predict tensor, trong khi do, trong mode 'inference' va 'inference_fast' cac raw predict tensor se duoc decoded (giai ma) thanh cac gia tri toa do tuyet doi, va duoc loc qua cac nguong tin cay (confidence thresholding), non-maximum suppression, va top-k filtering. - Su khac biet giua hai mode 'inference' va 'inference_fast' do la 'inference' tuan theo quy trinh theo trien khai Caffe ban dau, trong khi do mode 'inference_fast' su dung thu tuc decoded cac prediction nhanh hon. * l2_regularization (float, optional): - Ty le chuan hoa L2. Ap dung cho tat ca cac layer convolutional. Dat thanh 0 de huy bo viec su dung chuan hoa L2. * min_scale (float, optional): - He so ty le nho nhat cho cac anchor box va no thuoc layer predict thap nhat. * max_scale (float, optional): - He so ty le lon nhat cho cac anchor box va no thuoc layer predict cao nhat. - Tat ca cac he so ty le cho cac anchor box cua cac layer predict o giua hai lop predict cao nhat va thap nhat se duoc noi suy tuyen tinh trong doan tu [min_scale, max_scale] - Ghi nho rang, he so ty le duoc noi suy tuyen tinh tu thu 2 den cuoi cung se thuc su su la he so ty le cho layer predict cuoi cung, trong khi he so ty le cuoi cung se duoc su dung cho second box cua aspect ratios = 1 trong layer predict cuoi cung, neu `two_boxes_for_ar1` duoc dat la `True`. * scales (list, optional): - Mot list cac phan tu type la float chua cac he so ty le duoc ap dung cho tung cac convolutional predictor layer. - List nay phai dai hon mot phan tu so voi so luong cac predict layers. - k phan tu dau tien la k cac he so ty le cho k layer predictor dau tien, trong khi phan tu cuoi cung duoc su dung cho second box co aspect ratio = 1 trong layer predictor cuoi cung neu `two_boxes_for_ar1` duoc dat la true. - He so ty le bo sung nay phai thoa man hai dieu kien sau, ke ca trong truong hop no khong duoc su dung: + Neu list nay duoc truyen vao, thi no se ghi de len cac gia tri min_scale va max_scale. + Tat ca cac phan tu he so ty le phai lon hon 0. * aspect_ratios_global (list, optional): - Mot list chua cac aspect ratio (ty le khung hinh) se duoc dung de tao ra cac anchor box. - List nay se duoc su dung cho toan bo cac layer trong model. * aspect_ratios_per_layer (list, optional): - Mot list chua aspect ratio cho tung lop predictor layers. - Dieu nay cho phep ban dat ty le khung hinh (aspect ratio) cho tung layer predictor rieng le, day la truong hop trien khai cua SSD300 goc. - Neu list nay duoc truyen vao thi no se ghi de len `aspect_ratios_global`. * two_boxes_for_ar1 (bool, optional): - Chi lien quan den gia tri aspect ratio = 1. - Se bi bo qua neu la truong hop khac - Neu mang gia tri `True`, hai anchor box se duoc tao cho ty le khung hinh aspect ratio = 1. + Anchor box dau tien se duoc tao bang viec su dung scale cua lop tuong ung + Anchor box thu hai se duoc tao bang viec su dung gia tri trung binh hinh hoc cua he so ty le scale cua layer dang xet va he so ty le cua layer tiep theo. * steps (list, optional): - `None` hoac la mot list chua so luong phan tu bang voi so luong cua cac layer predict. - Cac phan tu co the la ints/floats hoac tuples cua ints/floats. - Nhung con so nay dai dien cho moi predict layer co bao nhieu pixels cach cac center cua cac anchor box theo chieu doc va chieu ngang. - Neu list chua ca ints/floats thi gia tri do se duoc su dung cho ca hai kich thuoc khong gian. - Neu list chua tuples cua hai ints/floats thi chung dai dien cho `(step_height, step_width)`. - Neu step khong duoc cung cap gia tri, thi chung se duoc tinh toan sao cho cac center point cua anchor box se tao thanh mot luoi cach deu chieu rong va chieu cao cua hinh anh. * offsets (list, optional): - `None` hoac mot list chua so luong phan tu bang voi so luong cac layer predictor - Cac phan tu co the la floats hoac tuples cua hai floats. - Nhung con so nay dai dien cho moi layer predictor co bao nhieu pixels tu phia tren cung ben trai cua hinh anh den cac diem trung tam (center point). - Co mot vai dieu quan trong o day: + The offsets are not absolute pixel values, but fractions of the step size specified in the `steps` argument. If the list contains floats, then that value will be used for both spatial dimensions. If the list contains tuples of two floats, then they represent `(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size. * clip_boxes (bool, optional): - Neu `true`, cat cac toa do anchor box de anchor box nam trong ranh gioi cua hinh anh. * variances (list, optional): - Mot list co 4 so floats > 0. - The anchor box offset for each coordinate will be divided by its respective variance value. * coords (str, optional): - Định dạng tọa độ hộp được sử dụng trong mô hình tuc la day khog phai la dinh dang dau vao cua ground truth boxes. Co the centroid cho dinh dang: `cx, cy, h, w`, `min, max` cho dinh dang (xmin, xmax, ymin, ymax) hoac 'conners' cho dinh dang `xmin, ymin, xmax, ymax` * normalize_coords (bool, optional): - Dat thanh 'True' neu model su dung toa do tuong doi thay vi toa do tuyet doi tuc la neu model du doan toa do cua cac box trong [0, 1] thay vi la toa do tuyet doi. * subtract_mean (array-like, optional): - `None` hoac mot doi tuong array chua cac so nguyen hoac cac gia tri dau phay dong. - The elements of this array will be subtracted from the image pixel intensity values. For example, pass a list of three integers to perform per-channel mean normalization for color images. * divide_by_stddev (array-like, optional): - `None` or an array-like object of non-zero integers or floating point values of any shape that is broadcast-compatible with the image shape. The image pixel intensity values will be divided by the elements of this array. For example, pass a list of three integers to perform per-channel standard deviation normalization for color images. * swap_channels (list, optional): - Either `False` or a list of integers representing the desired order in which the input image channels should be swapped. * confidence_thresh (float, optional): - A float in [0,1), the minimum classification confidence in a specific positive class in order to be considered for the non-maximum suppression stage for the respective class. - A lower value will result in a larger part of the selection process being done by the non-maximum suppression stage, while a larger value will result in a larger part of the selection process happening in the confidence thresholding stage. * iou_threshold (float, optional): - A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold` with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers to the box's confidence score. * top_k (int, optional): - The number of highest scoring predictions to be kept for each batch item after the non-maximum suppression stage. * nms_max_output_size (int, optional): - The maximal number of predictions that will be left over after the NMS stage. * return_predictor_sizes (bool, optional): - If `True`, this function not only returns the model, but also a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since you can always get their sizes easily via the Keras API, but it's convenient and less error-prone to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the spatial dimensions of the predictor layers), for inference you don't need them. Returns: model: The Keras SSD300 model. predictor_sizes (optional): A Numpy array containing the `(height, width)` portion of the output tensor shape for each convolutional predictor layer. During training, the generator function needs this in order to transform the ground truth labels into tensors of identical structure as the output tensors of the model, which is in turn needed for the cost function. References: https://arxiv.org/abs/1512.02325v5 ''' ######################################################################### # Thuc hien trien khai viec build model SSD300 bang Keras ######################################################################### # So class duoc du doan trong SSD goc n_predictor_layers = 6 # Cong them mot class bieu thi cho background n_classes += 1 # Dat lai ten cho chuan hoa L2 duoc ngan gon l2_reg = l2_regularization # Lay kich thuoc cua hinh anh dau vao img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # Dua ra mot vai ngoai le truoc khi thuc hien build models ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` va `aspect_ratios_per_layer` khong duoc de ca hai deu None. At least one needs to be specifiedIt nhat mot gia tri phai duoc chi dinh." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "No phai thuoc mot trong hai truong hop: aspect_ratios_per_layer la None hoac len(aspect_ratios_per_layer) == {}, nhung len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Ca `min_scale` va `max_scale` hoac `scales` can duoc chi dinh!") if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "No phai thuoc mot trong hai truong hop scales la None hoac len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # Neu khong co list scale duoc truyen vao mot cach ro rang, thi se tinh cac scale thong qua hai gia tri min_scale va max_scale scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 gia tri variance phai duoc truyen vao nhung co {} gia tri duoc nhan!" .format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "Tat ca cac gia tri cua variance >0, nhung variances duoc nhan la {}" .format(variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "Ban phai cung cap it nhat mot gia tri step cho moi predictor layers!" ) if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "Ban phai cung cap it nhat mot gia tri offset cho moi predictor layers!" ) ############################################################################ # Tinh toan cac tham so cho Anchor Box ############################################################################ # Dat ca aspect ratio (ty le khung hinh) cho moi lop predictor layer # Dieu nay la can thiet cho cac Anchor box layers if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Tinh toan so luong cac boxes duoc predicted tren moi cell cua moi predictor layers. # Chung ta can dieu nay de chung ta biet co bao nhieu channels ma cac predictor layers can phai co. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # Neu chi co global aspect ratio list duoc truyen vao thi so luong boxes du doan # tren moi cell la nhu nhau doi voi cac predictor layers. if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Xac dinh cac function cho cac layer custom Lambda ben duoi ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # Build Network SSD (Xay dung mang SSD) ############################################################################ # Layer dau vao (kich thuoc bang kich thuoc cua hinh anh duoc truyen vao) x = Input(shape=(img_height, img_width, img_channels)) # Cac Layer tien xu ly hinh anh (chuan hoa hinh anh dau vao) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) # Xay dung lai model VGG-16 # Block 1 conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_1')(x1) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_2')(conv1_1) pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1')(conv1_2) # Block 2 conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_1')(pool1) conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_2')(conv2_1) pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2')(conv2_2) # Block 3 conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_1')(pool2) conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_2')(conv3_1) conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_3')(conv3_2) pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3')(conv3_3) # Block 4 conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_1')(pool3) conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_2')(conv4_1) conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3')(conv4_2) pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4')(conv4_3) # Block 5 conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_1')(pool4) conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_2')(conv5_1) conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_3')(conv5_2) pool5 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same', name='pool5')(conv5_3) # Het model VGG-16 ################################################################# # Bat dau xay dung cac predictor layers ################################################################# # Phan 1: Noi cac convolution voi nhau # FC6 fc6 = Conv2D(1024, (3, 3), dilation_rate=(6, 6), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc6')(pool5) # FC7 fc7 = Conv2D(1024, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7')(fc6) # Conv8_2 conv6_1 = Conv2D(256, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_1')(fc7) conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv6_padding')(conv6_1) conv6_2 = Conv2D(512, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2')(conv6_1) # Conv9_2 conv7_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_1')(conv6_2) conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv7_padding')(conv7_1) conv7_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2')(conv7_1) # Conv10_2 conv8_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_1')(conv7_2) conv8_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2')(conv8_1) # Conv11_2 conv9_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_1')(conv8_2) conv9_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2')(conv9_1) # Phan 2: Tao cac predictor convolution # Feed conv4_3 into the L2 normalization layer conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3) ### Build cac convolutional predictor layers tren top (dinh) cua base network # Phan 2.1: Xay dung trinh layers confidences: do tin cay cho moi box du doan # Chung toi predict `n_clases` gia tri confidence cho moi box, do do cac confidence predictor co # do sau `n_boxes * n_classes` # Hinh dang dau ra cua layers confidences: `(batch, height, width, n_boxes * n_classes)` conv4_3_norm_mbox_conf = Conv2D( n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_conf')(conv4_3_norm) fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_conf')(fc7) conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf')(conv6_2) conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf')(conv7_2) conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf')(conv8_2) conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf')(conv9_2) # Phan 2.2: Xay dung phan predict toa do cho moi box du doan # Chung toi du doan 4 toa do cho moi box, # do do, cac yeu to predictor localization se co chieu sau la `n_boxes * 4` # Hinh dang dau ra cua layers localization: `(batch, height, width, n_boxes * 4)` conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_loc')(conv4_3_norm) fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_loc')(fc7) conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc')(conv6_2) conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc')(conv7_2) conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(conv8_2) conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(conv9_2) # Phan 2.3: Tao cac Anchor boxs ### Tao ra cac anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) # Dau ra cua cac Anchors: `(batch, height, width, n_boxes, 8)` conv4_3_norm_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc) fc7_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='fc7_mbox_priorbox')(fc7_mbox_loc) conv6_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) ### Reshape # Dinh hinh lai shape cua class predictor, mang lai 3D tensors voi shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv4_3_norm_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape( (-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) # Reshape the box predictions, mang lai 3D tensors voi shape `(batch, height * width * n_boxes, 4)` # Chung toi muon 4 toa do se bi co lap (isolated) o truc cuoi (last axis) de co the tinh # toan cho smooth L1 loss conv4_3_norm_mbox_loc_reshape = Reshape( (-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape((-1, 4), name='fc7_mbox_loc_reshape')(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) # Reshape the anchor box tensors, mang lai 3D tensors voi shape `(batch, height * width * n_boxes, 8)` conv4_3_norm_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape( (-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox) ### Noi cac predictions tu cac lop khac nhau lai voi nhau # Axis 0 (batch) va axis 2 (n_classes hoac 4, tuong ung) la giong het nhau cho tat ca cac layers, # vi vay chung toi muon noi doc theo truc axis 1, do la so luong box tren moi layers # Output shape cua `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([ conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape ]) # Output shape cua `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([ conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape ]) # Output shape cua `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([ conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape ]) # Cac predictor ve toa do se di vao ham loss function theo cach cua chung, # nhung con doi voi cac predict class, Chung ta se ap dung lop kich hoat softmax # truoc mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Nối các dự đoán của lớp (class) và box và các anchor vào một vectơ dự đoán lớn # Output shape cua `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ conv4_3_norm_mbox_conf._keras_shape[1:3], fc7_mbox_conf._keras_shape[1:3], conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def ssd_512(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0]], two_boxes_for_ar1=True, steps=[8, 4, 2], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): n_predictor_layers = 3 # The number of predictor conv layers in the network is 7 for the original SSD512 n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) # 512 step=1 conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_1')(x1) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_2')(conv1_1) pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1')(conv1_2) # 256 step=2 conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_1')(pool1) conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_2')(conv2_1) pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2')(conv2_2) # 128 step=4 conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_1')(pool2) conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_2')(conv3_1) conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_3')(conv3_2) pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3')(conv3_3) # 64 step=8 conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_1')(pool3) conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_2')(conv4_1) conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3')(conv4_2) pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4')(conv4_3) # 32 step=16 conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_1')(pool4) conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_2')(conv5_1) conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_3')(conv5_2) # 16 step=32 pool5 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool5')(conv5_3) # 16 step=32 dilateconv6_1 = Conv2D(512, (3, 3), strides=(1, 1), dilation_rate=2, activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='dilateconv6_1')(pool5) dilateconv6_2 = Conv2D(512, (3, 3), strides=(1, 1), dilation_rate=2, activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='dilateconv6_2')(dilateconv6_1) dilateconv6_3 = Conv2D(512, (3, 3), strides=(1, 1), dilation_rate=2, activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='dilateconv6_3')(dilateconv6_2) dilateconv6_4 = Conv2D(512, (3, 3), strides=(1, 1), dilation_rate=2, activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='dilateconv6_4')(dilateconv6_3) conv7_add = concatenate([pool5, dilateconv6_4], axis=3) # 32 step=16 deconv1_1 = Conv2DTranspose(512, (3, 3), strides=(2, 2), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv1_1')(conv7_add) deconv1_2 = Conv2DTranspose(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv1_2')(deconv1_1) deconv1 = concatenate([deconv1_2, conv5_3], axis=3) # 64 step=8 deconv2_1 = Conv2DTranspose(512, (3, 3), strides=(2, 2), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv2_1')(deconv1) deconv2_2 = Conv2DTranspose(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv2_2')(deconv2_1) deconv2 = concatenate([deconv2_2, conv4_3], axis=3) # 128 step=4 deconv3_1 = Conv2DTranspose(512, (3, 3), strides=(2, 2), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv3_1')(deconv2) deconv3_2 = Conv2DTranspose(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv3_2')(deconv3_1) deconv3 = concatenate([deconv3_2, conv3_3], axis=3) # Feed conv4_3 into the L2 normalization layer # conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3) # We precidt `n_classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * n_classes` # Output shape of the confidence layers: `(batch, height, width, n_boxes * n_classes)` deconv1_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv1_mbox_conf')(deconv1) deconv2_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv2_mbox_conf')(deconv2) deconv3_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv3_mbox_conf')(deconv3) # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4` # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)` deconv1_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv1_mbox_loc')(deconv1) deconv2_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv2_mbox_loc')(deconv2) deconv3_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='deconv3_mbox_loc')(deconv3) # Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) # Output shape of anchors: `(batch, height, width, n_boxes, 8)` deconv1_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='deconv1_mbox_priorbox')(deconv1_mbox_loc) deconv2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='deconv2_mbox_priorbox')(deconv2_mbox_loc) deconv3_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='deconv3_mbox_priorbox')(deconv3_mbox_loc) # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them deconv1_mbox_conf_reshape = Reshape( (-1, n_classes), name='deconv1_mbox_conf_reshape')(deconv1_mbox_conf) deconv2_mbox_conf_reshape = Reshape( (-1, n_classes), name='deconv2_mbox_conf_reshape')(deconv2_mbox_conf) deconv3_mbox_conf_reshape = Reshape( (-1, n_classes), name='deconv3_mbox_conf_reshape')(deconv3_mbox_conf) # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss deconv1_mbox_loc_reshape = Reshape( (-1, 4), name='deconv1_mbox_loc_reshape')(deconv1_mbox_loc) deconv2_mbox_loc_reshape = Reshape( (-1, 4), name='deconv2_mbox_loc_reshape')(deconv2_mbox_loc) deconv3_mbox_loc_reshape = Reshape( (-1, 4), name='deconv3_mbox_loc_reshape')(deconv3_mbox_loc) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` deconv1_mbox_priorbox_reshape = Reshape( (-1, 8), name='deconv1_mbox_priorbox_reshape')(deconv1_mbox_priorbox) deconv2_mbox_priorbox_reshape = Reshape( (-1, 8), name='deconv2_mbox_priorbox_reshape')(deconv2_mbox_priorbox) deconv3_mbox_priorbox_reshape = Reshape( (-1, 8), name='deconv3_mbox_priorbox_reshape')(deconv3_mbox_priorbox) # Concatenate the predictions from the different layers # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1, the number of boxes per layer # Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([ deconv1_mbox_conf_reshape, deconv2_mbox_conf_reshape, deconv3_mbox_conf_reshape ]) # Output shape of `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([ deconv1_mbox_loc_reshape, deconv2_mbox_loc_reshape, deconv3_mbox_loc_reshape ]) # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([ deconv1_mbox_priorbox_reshape, deconv2_mbox_priorbox_reshape, deconv3_mbox_priorbox_reshape ]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate the class and box predictions and the anchors to one large predictions vector # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ deconv1_mbox_conf._keras_shape[1:3], deconv2_mbox_conf._keras_shape[1:3], deconv3_mbox_conf._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def MobileNetV2(input_shape, n_classes, mode='training', weights=None, input_tensor=None, min_scale=None, max_scale=None, scales=[0.07, 0.15, 0.33, 0.51, 0.69, 0.87, 1.05], aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0]], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False, alpha=1.0, **kwargs): l2_reg = kwargs.get('l2_reg', 0.00005) alpha = kwargs.get('alpha', 1.) preprocess = kwargs.get('preprocess', True) if weights and os.path.exists(weights): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `imagenet` ' '(pre-training on ImageNet), ' 'or the path to the weights file to be loaded.') # Determine proper input shape and default size. # If both input_shape and input_tensor are used, they should match if alpha not in [0.35, 0.50, 0.75, 1.0, 1.3, 1.4]: raise ValueError('If imagenet weights are being loaded, ' 'alpha can be one of `0.35`, `0.50`, `0.75`, ' '`1.0`, `1.3` or `1.4` only.') # prepare config n_predictor_layers = 6 # The number of predictor conv layers in the network is 6 for the original SSD300. n_classes += 1 # Account for the background class. img_height, img_width = input_shape[0], input_shape[1] ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers img_input = Input(shape=input_shape) if preprocess: x = Lambda(lambda x: x / 256. - 0.5)(img_input) if mode == 'hardware': x = img_input # 300 x = _inverted_res_block(x, filters=16, alpha=alpha, stride=1, expansion=1, block_id=0) # 150 x = _inverted_res_block(x, filters=24, stride=2, expansion=6, block_id=1, **kwargs) x = _inverted_res_block(x, filters=24, stride=1, expansion=6, block_id=2, **kwargs) # 75 x = _inverted_res_block(x, filters=32, stride=2, expansion=6, block_id=3, **kwargs) x = _inverted_res_block(x, filters=32, stride=1, expansion=6, block_id=4, **kwargs) x = _inverted_res_block(x, filters=32, stride=1, expansion=6, block_id=5, **kwargs) # 38 x = _inverted_res_block(x, filters=64, stride=2, expansion=6, block_id=6, **kwargs) x = _inverted_res_block(x, filters=64, stride=1, expansion=6, block_id=7, **kwargs) x = _inverted_res_block(x, filters=64, stride=1, expansion=6, block_id=8, **kwargs) x = _inverted_res_block(x, filters=64, stride=1, expansion=6, block_id=9, **kwargs) x = _inverted_res_block(x, filters=96, stride=1, expansion=6, block_id=10, **kwargs) x = _inverted_res_block(x, filters=96, stride=1, expansion=6, block_id=11, **kwargs) x = _inverted_res_block(x, filters=96, stride=1, expansion=6, block_id=12, **kwargs) # 19 conv0_pw = _inverted_res_block(x, filters=160, stride=2, expansion=6, block_id=13, **kwargs) x = _inverted_res_block(conv0_pw, filters=160, stride=1, expansion=6, block_id=14, **kwargs) x = _inverted_res_block(x, filters=160, stride=1, expansion=6, block_id=15, **kwargs) x = _inverted_res_block(x, filters=320, stride=1, expansion=6, block_id=16, **kwargs) # 10 # SSD lite x = Conv2D(1280, kernel_size=1, strides=(2, 2), use_bias=False, kernel_regularizer=l2(l2_reg), name='conv_ft_1')(x) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='bn_ft_1')(x) conv1_pw = ReLU(6., name='relu_ft_1')(x) # 5 conv2_pw = LiteConv(conv1_pw, 2, 512) # 3 conv3_pw = LiteConv(conv2_pw, 3, 256) # 2 conv4_pw = LiteConv(conv3_pw, 4, 128) # 1 conv5_pw = LiteConv(conv4_pw, 5, 128) conv0_mbox_conf = pred_cls(conv0_pw, n_boxes[0] * n_classes, name='conv0_mbox_conf') conv1_mbox_conf = pred_cls(conv1_pw, n_boxes[1] * n_classes, name='conv1_mbox_conf') conv2_mbox_conf = pred_cls(conv2_pw, n_boxes[2] * n_classes, name='conv2_mbox_conf') conv3_mbox_conf = pred_cls(conv3_pw, n_boxes[3] * n_classes, name='conv3_mbox_conf') conv4_mbox_conf = pred_cls(conv4_pw, n_boxes[4] * n_classes, name='conv4_mbox_conf') conv5_mbox_conf = pred_cls(conv5_pw, n_boxes[5] * n_classes, name='conv5_mbox_conf') # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4` # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)` conv0_mbox_loc = pred_cls(conv0_pw, n_boxes[0] * 4, name='conv0_mbox_loc') conv1_mbox_loc = pred_cls(conv1_pw, n_boxes[1] * 4, name='conv1_mbox_loc') conv2_mbox_loc = pred_cls(conv2_pw, n_boxes[2] * 4, name='conv2_mbox_loc') conv3_mbox_loc = pred_cls(conv3_pw, n_boxes[3] * 4, name='conv3_mbox_loc') conv4_mbox_loc = pred_cls(conv4_pw, n_boxes[4] * 4, name='conv4_mbox_loc') conv5_mbox_loc = pred_cls(conv5_pw, n_boxes[5] * 4, name='conv5_mbox_loc') ### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) # Output shape of anchors: `(batch, height, width, n_boxes, 8)` conv0_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv0_mbox_priorbox')(conv0_mbox_loc) conv1_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv1_mbox_priorbox')(conv1_mbox_loc) conv2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv2_mbox_priorbox')(conv2_mbox_loc) conv3_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv3_mbox_priorbox')(conv3_mbox_loc) conv4_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv4_mbox_priorbox')(conv4_mbox_loc) conv5_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv5_mbox_priorbox')(conv5_mbox_loc) ### Reshape # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv0_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv0_mbox_conf_reshape')(conv0_mbox_conf) conv1_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv1_mbox_conf_reshape')(conv1_mbox_conf) conv2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv2_mbox_conf_reshape')(conv2_mbox_conf) conv3_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv3_mbox_conf_reshape')(conv3_mbox_conf) conv4_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv4_mbox_conf_reshape')(conv4_mbox_conf) conv5_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv5_mbox_conf_reshape')(conv5_mbox_conf) # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss conv0_mbox_loc_reshape = Reshape( (-1, 4), name='conv0_mbox_loc_reshape')(conv0_mbox_loc) conv1_mbox_loc_reshape = Reshape( (-1, 4), name='conv1_mbox_loc_reshape')(conv1_mbox_loc) conv2_mbox_loc_reshape = Reshape( (-1, 4), name='conv2_mbox_loc_reshape')(conv2_mbox_loc) conv3_mbox_loc_reshape = Reshape( (-1, 4), name='conv3_mbox_loc_reshape')(conv3_mbox_loc) conv4_mbox_loc_reshape = Reshape( (-1, 4), name='conv4_mbox_loc_reshape')(conv4_mbox_loc) conv5_mbox_loc_reshape = Reshape( (-1, 4), name='conv5_mbox_loc_reshape')(conv5_mbox_loc) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` conv0_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv0_mbox_priorbox_reshape')(conv0_mbox_priorbox) conv1_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv1_mbox_priorbox_reshape')(conv1_mbox_priorbox) conv2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv2_mbox_priorbox_reshape')(conv2_mbox_priorbox) conv3_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv3_mbox_priorbox_reshape')(conv3_mbox_priorbox) conv4_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv4_mbox_priorbox_reshape')(conv4_mbox_priorbox) conv5_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv5_mbox_priorbox_reshape')(conv5_mbox_priorbox) ### Concatenate the predictions from the different layers # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1, the number of boxes per layer # Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([ conv0_mbox_conf_reshape, conv1_mbox_conf_reshape, conv2_mbox_conf_reshape, conv3_mbox_conf_reshape, conv4_mbox_conf_reshape, conv5_mbox_conf_reshape ]) # Output shape of `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([ conv0_mbox_loc_reshape, conv1_mbox_loc_reshape, conv2_mbox_loc_reshape, conv3_mbox_loc_reshape, conv4_mbox_loc_reshape, conv5_mbox_loc_reshape ]) # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([ conv0_mbox_priorbox_reshape, conv1_mbox_priorbox_reshape, conv2_mbox_priorbox_reshape, conv3_mbox_priorbox_reshape, conv4_mbox_priorbox_reshape, conv5_mbox_priorbox_reshape ]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate the class and box predictions and the anchors to one large predictions vector # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) if mode == 'training': model = Model(inputs=img_input, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=img_input, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=img_input, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ conv0_mbox_conf._keras_shape[1:3], conv1_mbox_conf._keras_shape[1:3], conv2_mbox_conf._keras_shape[1:3], conv3_mbox_conf._keras_shape[1:3], conv4_mbox_conf._keras_shape[1:3], conv5_mbox_conf._keras_shape[1:3] ]) # Load weights. if weights is not None: model.load_weights(weights) return model, predictor_sizes else: # Load weights. if weights is not None: model.load_weights(weights) return model
def ssd_300(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): n_predictor_layers = 6 # Số lượng các preductor convolutional layers trong network là 6 cho original SSD300. n_classes += 1 # Số lượng classes, + 1 để tính thêm background class. l2_reg = l2_regularization # tham số chuẩn hóa của norm chuẩn l2. img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2] ############################################################################ # Một số lỗi ngoại lệ. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified.") if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}.".format( n_predictor_layers, len(aspect_ratios_per_layer))) # Tạo list scales if (min_scale is None or max_scale is None) and scales is None: raise ValueError("Either `min_scale` and `max_scale` or `scales` need to be specified.") if scales: if len(scales) != n_predictor_layers + 1: raise ValueError("It must be either scales is None or len(scales) == {}, but len(scales) == {}.".format( n_predictor_layers + 1, len(scales))) else: scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError("4 variance values must be pased, but {} values were received.".format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError("All variances must be >0, but the variances given are {}".format(variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError("You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError("You must provide at least one offset value per predictor layer.") ############################################################################ # Tính các tham số của anchor box. ############################################################################ # Thiết lập aspect ratios cho mỗi predictor layer (chỉ cần thiết cho tính toán anchor box layers). if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Tính số lượng boxes được dự báo / 1 cell cho mỗi predictor layer. # Chúng ta cần biết bao nhiêu channels các predictor layers cần có. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 cho trường hợp aspect ratio = 1 else: n_boxes.append(len(ar)) else: # Nếu chỉ 1 global aspect ratio list được truyền vào thì số lượng boxes là như nhau cho mọi layers. if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Xác định các hàm số cho Lambda layers bên dưới. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack( [tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]]], axis=-1) elif len(swap_channels) == 4: return K.stack([tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]]], axis=-1) ############################################################################ # Bước 1: Xây dựng network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')( x1) ############################################################################ # Bước 1.1: Tính toán base network là mạng VGG16 ############################################################################ conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3')(x1) ############################################################################ # Feed conv4_3 vào the L2 normalization layer conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(x1) ############################################################################ # Bước 1.3: Xác định output phân phối xác suất theo các classes ứng với mỗi một default bounding box. ############################################################################ ### Xây dựng các convolutional predictor layers tại top của base network # Chúng ta dự báo các giá trị confidence cho mỗi box, do đó confidence predictors có độ sâu `n_boxes * n_classes` # Đầu ra của confidence layers có shape: `(batch, height, width, n_boxes * n_classes)` conv4_3_norm_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_conf')(conv4_3_norm) fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_conf')(x1) conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf')(x1) conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf')(x1) conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf')(x1) conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf')(x1) ############################################################################ # Bước 1.4: Xác định output các tham số offset của default bounding boxes tương ứng với mỗi cell trên các features map. ############################################################################ # Chúng ta dự báo 4 tọa độ cho mỗi box, do đó localization predictors có độ sâu `n_boxes * 4` # Output shape của localization layers: `(batch, height, width, n_boxes * 4)` conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_loc')(conv4_3_norm) fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_loc')(x1) conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc')(x1) conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc')(x1) conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(x1) conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(x1) ############################################################################ # Bước 1.5: Tính toán các AnchorBoxes làm cơ sở để dự báo offsets cho các predicted bounding boxes bao quan vật thể ############################################################################ ### Khởi tạo các anchor boxes (được gọi là "priors" trong code gốc Caffe/C++ của mô hình) # Shape output của anchors: `(batch, height, width, n_boxes, 8)` conv4_3_norm_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc) fc7_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='fc7_mbox_priorbox')(fc7_mbox_loc) conv6_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) ############################################################################ # Bước 2: Reshape lại các output tensor shape ############################################################################ ############################################################################ # Bước 2.1: Reshape output của class predictions ############################################################################ # Reshape các class predictions, trả về 3D tensors có shape `(batch, height * width * n_boxes, n_classes)` # Chúng ta muốn các classes là tách biệt nhau trên last axis để tính softmax trên chúng. conv4_3_norm_mbox_conf_reshape = Reshape((-1, n_classes), name='conv4_3_norm_mbox_conf_reshape')( conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape((-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) ############################################################################ # Bước 2.2: Reshape output của bounding box predictions ############################################################################ # Reshape các box predictions, trả về 3D tensors có shape `(batch, height * width * n_boxes, 4)` # Chúng ta muốn 4 tọa độ box là tách biệt nhau trên last axis để tính hàm smooth L1 loss conv4_3_norm_mbox_loc_reshape = Reshape((-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape((-1, 4), name='fc7_mbox_loc_reshape')(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape((-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape((-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape((-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape((-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) ############################################################################ # Bước 2.3: Reshape output của anchor box ############################################################################ # Reshape anchor box tensors, trả về 3D tensors có shape `(batch, height * width * n_boxes, 8)` conv4_3_norm_mbox_priorbox_reshape = Reshape((-1, 8), name='conv4_3_norm_mbox_priorbox_reshape')( conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape((-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape((-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape((-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape((-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape((-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox) ### Concatenate các predictions từ các layers khác nhau ############################################################################ # Bước 3: Concatenate các boxes trên layers ############################################################################ ############################################################################ # Bước 3.1: Concatenate confidence output box ############################################################################ # Axis 0 (batch) và axis 2 (n_classes hoặc 4) là xác định duy nhất cho toàn bộ các predictions layer # nên chúng ta muốn concatenate theo axis 1, số lượng các boxes trên layer # Output shape của `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape]) ############################################################################ # Bước 3.2: Concatenate location output box ############################################################################ # Output shape của `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape]) ############################################################################ # Bước 3.3: Concatenate anchor output box ############################################################################ # Output shape của `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape]) ############################################################################ # Bước 4: Tính toán output ############################################################################ ############################################################################ # Bước 4.1 : Xây dựng các hàm loss function cho confidence ############################################################################ # tọa độ của box predictions sẽ được truyền vào hàm loss function, # nhưng cho các dự báo lớp, chúng ta sẽ áp dụng một hàm softmax activation layer đầu tiên mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate các class và box predictions và the anchors thành một large predictions vector # Đầu ra của `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')([mbox_conf_softmax, mbox_loc, mbox_priorbox]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections(confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast(confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'.".format(mode)) if return_predictor_sizes: predictor_sizes = np.array([conv4_3_norm_mbox_conf._keras_shape[1:3], fc7_mbox_conf._keras_shape[1:3], conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3]]) return model, predictor_sizes else: return model
def build_model(image_size, n_classes, mode='training', l2_regularization=0.0, min_scale=0.1, max_scale=0.9, scales=None, aspect_ratios_global=[0.5, 1.0, 2.0], aspect_ratios_per_layer=None, two_boxes_for_ar1=True, steps=None, offsets=None, clip_boxes=False, variances=[1.0, 1.0, 1.0, 1.0], coords='centroids', normalize_coords=False, subtract_mean=None, divide_by_stddev=None, swap_channels=False, confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): ''' Build a Keras model with SSD architecture, see references. The model consists of convolutional feature layers and a number of convolutional predictor layers that take their input from different feature layers. The model is fully convolutional. The implementation found here is a smaller version of the original architecture used in the paper (where the base network consists of a modified VGG-16 extended by a few convolutional feature layers), but of course it could easily be changed to an arbitrarily large SSD architecture by following the general design pattern used here. This implementation has 7 convolutional layers and 4 convolutional predictor layers that take their input from layers 4, 5, 6, and 7, respectively. Most of the arguments that this function takes are only needed for the anchor box layers. In case you're training the network, the parameters passed here must be the same as the ones used to set up `SSDBoxEncoder`. In case you're loading trained weights, the parameters passed here must be the same as the ones used to produce the trained weights. Some of these arguments are explained in more detail in the documentation of the `SSDBoxEncoder` class. Note: Requires Keras v2.0 or later. Training currently works only with the TensorFlow backend (v1.0 or later). Arguments: image_size (tuple): The input image size in the format `(height, width, channels)`. n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO. mode (str, optional): One of 'training', 'inference' and 'inference_fast'. In 'training' mode, the model outputs the raw prediction tensor, while in 'inference' and 'inference_fast' modes, the raw predictions are decoded into absolute coordinates and filtered via confidence thresholding, non-maximum suppression, and top-k filtering. The difference between latter two modes is that 'inference' follows the exact procedure of the original Caffe implementation, while 'inference_fast' uses a faster prediction decoding procedure. l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers. min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. All scaling factors between the smallest and the largest will be linearly interpolated. Note that the second to last of the linearly interpolated scaling factors will actually be the scaling factor for the last predictor layer, while the last scaling factor is used for the second box for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer. This list must be one element longer than the number of predictor layers. The first `k` elements are the scaling factors for the `k` predictor layers, while the last element is used for the second box for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional last scaling factor must be passed either way, even if it is not being used. If a list is passed, this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero. aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be generated. This list is valid for all predictor layers. The original implementation uses more aspect ratios for some predictor layers and fewer for others. If you want to do that, too, then use the next argument instead. aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each predictor layer. This allows you to set the aspect ratios for each predictor layer individually. If a list is passed, it overrides `aspect_ratios_global`. two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise. If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated using the scaling factor for the respective layer, the second one will be generated using geometric mean of said scaling factor and next bigger scaling factor. steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over the image. If the list contains ints/floats, then that value will be used for both spatial dimensions. If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`. If no steps are provided, then they will be computed such that the anchor box center points will form an equidistant grid within the image dimensions. offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either floats or tuples of two floats. These numbers represent for each predictor layer how many pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions of the step size specified in the `steps` argument. If the list contains floats, then that value will be used for both spatial dimensions. If the list contains tuples of two floats, then they represent `(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size, which is also the recommended setting. clip_boxes (bool, optional): If `True`, clips the anchor box coordinates to stay within image boundaries. variances (list, optional): A list of 4 floats >0. The anchor box offset for each coordinate will be divided by its respective variance value. coords (str, optional): The box coordinate format to be used internally by the model (i.e. this is not the input format of the ground truth labels). Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width, and height), 'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`. normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates, i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates. subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values of any shape that is broadcast-compatible with the image shape. The elements of this array will be subtracted from the image pixel intensity values. For example, pass a list of three integers to perform per-channel mean normalization for color images. divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or floating point values of any shape that is broadcast-compatible with the image shape. The image pixel intensity values will be divided by the elements of this array. For example, pass a list of three integers to perform per-channel standard deviation normalization for color images. swap_channels (list, optional): Either `False` or a list of integers representing the desired order in which the input image channels should be swapped. confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific positive class in order to be considered for the non-maximum suppression stage for the respective class. A lower value will result in a larger part of the selection process being done by the non-maximum suppression stage, while a larger value will result in a larger part of the selection process happening in the confidence thresholding stage. iou_threshold (float, optional): A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold` with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers to the box's confidence score. top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the non-maximum suppression stage. nms_max_output_size (int, optional): The maximal number of predictions that will be left over after the NMS stage. return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since you can always get their sizes easily via the Keras API, but it's convenient and less error-prone to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the spatial dimensions of the predictor layers), for inference you don't need them. Returns: model: The Keras SSD model. predictor_sizes (optional): A Numpy array containing the `(height, width)` portion of the output tensor shape for each convolutional predictor layer. During training, the generator function needs this in order to transform the ground truth labels into tensors of identical structure as the output tensors of the model, which is in turn needed for the cost function. References: https://arxiv.org/abs/1512.02325v5 ''' n_predictor_layers = 6 # The number of predictor conv layers in the network is 6 for the original SSD300. n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def _conv_block(inputs, filters, kernel, strides): channel_axis = 1 if K.image_data_format() == 'channels_first' else -1 x = Conv2D(filters, kernel, padding='same', use_bias=False, strides=strides)(inputs) x = BatchNormalization(axis=channel_axis)(x) return ReLU(6)(x) def correct_pad(inputs, kernel_size): img_dim = 2 if K.image_data_format() == 'channels_first' else 1 input_size = K.int_shape(inputs)[img_dim:(img_dim + 2)] if isinstance(kernel_size, int): kernel_size = (kernel_size, kernel_size) if input_size[0] is None: adjust = (1, 1) else: adjust = (1 - input_size[0] % 2, 1 - input_size[1] % 2) correct = (kernel_size[0] // 2, kernel_size[1] // 2) return ((correct[0] - adjust[0], correct[0]), (correct[1] - adjust[1], correct[1])) def _bottleneck(inputs, filters, kernel, t, s, r=False, name='sosi'): channel_axis = 1 if K.image_data_format() == 'channels_first' else -1 tchannel = K.int_shape(inputs)[channel_axis] * t x = _conv_block(inputs, tchannel, (1, 1), (1, 1)) if s == 2: x = layers.ZeroPadding2D(padding=correct_pad(x, 3))(x) padding = 'same' if s == 1 else 'valid' x = DepthwiseConv2D(kernel, strides=(s, s), depth_multiplier=1, use_bias=False, padding=padding)(x) x = BatchNormalization(axis=channel_axis)(x) x = Activation('relu')(x) x = Conv2D(filters, (1, 1), strides=(1, 1), padding='same')(x) x = BatchNormalization(axis=channel_axis)(x) if s == 1 and filters == K.int_shape(inputs)[channel_axis]: x = Add()([x, inputs]) return x def _inverted_residual_block(inputs, filters, kernel, t, strides, n, name='inv_res_block'): x = _bottleneck(inputs, filters, kernel, t, strides, name) for i in range(1, n): x = _bottleneck(x, filters, kernel, t, 1, True) return x ############################################################################ # Build the network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) '''MOBILENET_V2''' conv0 = _conv_block(x1, 32, (3, 3), strides=(2, 2)) conv1 = _inverted_residual_block(conv0, 16, (3, 3), t=1, strides=1, n=1) conv2 = _inverted_residual_block(conv1, 24, (3, 3), t=6, strides=2, n=2) conv3 = _inverted_residual_block(conv2, 32, (3, 3), t=6, strides=2, n=3) conv4 = _inverted_residual_block(conv3, 64, (3, 3), t=6, strides=2, n=4) conv5 = _inverted_residual_block(conv4, 64, (3, 3), t=6, strides=1, n=3) conv6 = _inverted_residual_block(conv5, 96, (3, 3), t=6, strides=2, n=3) conv7 = _inverted_residual_block(conv6, 160, (3, 3), t=6, strides=1, n=1) conv8 = _inverted_residual_block(conv7, 160, (3, 3), t=6, strides=1, n=1) conv9 = _conv_block(conv8, 1280 // 2, (1, 1), strides=(1, 1)) # x = GlobalAveragePooling2D()(x) # x = Reshape((1, 1, 1280))(x) # x = Dropout(0.3, name='Dropout')(x) # x = Conv2D(k, (1, 1), padding='same')(x) # x = Activation('softmax', name='softmax')(x) '''END OF MOBILENET_V2''' '''RESNET-18''' # model = ResnetBuilder.build_resnet_18((), ) '''END OF RESNET-18''' # The next part is to add the convolutional predictor layers on top of the base network # that we defined above. Note that I use the term "base network" differently than the paper does. # To me, the base network is everything that is not convolutional predictor layers or anchor # box layers. In this case we'll have four predictor layers, but of course you could # easily rewrite this into an arbitrarily deep base network and add an arbitrary number of # predictor layers on top of the base network by simply following the pattern shown here. # Build the convolutional predictor layers on top of conv layers 4, 5, 6, and 7. # We build two predictor layers on top of each of these layers: One for class prediction (classification), one for box coordinate prediction (localization) # We precidt `n_classes` confidence values for each box, hence the `classes` predictors have depth `n_boxes * n_classes` # We predict 4 box coordinates for each box, hence the `boxes` predictors have depth `n_boxes * 4` # Output shape of `classes`: `(batch, height, width, n_boxes * n_classes)` conv4_3_norm_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_conf')(conv4) fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_conf')(conv5) conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf')(conv6) conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf')(conv7) conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf')(conv8) conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf')(conv9) # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4` # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)` conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_loc')(conv4) fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_loc')(conv5) conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc')(conv6) conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc')(conv7) conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(conv8) conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(conv9) ### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) # Output shape of anchors: `(batch, height, width, n_boxes, 8)` conv4_3_norm_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc) fc7_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='fc7_mbox_priorbox')(fc7_mbox_loc) conv6_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) ### Reshape # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv4_3_norm_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape( (-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss conv4_3_norm_mbox_loc_reshape = Reshape( (-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape((-1, 4), name='fc7_mbox_loc_reshape')(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` conv4_3_norm_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape( (-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox) ### Concatenate the predictions from the different layers # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1, the number of boxes per layer # Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([ conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape ]) # Output shape of `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([ conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape ]) # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([ conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape ]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate the class and box predictions and the anchors to one large predictions vector # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ conv4_3_norm_mbox_conf._keras_shape[1:3], fc7_mbox_conf._keras_shape[1:3], conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def dronenet(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False, bottleneck=True, reduction=0.0, dropout_rate=None, weight_decay=1e-4): n_predictor_layers = 4 # The number of predictor conv layers in the network is 6 for the original SSD300. n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError("`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified.") if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError("It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}.".format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError("Either `min_scale` and `max_scale` or `scales` need to be specified.") if scales: if len(scales) != n_predictor_layers+1: raise ValueError("It must be either scales is None or len(scales) == {}, but len(scales) == {}.".format(n_predictor_layers+1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers+1) if len(variances) != 4: raise ValueError("4 variance values must be pased, but {} values were received.".format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError("All variances must be >0, but the variances given are {}".format(variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError("You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError("You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]]], axis=-1) elif len(swap_channels) == 4: return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]], tensor[...,swap_channels[3]]], axis=-1) ############################################################################ # Build the network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) def dense_block(prevDense, stage, nb_layers, nb_filter, growth_rate, bottleneck=True, dropout_rate=None, weight_decay=1e-4, grow_nb_filters=True): ''' Build a dense_block where the output of each conv_block is fed to subsequent ones # Arguments x: input tensor stage: index for dense block nb_layers: the number of layers of conv_block to append to the model. nb_filter: number of filters growth_rate: growth rate dropout_rate: dropout rate weight_decay: weight decay factor grow_nb_filters: flag to decide to allow number of filters to grow ''' for i in range(nb_layers): branch = i + 1 dense = conv_block(prevDense, stage, branch, growth_rate, bottleneck, dropout_rate, weight_decay) #print('layer', stage, branch, nb_filter, prevDense.shape) prevDense = concatenate([prevDense, dense], axis=3, name='concat_'+str(stage)+'_'+str(branch)) #print('concate', stage, nb_filter, prevDense.shape) if grow_nb_filters: nb_filter += growth_rate print('dense', stage, nb_filter, prevDense.shape) return prevDense, nb_filter def conv_block(prevConv, stage, branch, nb_filter, bottleneck=True, dropout_rate=None, weight_decay=1e-4): '''Apply BatchNorm, Relu, bottleneck 1x1 Conv2D, 3x3 Conv2D, and option dropout # Arguments prevConv: input tensor nb_filter: number of filters dropout_rate: dropout rate weight_decay: weight decay factor ''' eps = 1.1e-5 conv_name_base = 'conv' + str(stage) + '_' + str(branch) relu_name_base = 'relu' + str(stage) + '_' + str(branch) prevConv = BatchNormalization(epsilon=eps, axis=3, name=conv_name_base+'_x1_bn')(prevConv) prevConv = Activation('relu', name=relu_name_base+'_x1')(prevConv) if bottleneck: inter_channel = nb_filter * 4 # Obtained from https://github.com/liuzhuang13/DenseNet/blob/master/densenet.lua prevConv = Conv2D(inter_channel, (1, 1), kernel_initializer='he_normal', padding='same', use_bias=False, kernel_regularizer=l2(weight_decay), name=conv_name_base+'_x1')(prevConv) if dropout_rate: prevConv = Dropout(dropout_rate)(prevConv) prevConv = BatchNormalization(epsilon=eps, axis=3, name=conv_name_base+'_x2_bn')(prevConv) prevConv = Activation('relu', name=relu_name_base+'_x2')(prevConv) prevConv = Conv2D(nb_filter, (3, 3), kernel_initializer='he_normal', padding='same', use_bias=False, name=conv_name_base+'_x2')(prevConv) if dropout_rate: prevConv = Dropout(dropout_rate)(prevConv) return prevConv def transition_block(prevTran, stage, nb_filter, compression=1.0, dropout_rate=None, weight_decay=1E-4): ''' Apply BatchNorm, 1x1 Convolution, averagePooling, optional compression, dropout # Arguments prevTran: input tensor stage: index for dense block nb_filter: number of filters compression: calculated as 1 - reduction. Reduces the number of feature maps in the transition block. dropout_rate: dropout rate weight_decay: weight decay factor ''' eps = 1.1e-5 conv_name_base = 'conv' + str(stage) + '_blk' relu_name_base = 'relu' + str(stage) + '_blk' pool_name_base = 'poolD' + str(stage) prevTran = BatchNormalization(epsilon=eps, axis=3, name=conv_name_base+'_bn')(prevTran) prevTran = Activation('relu', name=relu_name_base)(prevTran) prevTran = Conv2D(int(nb_filter * compression), (1, 1), activation='relu', kernel_initializer='he_normal', padding='same', use_bias=False, kernel_regularizer=l2(weight_decay), name=conv_name_base)(prevTran) if dropout_rate: prevTran = Dropout(dropout_rate)(prevTran) # if stage !=3: # prevTran = MaxPooling2D(pool_size=(2, 2), name=pool_name_base)(prevTran) print('tran', stage, prevTran.shape) return prevTran # DenseNet Parameters eps = 1.1e-5 nb_filter =64 t_nb_filter = 256 growth_rate = 32 nb_layers = [5, 7, 7, 7] compression = 1.0 - reduction conv1_1 = Conv2D(64, (3, 3), strides=(2, 2), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_1')(x1) conv1_2 = BatchNormalization(epsilon=eps, axis=3, name='conv1_2_bn')(conv1_1) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_2')(conv1_2) conv1_3 = BatchNormalization(epsilon=eps, axis=3, name='conv1_3_bn')(conv1_2) conv1_3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_3')(conv1_3) pool1_3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1_3')(conv1_3) # Add densBlock1 stage = 1 conv1, nb_filter = dense_block(pool1_3, stage, nb_layers[0], nb_filter, growth_rate, bottleneck=bottleneck, dropout_rate=None, weight_decay=weight_decay) trans1 = transition_block(conv1, stage, t_nb_filter, compression=compression, weight_decay=weight_decay) pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1')(trans1) nb_filter = int(nb_filter * compression) # Add densBlock2 stage = 2 conv2, nb_filter = dense_block(pool1, stage, nb_layers[1], nb_filter, growth_rate, bottleneck=bottleneck, dropout_rate=None, weight_decay=weight_decay) trans2 = transition_block(conv2, stage, t_nb_filter, compression=compression, weight_decay=weight_decay) pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2')(trans2) nb_filter = int(nb_filter * compression) # Add densBlock3 stage = 3 conv3, nb_filter = dense_block(pool2, stage, nb_layers[2], nb_filter, growth_rate, bottleneck=bottleneck, dropout_rate=None, weight_decay=weight_decay) trans3 = transition_block(conv3, stage, t_nb_filter, compression=compression, weight_decay=weight_decay) pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3')(trans3) nb_filter = int(nb_filter * compression) # Add densBlock4 stage = 4 conv4, nb_filter = dense_block(pool3, stage, nb_layers[3], nb_filter, 26, bottleneck=bottleneck, dropout_rate=None, weight_decay=weight_decay) trans4 = transition_block(conv4, stage, t_nb_filter, compression=compression, weight_decay=weight_decay) #pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4')(trans4) M5 = BatchNormalization(epsilon=eps, axis=3, name='m5_bn1')(trans4) M5 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='M5P')(M5) M4 = UpSampling2D(size=(2, 2))(M5) # M4E = ZeroPadding2D(padding=((0, 1), (1, 0)), name='trans3_padding')(trans3) M4 = Add()([M4, trans3]) M4 = BatchNormalization(epsilon=eps, axis=3, name='M4_bn')(M4) M4 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='M4P')(M4) M3 = UpSampling2D(size=(2, 2))(M4) # M3E = ZeroPadding2D(padding=((1, 1), (1, 1)), name='trans3_padding')(trans2) M3 = Add()([M3, trans2]) M3 = BatchNormalization(epsilon=eps, axis=3, name='M3_bn')(M3) M3 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='M3P')(M3) M2 = UpSampling2D(size=(2, 2))(M3) # M2E = ZeroPadding2D(padding=((2, 2), (2, 2)), name='trans3_padding')(trans1) M2 = Add()([M2, trans1]) M2 = BatchNormalization(epsilon=eps, axis=3, name='M2_bn')(M2) M2 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='M2P')(M2) print('M5', M5.shape) print('M4', M4.shape) print('M3', M3.shape) print('M2', M2.shape) print() ### Build the convolutional predictor layers on top of the base network # We precidt `n_classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * n_classes` # Output shape of the confidence layers: `(batch, height, width, n_boxes * n_classes)` # conv6_2_mbox = BatchNormalization(epsilon=eps, axis=3, name='conv6_2_mbox_conf_bn1')(M2) # conv6_2_mbox = Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf1')(conv6_2_mbox) conv6_2_mbox_conf = BatchNormalization(epsilon=eps, axis=3, name='conv6_2_mbox_conf_bn2')(M2) conv6_2_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf2')(conv6_2_mbox_conf) # conv7_2_mbox = BatchNormalization(epsilon=eps, axis=3, name='conv7_2_mbox_conf_bn1')(M3) # conv7_2_mbox = Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf1')(conv7_2_mbox) conv7_2_mbox_conf = BatchNormalization(epsilon=eps, axis=3, name='conv7_2_mbox_conf_bn2')(M3) conv7_2_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf2')(conv7_2_mbox_conf) # conv8_2_mbox = BatchNormalization(epsilon=eps, axis=3, name='conv8_2_mbox_conf_bn1')(M4) # conv8_2_mbox = Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf1')(conv8_2_mbox) conv8_2_mbox_conf = BatchNormalization(epsilon=eps, axis=3, name='conv8_2_mbox_conf_bn2')(M4) conv8_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf2')(conv8_2_mbox_conf) # conv9_2_mbox = BatchNormalization(epsilon=eps, axis=3, name='conv9_2_mbox_conf_bn1')(M5) # conv9_2_mbox = Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf1')(conv9_2_mbox) conv9_2_mbox_conf = BatchNormalization(epsilon=eps, axis=3, name='conv9_2_mbox_conf_bn2')(M5) conv9_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf2')(conv9_2_mbox_conf) print('conv6_2_mbox_conf', conv6_2_mbox_conf.shape) print('conv7_2_mbox_conf', conv7_2_mbox_conf.shape) print('conv8_2_mbox_conf', conv8_2_mbox_conf.shape) print('conv9_2_mbox_conf', conv9_2_mbox_conf.shape) print() # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4` # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)` conv6_2_mbox_loc = BatchNormalization(epsilon=eps, axis=3, name='conv6_2_mbox_loc_bn')(M2) conv6_2_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc')(conv6_2_mbox_loc) conv7_2_mbox_loc = BatchNormalization(epsilon=eps, axis=3, name='conv7_2_mbox_loc_bn')(M3) conv7_2_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc')(conv7_2_mbox_loc) conv8_2_mbox_loc = BatchNormalization(epsilon=eps, axis=3, name='conv8_2_mbox_loc_bn')(M4) conv8_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(conv8_2_mbox_loc) conv9_2_mbox_loc = BatchNormalization(epsilon=eps, axis=3, name='conv9_2_mbox_loc_bn')(M5) conv9_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(conv9_2_mbox_loc) print('conv6_2_mbox_loc', conv6_2_mbox_loc.shape) print('conv7_2_mbox_loc', conv7_2_mbox_loc.shape) print('conv8_2_mbox_loc', conv8_2_mbox_loc.shape) print('conv9_2_mbox_loc', conv9_2_mbox_loc.shape) print() ### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) conv6_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) print('conv6_2_mbox_priorbox', conv6_2_mbox_priorbox.shape) print('conv7_2_mbox_priorbox', conv7_2_mbox_priorbox.shape) print('conv8_2_mbox_priorbox', conv8_2_mbox_priorbox.shape) print('conv9_2_mbox_priorbox', conv9_2_mbox_priorbox.shape) print() ### Reshape # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv6_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) print('conv6_2_mbox_conf_reshape', conv6_2_mbox_conf_reshape.shape) print('conv7_2_mbox_conf_reshape', conv7_2_mbox_conf_reshape.shape) print('conv8_2_mbox_conf_reshape', conv8_2_mbox_conf_reshape.shape) print('conv9_2_mbox_conf_reshape', conv9_2_mbox_conf_reshape.shape) print() # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss conv6_2_mbox_loc_reshape = Reshape((-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape((-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape((-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape((-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) print('conv6_2_mbox_loc_reshape', conv6_2_mbox_loc_reshape.shape) print('conv7_2_mbox_loc_reshape', conv7_2_mbox_loc_reshape.shape) print('conv8_2_mbox_loc_reshape', conv8_2_mbox_loc_reshape.shape) print('conv9_2_mbox_loc_reshape', conv9_2_mbox_loc_reshape.shape) print() # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` conv6_2_mbox_priorbox_reshape = Reshape((-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape((-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape((-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape((-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox) print('conv6_2_mbox_priorbox_reshape', conv6_2_mbox_priorbox_reshape.shape) print('conv7_2_mbox_priorbox_reshape', conv7_2_mbox_priorbox_reshape.shape) print('conv8_2_mbox_priorbox_reshape', conv8_2_mbox_priorbox_reshape.shape) print('conv9_2_mbox_priorbox_reshape', conv9_2_mbox_priorbox_reshape.shape) print() ### Concatenate the predictions from the different layers # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1, the number of boxes per layer # Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape ,conv9_2_mbox_conf_reshape ]) print('mbox_conf', mbox_conf.shape) print() # Output shape of `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape ,conv9_2_mbox_loc_reshape ]) print('mbox_loc', mbox_loc.shape) print() # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape ,conv9_2_mbox_priorbox_reshape ]) print('mbox_priorbox', mbox_priorbox.shape) print() # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) print('mbox_conf_softmax', mbox_conf_softmax.shape) print() # Concatenate the class and box predictions and the anchors to one large predictions vector # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')([mbox_conf_softmax, mbox_loc, mbox_priorbox]) print('predictions', predictions.shape) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections(confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast(confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError("`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'.".format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3]]) return model, predictor_sizes else: return model
def drbox(image_size, n_classes, l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, angles_global = [0, np.pi/6, np.pi/3, np.pi/2, 2*np.pi/3, 5*np.pi/6], angles_per_layer = None, aspect_ratios_global=None, aspect_ratios_per_layer = [[ 2.0, 4.0], [ 2.0, 4.0, 6.0], [ 2.0, 4.0, 6.0], [ 2.0, 4.0, 6.0], [ 2.0, 4.0, 6.0], [ 2.0, 4.0]], steps=[8, 16, 32, 64, 128, 256], offsets=None, variances=[0.1, 0.1, 0.2, 0.2, 1], normalize_coords=True, subtract_mean=84, divide_by_stddev=None, return_predictor_sizes=False): ''' Build a DRBox Keras model architecture, see references. The base network is a reduced atrous VGG-16, extended by the DRBox architecture, as described in the paper. Most of the arguments that this function takes are only needed for the anchor box layers. In case you're training the network, the parameters passed here must be the same as the ones used to set up `DRBoxEncoder`. In case you're loading trained weights, the parameters passed here must be the same as the ones used to produce the trained weights. Some of these arguments are explained in more detail in the documentation of the `DRBoxEncoder` class. Note: Works only with the TensorFlow backend (v1.0 or later). Arguments: image_size (tuple): The input image size in the format `(height, width)`. n_classes (int): The number of positive classes l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers. Set to zero to deactivate L2-regularization. min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. All scaling factors between the smallest and the largest will be linearly interpolated. scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer. The first `k` elements are the scaling factors for the `k` predictor layers. If a list is passed, this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero. angles_global (list, optional): A list of floats >0 containing different angles for wich to create corresponding anchor boxes in radians. angles_per_layer (list, optional): A list containing one angles list for each prediction layer in radians. If a list is passed, it overrides `angles_global`. Defaults to `None`. Note that you should set the angles such that the resulting anchor box shapes very roughly correspond to the shapes of the objects you are trying to detect. aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be generated. This list is valid for all prediction layers. aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each prediction layer. This allows you to set the aspect ratios for each predictor layer individually. If a list is passed, it overrides `aspect_ratios_global`. steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over the image. If the list contains ints/floats, then that value will be used for both spatial dimensions. If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`. If no steps are provided, then they will be computed such that the anchor box center points will form an equidistant grid within the image dimensions. offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either floats or tuples of two floats. These numbers represent for each predictor layer how many pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions of the step size specified in the `steps` argument. If the list contains floats, then that value will be used for both spatial dimensions. If the list contains tuples of two floats, then they represent `(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size. variances (list, optional): A list of 5 floats >0. The anchor box offset for each coordinate will be divided by its respective variance value. normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates, i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates. subtract_mean (float, optional): `None` or an integer or float. The element will be subtracted from the image pixel intensity values. divide_by_stddev (float, optional): `None` or an integer or float. The image pixel intensity values will be divided by the elements of this array. return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since you can always get their sizes easily via the Keras API, but it's convenient and less error-prone to get them this way. They are only relevant for training anyway (DRBoxEncoder needs to know the spatial dimensions of the predictor layers), for inference you don't need them. Returns: model: The Keras SSD512 model. predictor_sizes (optional): A Numpy array containing the `(height, width)` portion of the output tensor shape for each convolutional predictor layer. During training, the generator function needs this in order to transform the ground truth labels into tensors of identical structure as the output tensors of the model, which is in turn needed for the cost function. References: https://arxiv.org/abs/1711.09405 ''' n_predictor_layers = 6 n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError("`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified.") if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError("It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(angles_per_layer) == {}.".format(n_predictor_layers, len(aspect_ratios_per_layer))) if angles_global is None and angles_per_layer is None: raise ValueError("`angles_global` and `angles_per_layer` cannot both be None. At least one needs to be specified.") if angles_per_layer: if len(angles_per_layer) != n_predictor_layers: raise ValueError("It must be either angles_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}.".format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError("Either `min_scale` and `max_scale` or `scales` need to be specified.") if scales: if len(scales) != n_predictor_layers: raise ValueError("It must be either scales is None or len(scales) == {}, but len(scales) == {}.".format(n_predictor_layers, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers+1) if len(variances) != 5: raise ValueError("5 variance values must be pased, but {} values were received.".format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError("All variances must be >0, but the variances given are {}".format(variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError("You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError("You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers if angles_per_layer: angles = angles_per_layer else: angles = [angles_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. n_boxes = [] for i in range(len(aspect_ratios)): n_boxes.append(len(aspect_ratios[i]) * len(angles[i])) if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) ############################################################################ # Build the network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) # if input shape is not (512,512) we change the name of the first layer # to avoid a shape error when loading pretrained weights if (img_height,img_width) == (512,512): name = 'conv1_1' else: name = 'conv1_1_bis' conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name=name)(x1) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_2')(conv1_1) pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1')(conv1_2) conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_1')(pool1) conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_2')(conv2_1) pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2')(conv2_2) conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_1')(pool2) conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_2')(conv3_1) conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_3')(conv3_2) pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3')(conv3_3) conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_1')(pool3) conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_2')(conv4_1) conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3')(conv4_2) pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4')(conv4_3) conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_1')(pool4) conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_2')(conv5_1) conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_3')(conv5_2) pool5 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same', name='pool5')(conv5_3) fc6 = Conv2D(1024, (3, 3), dilation_rate=(6, 6), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc6')(pool5) fc7 = Conv2D(1024, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7')(fc6) conv6_1 = Conv2D(256, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_1')(fc7) conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv6_padding')(conv6_1) conv6_2 = Conv2D(512, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2')(conv6_1) conv7_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_1')(conv6_2) conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv7_padding')(conv7_1) conv7_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2')(conv7_1) conv8_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_1')(conv7_2) conv8_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv8_padding')(conv8_1) conv8_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2')(conv8_1) conv9_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_1')(conv8_2) conv9_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv9_padding')(conv9_1) conv9_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2')(conv9_1) # Feed conv4_3 into the L2 normalization layer conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3) ### Build the convolutional predictor layers on top of the base network # We precidt `n_classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * n_classes` # Output shape of the confidence layers: `(batch, height, width, n_boxes * n_classes)` conv4_3_norm_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_conf')(conv4_3_norm) fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_conf')(fc7) conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf')(conv6_2) conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf')(conv7_2) conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf')(conv8_2) conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf')(conv9_2) # We predict 5 box coordinates for each box, hence the localization predictors have depth `n_boxes * 5` # Output shape of the localization layers: `(batch, height, width, n_boxes * 5)` conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 5, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_loc')(conv4_3_norm) fc7_mbox_loc = Conv2D(n_boxes[1] * 5, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_loc')(fc7) conv6_2_mbox_loc = Conv2D(n_boxes[2] * 5, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc')(conv6_2) conv7_2_mbox_loc = Conv2D(n_boxes[3] * 5, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc')(conv7_2) conv8_2_mbox_loc = Conv2D(n_boxes[4] * 5, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(conv8_2) conv9_2_mbox_loc = Conv2D(n_boxes[5] * 5, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(conv9_2) ### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) # Output shape of anchors: `(batch, height, width, n_boxes, 10)` conv4_3_norm_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[0], aspect_ratios=aspect_ratios[0], this_steps=steps[0], this_offsets=offsets[0], angles = angles[0], variances=variances, normalize_coords=normalize_coords, name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc) fc7_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], aspect_ratios=aspect_ratios[1], this_steps=steps[1], this_offsets=offsets[1], angles = angles[1], variances=variances, normalize_coords=normalize_coords, name='fc7_mbox_priorbox')(fc7_mbox_loc) conv6_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[2], aspect_ratios=aspect_ratios[2], this_steps=steps[2], this_offsets=offsets[2], angles = angles[2], variances=variances, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[3], aspect_ratios=aspect_ratios[3], this_steps=steps[3], this_offsets=offsets[3], angles = angles[3], variances=variances, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[4], aspect_ratios=aspect_ratios[4], this_steps=steps[4], this_offsets=offsets[4], angles = angles[4], variances=variances, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[5], aspect_ratios=aspect_ratios[5], this_steps=steps[5], this_offsets=offsets[5], angles = angles[5], variances=variances, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) ### Reshape # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv4_3_norm_mbox_conf_reshape = Reshape((-1, n_classes), name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape((-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape((-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 5)` # We want the five box coordinates isolated in the last axis to compute the smooth L1 loss conv4_3_norm_mbox_loc_reshape = Reshape((-1, 5), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape((-1, 5), name='fc7_mbox_loc_reshape')(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape((-1, 5), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape((-1, 5), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape((-1, 5), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape((-1, 5), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 10)` conv4_3_norm_mbox_priorbox_reshape = Reshape((-1, 10), name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape((-1, 10), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape((-1, 10), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape((-1, 10), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape((-1, 10), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape((-1, 10), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox) ### Concatenate the predictions from the different layers # Axis 0 (batch) and axis 2 (n_classes or 5, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1, the number of boxes per layer # Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape]) # Output shape of `mbox_loc`: (batch, n_boxes_total, 5) mbox_loc = Concatenate(axis=1, name='mbox_loc')([conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape]) # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 10) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate the class and box predictions and the anchors to one large predictions vector # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 5 + 10) predictions = Concatenate(axis=2, name='predictions')([mbox_conf_softmax, mbox_loc, mbox_priorbox]) model = Model(inputs=x, outputs=predictions) if return_predictor_sizes: predictor_sizes = np.array([conv4_3_norm_mbox_conf._keras_shape[1:3], fc7_mbox_conf._keras_shape[1:3], conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3]]) return model, predictor_sizes else: return model
def ssdmod(x, geo_1, geo_2, suf): x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer' + '_' + suf)(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization' + '_' + suf)(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization' + '_' + suf)(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap' + '_' + suf)(x1) conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_1' + '_' + suf)(x1) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_2' + '_' + suf)(conv1_1) pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1' + '_' + suf)(conv1_2) conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_1' + '_' + suf)(pool1) conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_2' + '_' + suf)(conv2_1) pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2' + '_' + suf)(conv2_2) conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_1' + '_' + suf)(pool2) conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_2' + '_' + suf)(conv3_1) conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_3' + '_' + suf)(conv3_2) pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3' + '_' + suf)(conv3_3) conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_1' + '_' + suf)(pool3) conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_2' + '_' + suf)(conv4_1) conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3' + '_' + suf)(conv4_2) pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4' + '_' + suf)(conv4_3) conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_1' + '_' + suf)(pool4) conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_2' + '_' + suf)(conv5_1) conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_3' + '_' + suf)(conv5_2) pool5 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same', name='pool5' + '_' + suf)(conv5_3) fc6 = Conv2D(1024, (3, 3), dilation_rate=(6, 6), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc6' + '_' + suf)(pool5) fc7 = Conv2D(1024, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7' + '_' + suf)(fc6) conv6_1 = Conv2D(256, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_1' + '_' + suf)(fc7) conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv6adding' + '_' + suf)(conv6_1) conv6_2 = Conv2D(512, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2' + '_' + suf)(conv6_1) conv7_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_1' + '_' + suf)(conv6_2) conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv7adding' + '_' + suf)(conv7_1) conv7_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2' + '_' + suf)(conv7_1) conv8_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_1' + '_' + suf)(conv7_2) conv8_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2' + '_' + suf)(conv8_1) conv9_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_1' + '_' + suf)(conv8_2) conv9_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2' + '_' + suf)(conv9_1) # Feed conv4_3 into the L2 normalization layer conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm' + '_' + suf)(conv4_3) ### Build the convolutional predictor layers on top of the base network # We precidt `n_classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * n_classes` # Output shape of the confidence layers: `(batch, height, width, n_boxes * n_classes)` conv4_3_norm_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_conf' + '_' + suf)(conv4_3_norm) fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_conf' + '_' + suf)(fc7) conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf' + '_' + suf)(conv6_2) conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf' + '_' + suf)(conv7_2) conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf' + '_' + suf)(conv8_2) conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf' + '_' + suf)(conv9_2) # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4` # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)` conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_loc' + '_' + suf)(conv4_3_norm) fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_loc' + '_' + suf)(fc7) conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc' + '_' + suf)(conv6_2) conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc' + '_' + suf)(conv7_2) conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc' + '_' + suf)(conv8_2) conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc' + '_' + suf)(conv9_2) ### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) # Output shape of anchors: `(batch, height, width, n_boxes, 8)` conv4_3_norm_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv4_3_norm_mbox_priorbox' + '_' + suf)(conv4_3_norm_mbox_loc) fc7_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='fc7_mbox_priorbox' + '_' + suf)(fc7_mbox_loc) conv6_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox' + '_' + suf)(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox' + '_' + suf)(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox' + '_' + suf)(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox' + '_' + suf)(conv9_2_mbox_loc) ### Reshape # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv4_3_norm_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv4_3_norm_mbox_conf_reshape' + '_' + suf)(conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape( (-1, n_classes), name='fc7_mbox_conf_reshape' + '_' + suf)(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv6_2_mbox_conf_reshape' + '_' + suf)(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv7_2_mbox_conf_reshape' + '_' + suf)(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv8_2_mbox_conf_reshape' + '_' + suf)(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv9_2_mbox_conf_reshape' + '_' + suf)(conv9_2_mbox_conf) # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss conv4_3_norm_mbox_loc_reshape = Reshape( (-1, 4), name='conv4_3_norm_mbox_loc_reshape' + '_' + suf)(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape( (-1, 4), name='fc7_mbox_loc_reshape' + '_' + suf)(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv6_2_mbox_loc_reshape' + '_' + suf)(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv7_2_mbox_loc_reshape' + '_' + suf)(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv8_2_mbox_loc_reshape' + '_' + suf)(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv9_2_mbox_loc_reshape' + '_' + suf)(conv9_2_mbox_loc) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` conv4_3_norm_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv4_3_norm_mbox_priorbox_reshape' + '_' + suf)(conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape( (-1, 8), name='fc7_mbox_priorbox_reshape' + '_' + suf)(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv6_2_mbox_priorbox_reshape' + '_' + suf)(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv7_2_mbox_priorbox_reshape' + '_' + suf)(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv8_2_mbox_priorbox_reshape' + '_' + suf)(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv9_2_mbox_priorbox_reshape' + '_' + suf)(conv9_2_mbox_priorbox) mbox_conf = Concatenate(axis=1, name='mbox_conf' + '_' + suf)([ conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape ]) # Output shape of `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc' + '_' + suf)([ conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape ]) # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox' + '_' + suf)([ conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape ]) ### Concatenate the predictions from the different layers model = Model(input=[x, geo_1, geo_2], output=[mbox_conf, mbox_loc, mbox_priorbox]) return model
def mobilenet_ssd_300(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): n_predictor_layers = 6 # The number of predictor conv layers in the network is 6 for the original SSD300. n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1 & len(n_boxes) != 0: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers # print("Boxes:{}".format(n_boxes)) ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # Build the network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) mobilenet = MobileNet(input_shape=(224, 224, 3), include_top=False, weights='imagenet') FeatureExtractor = Model( inputs=mobilenet.input, outputs=mobilenet.get_layer('conv_pw_5_relu').output) mobilenet_conv_pw_5_relu = FeatureExtractor(x1) conv6dw = SeparableConv2D(256, (3, 3), padding='same', strides=(2, 2), name='conv_dw_6')(mobilenet_conv_pw_5_relu) conv6dw = BatchNormalization(momentum=0.99, name='conv_dw_6_bn')(conv6dw) conv6dw = ReLU(6., name='conv_dw_6_relu')(conv6dw) conv6pw = Conv2D(256, (1, 1), padding='same', name='conv_pw_6')(conv6dw) conv6pw = BatchNormalization(momentum=0.99, name='conv_pw_6_bn')(conv6pw) conv6pw = ReLU(6., name='conv_pw_6_relu')(conv6pw) conv7dw = SeparableConv2D(512, (3, 3), padding='same', name='conv_dw_7')(conv6pw) conv7dw = BatchNormalization(momentum=0.99, name='conv_dw_7_bn')(conv7dw) conv7dw = ReLU(6., name='conv_dw_7_relu')(conv7dw) conv7pw = Conv2D(512, (1, 1), padding='same', name='conv_pw_7')(conv7dw) conv7pw = BatchNormalization(momentum=0.99, name='conv_pw_7_bn')(conv7pw) conv7pw = ReLU(6., name='conv_pw_7_relu')(conv7pw) conv8dw = SeparableConv2D(512, (3, 3), padding='same', name='conv_dw_8')(conv7pw) conv8dw = BatchNormalization(momentum=0.99, name='conv_dw_8_bn')(conv8dw) conv8dw = ReLU(6., name='conv_dw_8_relu')(conv8dw) conv8pw = Conv2D(512, (1, 1), padding='same', name='conv_pw_8')(conv8dw) conv8pw = BatchNormalization(momentum=0.99, name='conv_pw_8_bn')(conv8pw) conv8pw = ReLU(6., name='conv_pw_8_relu')(conv8pw) conv9dw = SeparableConv2D(512, (3, 3), padding='same', name='conv_dw_9')(conv8pw) conv9dw = BatchNormalization(momentum=0.99, name='conv_dw_9_bn')(conv9dw) conv9dw = ReLU(6., name='conv_dw_9_relu')(conv9dw) conv9pw = Conv2D(512, (1, 1), padding='same', name='conv_pw_9')(conv9dw) conv9pw = BatchNormalization(momentum=0.99, name='conv_pw_9_bn')(conv9pw) conv9pw = ReLU(6., name='conv_pw_9_relu')(conv9pw) conv10dw = SeparableConv2D(512, (3, 3), padding='same', name='conv_dw_10')(conv9pw) conv10dw = BatchNormalization(momentum=0.99, name='conv_dw_10_bn')(conv10dw) conv10dw = ReLU(6., name='conv_dw_10_relu')(conv10dw) conv10pw = Conv2D(512, (1, 1), padding='same', name='conv_pw_10')(conv10dw) conv10pw = BatchNormalization(momentum=0.99, name='conv_pw_10_bn')(conv10pw) conv10pw = ReLU(6., name='conv_pw_10_relu')(conv10pw) conv11dw = SeparableConv2D(512, (3, 3), padding='same', name='conv_dw_11')(conv10pw) conv11dw = BatchNormalization(momentum=0.99, name='conv_dw_11_bn')(conv11dw) conv11dw = ReLU(6., name='conv_dw_11_relu')(conv11dw) conv11pw = Conv2D(512, (1, 1), padding='same', name='conv_pw_11')(conv11dw) conv11pw = BatchNormalization(momentum=0.99, name='conv_pw_11_bn')(conv11pw) conv11pw = ReLU(6., name='conv_pw_11_relu')(conv11pw) conv12dw = SeparableConv2D(512, (3, 3), strides=(2, 2), padding='same', name='conv_dw_12')(conv11pw) conv12dw = BatchNormalization(momentum=0.99, name='conv_dw_12_bn')(conv12dw) conv12dw = ReLU(6., name='conv_dw_12_relu')(conv12dw) conv12pw = Conv2D(1024, (1, 1), padding='same', name='conv_pw_12')(conv12dw) conv12pw = BatchNormalization(momentum=0.99, name='conv_pw_12_bn')(conv12pw) conv12pw = ReLU(6., name='conv_pw_12_relu')(conv12pw) conv13dw = SeparableConv2D(1024, (3, 3), padding='same', name='conv_dw_13')(conv12pw) conv13dw = BatchNormalization(momentum=0.99, name='conv_dw_13_bn')(conv13dw) conv13dw = ReLU(6., name='conv_dw_13_relu')(conv13dw) conv13pw = Conv2D(1024, (1, 1), padding='same', name='conv_pw_13')(conv13dw) conv13pw = BatchNormalization(momentum=0.99, name='conv_pw_13_bn')(conv13pw) conv13pw = ReLU(6., name='conv_pw_13_relu')(conv13pw) conv14_1 = Conv2D(256, (1, 1), padding='same', name='conv14_1')(conv13pw) conv14_1 = BatchNormalization(momentum=0.99, name='bn14_1')(conv14_1) conv14_1 = ReLU(6., name='conv14_1_relu')(conv14_1) conv14_2dw = SeparableConv2D(512, (3, 3), strides=(2, 2), padding='same', name='conv_dw_14_2')(conv14_1) conv14_2dw = BatchNormalization(momentum=0.99, name='conv_dw_14_2_bn')(conv14_2dw) conv14_2dw = ReLU(6., name='conv_dw_14_2_relu')(conv14_2dw) conv14_2pw = Conv2D(512, (1, 1), padding='same', name='conv_pw_14_2')(conv14_2dw) conv14_2pw = BatchNormalization(momentum=0.99, name='conv_pw_14_2_bn')(conv14_2pw) conv14_2pw = ReLU(6., name='conv_pw_14_2_relu')(conv14_2pw) conv15_1 = Conv2D(128, (1, 1), padding='same', name='conv15_1')(conv14_2pw) conv15_1 = BatchNormalization(momentum=0.99, name='bn15_1')(conv15_1) conv15_1 = ReLU(6., name='conv15_1_relu')(conv15_1) conv15_2dw = SeparableConv2D(256, (3, 3), name='conv_dw_15_2')(conv15_1) conv15_2dw = BatchNormalization(momentum=0.99, name='conv_dw_15_2_bn')(conv15_2dw) conv15_2dw = ReLU(6., name='conv_dw_15_2_relu')(conv15_2dw) conv15_2pw = Conv2D(256, (1, 1), padding='same', name='conv_pw_15_2')(conv15_2dw) conv15_2pw = BatchNormalization(momentum=0.99, name='conv_pw_15_2_bn')(conv15_2pw) conv15_2pw = ReLU(6., name='conv_pw_15_2_relu')(conv15_2pw) conv16_1 = Conv2D(128, (1, 1), padding='same', name='conv16_1')(conv15_2pw) conv16_1 = BatchNormalization(momentum=0.99, name='bn16_1')(conv16_1) conv16_1 = ReLU(6., name='conv16_1_relu')(conv16_1) conv16_2 = SeparableConv2D(256, (3, 3), name='conv16_2_')(conv16_1) conv16_2 = BatchNormalization(momentum=0.99, name='bn16_2')(conv16_2) conv16_2 = ReLU(6., name='conv16_2_relu')(conv16_2) conv5_mbox_loc = Conv2D(n_boxes[0] * 4, (1, 1), padding='same', name='conv5_mbox_loc')(mobilenet_conv_pw_5_relu) conv11_mbox_loc = Conv2D(n_boxes[1] * 4, (1, 1), padding='same', name='conv11_mbox_loc_')(conv11pw) conv13_mbox_loc = Conv2D(n_boxes[2] * 4, (1, 1), padding='same', name='conv13_mbox_loc_')(conv13pw) conv14_mbox_loc = Conv2D(n_boxes[3] * 4, (1, 1), padding='same', name='conv14_mbox_loc_')(conv14_2pw) conv15_mbox_loc = Conv2D(n_boxes[4] * 4, (1, 1), padding='same', name='conv15_mbox_loc_')(conv15_2pw) conv16_mbox_loc = Conv2D(n_boxes[5] * 4, (1, 1), padding='same', name='conv16_mbox_loc_')(conv16_2) conv5_mbox_loc_reshape = Reshape( (-1, 4), name='conv5_mbox_loc_reshape')(conv5_mbox_loc) conv11_mbox_loc_reshape = Reshape( (-1, 4), name='conv11_mbox_loc_reshape')(conv11_mbox_loc) conv13_mbox_loc_reshape = Reshape( (-1, 4), name='conv13_mbox_loc_reshape')(conv13_mbox_loc) conv14_mbox_loc_reshape = Reshape( (-1, 4), name='conv14_mbox_loc_reshape')(conv14_mbox_loc) conv15_mbox_loc_reshape = Reshape( (-1, 4), name='conv15_mbox_loc_reshape')(conv15_mbox_loc) conv16_mbox_loc_reshape = Reshape( (-1, 4), name='conv16_mbox_loc_reshape')(conv16_mbox_loc) conv5_mbox_conf = Conv2D(n_boxes[0] * n_classes, (1, 1), padding='same', name='conv5_mbox_conf')(mobilenet_conv_pw_5_relu) conv11_mbox_conf = Conv2D(n_boxes[1] * n_classes, (1, 1), padding='same', name='conv11_mbox_conf_')(conv11pw) conv13_mbox_conf = Conv2D(n_boxes[2] * n_classes, (1, 1), padding='same', name='conv13_mbox_conf_')(conv13pw) conv14_mbox_conf = Conv2D(n_boxes[3] * n_classes, (1, 1), padding='same', name='conv14_mbox_conf_')(conv14_2pw) conv15_mbox_conf = Conv2D(n_boxes[4] * n_classes, (1, 1), padding='same', name='conv15_mbox_conf_')(conv15_2pw) conv16_mbox_conf = Conv2D(n_boxes[5] * n_classes, (1, 1), padding='same', name='conv16_mbox_conf_')(conv16_2) conv5_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv5_mbox_conf_reshape')(conv5_mbox_conf) conv11_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv11_mbox_conf_reshape')(conv11_mbox_conf) conv13_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv13_mbox_conf_reshape')(conv13_mbox_conf) conv14_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv14_mbox_conf_reshape')(conv14_mbox_conf) conv15_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv15_mbox_conf_reshape')(conv15_mbox_conf) conv16_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv16_mbox_conf_reshape')(conv16_mbox_conf) conv5_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv5_mbox_priorbox')(mobilenet_conv_pw_5_relu) conv11_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv11_mbox_priorbox')(conv11pw) conv13_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv13_mbox_priorbox')(conv13pw) conv14_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv14_mbox_priorbox')(conv14_2pw) conv15_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv15_mbox_priorbox')(conv15_2pw) conv16_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv16_mbox_priorbox')(conv16_2) conv5_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv5_mbox_priorbox_reshape')(conv5_mbox_priorbox) conv11_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv11_mbox_priorbox_reshape')(conv11_mbox_priorbox) conv13_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv13_mbox_priorbox_reshape')(conv13_mbox_priorbox) conv14_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv14_mbox_priorbox_reshape')(conv14_mbox_priorbox) conv15_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv15_mbox_priorbox_reshape')(conv15_mbox_priorbox) conv16_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv16_mbox_priorbox_reshape')(conv16_mbox_priorbox) mbox_loc = concatenate([ conv5_mbox_loc_reshape, conv11_mbox_loc_reshape, conv13_mbox_loc_reshape, conv14_mbox_loc_reshape, conv15_mbox_loc_reshape, conv16_mbox_loc_reshape ], axis=1, name='mbox_loc') mbox_conf = concatenate([ conv5_mbox_conf_reshape, conv11_mbox_conf_reshape, conv13_mbox_conf_reshape, conv14_mbox_conf_reshape, conv15_mbox_conf_reshape, conv16_mbox_conf_reshape ], axis=1, name='mbox_conf') mbox_priorbox = concatenate([ conv5_mbox_priorbox_reshape, conv11_mbox_priorbox_reshape, conv13_mbox_priorbox_reshape, conv14_mbox_priorbox_reshape, conv15_mbox_priorbox_reshape, conv16_mbox_priorbox_reshape ], axis=1, name='mbox_priorbox') mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) predictions = concatenate([mbox_conf_softmax, mbox_loc, mbox_priorbox], axis=2, name='predictions') if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ conv5_mbox_conf._keras_shape[1:3], conv11_mbox_conf._keras_shape[1:3], conv13_mbox_conf._keras_shape[1:3], conv14_mbox_conf._keras_shape[1:3], conv15_mbox_conf._keras_shape[1:3], conv16_mbox_conf._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def ssd_300(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): ''' Xây dựng model SSD300 với keras. Base network được sử dụng là VGG16. Chú ý: Yêu cầu Keras>=v2.0; TensorFlow backend>=v1.0. Arguments: image_size (tuple): Kích thước image input `(height, width, channels)`. n_classes (int): Số classes, chẳng hạn 20 cho Pascal VOC dataset, 80 cho MS COCO dataset. mode (str, optional): Một trong những dạng 'training', 'inference' và 'inference_fast'. 'training' mode: Đầu ra của model là raw prediction tensor. 'inference' và 'inference_fast' modes: raw predictions được decoded thành tọa độ đã được filtered thông qua threshold. l2_regularization (float, optional): L2-regularization rate. Áp dụng cho toàn bộ các convolutional layers. min_scale (float, optional): Nhân tố scaling nhỏ nhất cho các size của anchor boxes. Tỷ lệ này được tính trên so sánh với cạnh ngắn hơn của hình ảnh input. max_scale (float, optional): Nhân tố scale lớn nhất cho các size của anchor boxes. scales (list, optional): List các số floats chứa các nhân tố scaling của các convolutional predictor layer. List này phải lớn hơn số lượng các predictor layers là 1 để sử dụng cho trường hợp aspect ratio = 1 sẽ tính thêm next scale. Trong TH sử dụng scales thì interpolate theo min_scale và max_scale để tính list scales sẽ không được sử dụng. aspect_ratios_global (list, optional): List của các aspect ratios mà các anchor boxes được tạo thành. List này được áp dụng chung trên toàn bộ các prediction layers. aspect_ratios_per_layer (list, optional): List của các list aspect ratio cho mỗi một prediction layer. Nếu được truyền vào sẽ override `aspect_ratios_global`. two_boxes_for_ar1 (bool, optional): Chỉ áp dụng khi aspect ratio lists chứa 1. Sẽ bị loại bỏ trong các TH khác. Nếu `True`, 2 anchor boxes sẽ được tạo ra ứng với aspect ratio = 1. anchor box đầu tiên tạo thành bằng cách sử scale, anchor box thứ 2 được tạo thành bằng trung bình hình học của scale và next scale. steps (list, optional): `None` hoặc là list với rất nhiều các phần tử có số lượng bằng với số lượng layers. Mỗi phần tử đại diện cho mỗi một predictor layer có bao nhiêu pixels khoảng cách giữa các tâm của anchor box. steps có thể gồm 2 số đại diện cho (step_width, step_height). nếu không có steps nào được đưa ra thì chúng ta sẽ tính để cho khoảng các giữa các tâm của anchor box là bằng nhau offsets (list, optional): None hoặc là các con số đại diện cho mỗi một predictor layer bao nhiêu pixels từ góc trên và bên trái mở rộng của ảnh clip_boxes (bool, optional): Nếu `True`, giới hạn tọa độ các anchor box để nằm trong boundaries của image. variances (list, optional): Một list gồm 4 số floats >0. Một anchor box offset tương ứng với mỗi tọa độ sẽ được chi cho giá trị variance tương ứng. coords (str, optional): Tọa độ của box được sử dụng bên trong model (chẳng hạn, nó không là input format của ground truth labels). Có thể là dạng 'centroids' format `(cx, cy, w, h)` (box center coordinates, width, and height), 'minmax' format `(xmin, xmax, ymin, ymax)`, hoặc 'corners' format `(xmin, ymin, xmax, ymax)`. normalize_coords (bool, optional): Được đặt là `True` nếu model được giả định sử dụng tọa độ tương đối thay vì tuyệt đối coordinates, chẳng hạn nếu model dự báo tọa độ box nằm trong [0, 1] thay vì tọa độ tuyệt đối. subtract_mean (array-like, optional): `None` hoặc một array object với bất kì shape nào mà dạng mở rộng phù hợp với shape của ảnh. Gía trị của nó được bớt đi từ độ lớn pixel của ảnh. The elements of this array will be Chẳng hạn truyền vào một list gồm 3 số nguyên để tính toán trung bình chuẩn hóa cho các kênh của ảnh. divide_by_stddev (array-like, optional): `None` hoặc một array object. Tương tự như subtract_mean nhưng được chia cho từ độ lớn của ảnh để tính chuẩn hóa. swap_channels (list, optional): Là `False` hoặc một list các số nguyên biểu diễn thứ tự kì vọng mà trong đó đầu vào các channels của ảnh có thể được hoán đổi. confidence_thresh (float, optional): Một số float nằm trong khoảng [0,1), là ngưỡng tin cậy nhỏ nhất trong phân loại của một lớp xảy ra. iou_threshold (float, optional): Một float nằm trong khoảng [0,1]. Tất cả các boxes có chỉ số Jaccard similarity lớn hơn hoặc bằng `iou_threshold` sẽ được xem xét là chứa vệt thể bên trong nó. top_k (int, optional): Điểm dự báo cáo nhất được giữ trong mỗi batch item sau bước non-maximum suppression stage. nms_max_output_size (int, optional): Số lượng lớn nhất các dự báo sẽ được chuyển qua bước NMS stage. return_predictor_sizes (bool, optional): Nếu `True`, hàm số này sẽ không chỉ trả về mô hình, mà còn trả về một list chứa các chiều của predictor layers. Returns: model: The Keras SSD300 model. predictor_sizes (optional): Một numpy array chứa các phần `(height, width)` của output tensor shape tương ứng với mỗi convolutional predictor layer. References: https://arxiv.org/abs/1512.02325v5 ''' n_predictor_layers = 6 # Số lượng các preductor convolutional layers trong network là 6 cho original SSD300. n_classes += 1 # Số lượng classes, + 1 để tính thêm background class. l2_reg = l2_regularization # tham số chuẩn hóa của norm chuẩn l2. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # Một số lỗi ngoại lệ. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) # Tạo list scales if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # Tính các tham số của anchor box. ############################################################################ # Thiết lập aspect ratios cho mỗi predictor layer (chỉ cần thiết cho tính toán anchor box layers). if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Tính số lượng boxes được dự báo / 1 cell cho mỗi predictor layer. # Chúng ta cần biết bao nhiêu channels các predictor layers cần có. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 cho trường hợp aspect ratio = 1 else: n_boxes.append(len(ar)) else: # Nếu chỉ 1 global aspect ratio list được truyền vào thì số lượng boxes là như nhau cho mọi layers. if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Xác định các hàm số cho Lambda layers bên dưới. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # Bước 1: Xây dựng network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) ############################################################################ # Bước 1.1: Tính toán base network là mạng VGG16 ############################################################################ conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_1')(x1) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_2')(conv1_1) pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1')(conv1_2) conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_1')(pool1) conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_2')(conv2_1) pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2')(conv2_2) conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_1')(pool2) conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_2')(conv3_1) conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_3')(conv3_2) pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3')(conv3_3) conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_1')(pool3) conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_2')(conv4_1) conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3')(conv4_2) pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4')(conv4_3) conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_1')(pool4) conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_2')(conv5_1) conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_3')(conv5_2) pool5 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same', name='pool5')(conv5_3) ############################################################################ # Bước 1.2: Áp dụng các convolutional filter có kích thước (3 x 3) để tính toán ra features map. ############################################################################ fc6 = Conv2D(1024, (3, 3), dilation_rate=(6, 6), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc6')(pool5) print('fully connected 6: ', fc6.get_shape()) fc7 = Conv2D(1024, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7')(fc6) print('fully connected 7: ', fc7.get_shape()) conv6_1 = Conv2D(256, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_1')(fc7) conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv6_padding')(conv6_1) conv6_2 = Conv2D(512, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2')(conv6_1) print('conv6_2: ', conv6_2.get_shape()) conv7_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_1')(conv6_2) conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv7_padding')(conv7_1) conv7_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2')(conv7_1) print('conv7_2: ', conv7_2.get_shape()) conv8_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_1')(conv7_2) conv8_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2')(conv8_1) print('conv8_2: ', conv8_2.get_shape()) conv9_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_1')(conv8_2) conv9_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2')(conv9_1) print('conv9_2: ', conv9_2.get_shape()) # Feed conv4_3 vào the L2 normalization layer conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3) print('conv4_3_norm.shape: ', conv4_3_norm.get_shape()) ############################################################################ # Bước 1.3: Xác định output phân phối xác suất theo các classes ứng với mỗi một default bounding box. ############################################################################ ### Xây dựng các convolutional predictor layers tại top của base network # Chúng ta dự báo các giá trị confidence cho mỗi box, do đó confidence predictors có độ sâu `n_boxes * n_classes` # Đầu ra của confidence layers có shape: `(batch, height, width, n_boxes * n_classes)` conv4_3_norm_mbox_conf = Conv2D( n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_conf')(conv4_3_norm) print('conv4_3_norm_mbox_conf.shape: ', conv4_3_norm_mbox_conf.get_shape()) fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_conf')(fc7) print('fc7_mbox_conf.shape: ', fc7_mbox_conf.get_shape()) conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf')(conv6_2) conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf')(conv7_2) conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf')(conv8_2) conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf')(conv9_2) print('conv9_2_mbox_conf: ', conv9_2_mbox_conf.get_shape()) ############################################################################ # Bước 1.4: Xác định output các tham số offset của default bounding boxes tương ứng với mỗi cell trên các features map. ############################################################################ # Chúng ta dự báo 4 tọa độ cho mỗi box, do đó localization predictors có độ sâu `n_boxes * 4` # Output shape của localization layers: `(batch, height, width, n_boxes * 4)` conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_loc')(conv4_3_norm) print('conv4_3_norm_mbox_loc: ', conv4_3_norm_mbox_loc.get_shape()) fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_loc')(fc7) conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc')(conv6_2) conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc')(conv7_2) conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(conv8_2) conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(conv9_2) print('conv9_2_mbox_loc: ', conv9_2_mbox_loc.get_shape()) ############################################################################ # Bước 1.5: Tính toán các AnchorBoxes làm cơ sở để dự báo offsets cho các predicted bounding boxes bao quan vật thể ############################################################################ ### Khởi tạo các anchor boxes (được gọi là "priors" trong code gốc Caffe/C++ của mô hình) # Shape output của anchors: `(batch, height, width, n_boxes, 8)` conv4_3_norm_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc) print('conv4_3_norm_mbox_priorbox: ', conv4_3_norm_mbox_priorbox.get_shape()) fc7_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='fc7_mbox_priorbox')(fc7_mbox_loc) print('fc7_mbox_priorbox: ', fc7_mbox_priorbox.get_shape()) conv6_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) print('conv6_2_mbox_priorbox: ', conv6_2_mbox_priorbox.get_shape()) conv7_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) print('conv7_2_mbox_priorbox: ', conv7_2_mbox_priorbox.get_shape()) conv8_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) print('conv8_2_mbox_priorbox: ', conv8_2_mbox_priorbox.get_shape()) conv9_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) print('conv9_2_mbox_priorbox: ', conv9_2_mbox_priorbox.get_shape()) ############################################################################ # Bước 2: Reshape lại các output tensor shape ############################################################################ ############################################################################ # Bước 2.1: Reshape output của class predictions ############################################################################ # Reshape các class predictions, trả về 3D tensors có shape `(batch, height * width * n_boxes, n_classes)` # Chúng ta muốn các classes là tách biệt nhau trên last axis để tính softmax trên chúng. conv4_3_norm_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape( (-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) print('conv4_3_norm_mbox_conf_reshape: ', conv4_3_norm_mbox_conf_reshape.get_shape()) print('fc7_mbox_conf_reshape: ', fc7_mbox_conf_reshape.get_shape()) print('conv9_2_mbox_conf_reshape: ', conv9_2_mbox_conf_reshape.get_shape()) print('conv9_2_mbox_conf_reshape: ', conv9_2_mbox_conf_reshape.get_shape()) print('conv9_2_mbox_conf_reshape: ', conv9_2_mbox_conf_reshape.get_shape()) ############################################################################ # Bước 2.2: Reshape output của bounding box predictions ############################################################################ # Reshape các box predictions, trả về 3D tensors có shape `(batch, height * width * n_boxes, 4)` # Chúng ta muốn 4 tọa độ box là tách biệt nhau trên last axis để tính hàm smooth L1 loss conv4_3_norm_mbox_loc_reshape = Reshape( (-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape((-1, 4), name='fc7_mbox_loc_reshape')(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) print('conv4_3_norm_mbox_loc_reshape: ', conv4_3_norm_mbox_loc_reshape.get_shape()) print('fc7_mbox_loc_reshape: ', fc7_mbox_loc_reshape.get_shape()) print('conv6_2_mbox_loc_reshape: ', conv6_2_mbox_loc_reshape.get_shape()) print('conv7_2_mbox_loc_reshape: ', conv7_2_mbox_loc_reshape.get_shape()) print('conv8_2_mbox_loc_reshape: ', conv8_2_mbox_loc_reshape.get_shape()) print('conv9_2_mbox_loc_reshape: ', conv9_2_mbox_loc_reshape.get_shape()) ############################################################################ # Bước 2.3: Reshape output của anchor box ############################################################################ # Reshape anchor box tensors, trả về 3D tensors có shape `(batch, height * width * n_boxes, 8)` conv4_3_norm_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape( (-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox) print('conv4_3_norm_mbox_priorbox_reshape: ', conv4_3_norm_mbox_priorbox_reshape.get_shape()) print('fc7_mbox_priorbox_reshape: ', fc7_mbox_priorbox_reshape.get_shape()) print('conv6_2_mbox_priorbox_reshape: ', conv6_2_mbox_priorbox_reshape.get_shape()) print('conv7_2_mbox_priorbox_reshape: ', conv7_2_mbox_priorbox_reshape.get_shape()) print('conv8_2_mbox_priorbox_reshape: ', conv8_2_mbox_priorbox_reshape.get_shape()) print('conv9_2_mbox_priorbox_reshape: ', conv9_2_mbox_priorbox_reshape.get_shape()) ### Concatenate các predictions từ các layers khác nhau ############################################################################ # Bước 3: Concatenate các boxes trên layers ############################################################################ ############################################################################ # Bước 3.1: Concatenate confidence output box ############################################################################ # Axis 0 (batch) và axis 2 (n_classes hoặc 4) là xác định duy nhất cho toàn bộ các predictions layer # nên chúng ta muốn concatenate theo axis 1, số lượng các boxes trên layer # Output shape của `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([ conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape ]) print('mbox_conf.shape: ', mbox_conf.get_shape()) ############################################################################ # Bước 3.2: Concatenate location output box ############################################################################ # Output shape của `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([ conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape ]) print('mbox_loc.shape: ', mbox_loc.get_shape()) ############################################################################ # Bước 3.3: Concatenate anchor output box ############################################################################ # Output shape của `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([ conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape ]) print('mbox_priorbox.shape: ', mbox_priorbox.get_shape()) ############################################################################ # Bước 4: Tính toán output ############################################################################ ############################################################################ # Bước 4.1 : Xây dựng các hàm loss function cho confidence ############################################################################ # tọa độ của box predictions sẽ được truyền vào hàm loss function, # nhưng cho các dự báo lớp, chúng ta sẽ áp dụng một hàm softmax activation layer đầu tiên mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate các class và box predictions và the anchors thành một large predictions vector # Đầu ra của `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) print('predictions.shape: ', predictions.get_shape()) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ conv4_3_norm_mbox_conf._keras_shape[1:3], fc7_mbox_conf._keras_shape[1:3], conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def mn_model( image_size, n_classes, min_scale=0.1, max_scale=0.9, scales=None, aspect_ratios_global=[0.5, 1.0, 2.0], aspect_ratios_per_layer=None, two_boxes_for_ar1=True, limit_boxes=True, variances=[1.0, 1.0, 1.0, 1.0], coords="centroids", normalize_coords=False, ): n_predictor_layers = 6 # The number of predictor conv layers in the network is 6 for the original SSD300 # Get a few exceptions out of the way first if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios_conv4_3 = aspect_ratios_per_layer[0] aspect_ratios_fc7 = aspect_ratios_per_layer[1] aspect_ratios_conv6_2 = aspect_ratios_per_layer[2] aspect_ratios_conv7_2 = aspect_ratios_per_layer[3] aspect_ratios_conv8_2 = aspect_ratios_per_layer[4] aspect_ratios_conv9_2 = aspect_ratios_per_layer[5] else: aspect_ratios_conv4_3 = aspect_ratios_global aspect_ratios_fc7 = aspect_ratios_global aspect_ratios_conv6_2 = aspect_ratios_global aspect_ratios_conv7_2 = aspect_ratios_global aspect_ratios_conv8_2 = aspect_ratios_global aspect_ratios_conv9_2 = aspect_ratios_global # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for aspect_ratios in aspect_ratios_per_layer: if (1 in aspect_ratios) & two_boxes_for_ar1: n_boxes.append(len(aspect_ratios) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(aspect_ratios)) n_boxes_conv4_3 = n_boxes[ 0] # 4 boxes per cell for the original implementation n_boxes_fc7 = n_boxes[ 1] # 6 boxes per cell for the original implementation n_boxes_conv6_2 = n_boxes[ 2] # 6 boxes per cell for the original implementation n_boxes_conv7_2 = n_boxes[ 3] # 6 boxes per cell for the original implementation n_boxes_conv8_2 = n_boxes[ 4] # 4 boxes per cell for the original implementation n_boxes_conv9_2 = n_boxes[ 5] # 4 boxes per cell for the original implementation else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes_conv4_3 = n_boxes n_boxes_fc7 = n_boxes n_boxes_conv6_2 = n_boxes n_boxes_conv7_2 = n_boxes n_boxes_conv8_2 = n_boxes n_boxes_conv9_2 = n_boxes print("Height, Width, Channels :", image_size[0], image_size[1], image_size[2]) # Input image format img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] input_shape = (img_height, img_width, img_channels) img_input = Input(shape=input_shape) alpha = 1.0 depth_multiplier = 1 x = Lambda( lambda z: z / 255.0, # Convert input feature range to [-1,1] output_shape=(img_height, img_width, img_channels), name="lambda1", )(img_input) x = Lambda( lambda z: z - 0.5, # Convert input feature range to [-1,1] output_shape=(img_height, img_width, img_channels), name="lambda2", )(x) x = Lambda( lambda z: z * 2.0, # Convert input feature range to [-1,1] output_shape=(img_height, img_width, img_channels), name="lambda3", )(x) x = _conv_block(x, 32, alpha, strides=(2, 2)) x = _depthwise_conv_block_classification(x, 64, alpha, depth_multiplier, block_id=1) x = _depthwise_conv_block_classification(x, 128, alpha, depth_multiplier, strides=(2, 2), block_id=2) x = _depthwise_conv_block_classification(x, 128, alpha, depth_multiplier, block_id=3) x = _depthwise_conv_block_classification(x, 256, alpha, depth_multiplier, strides=(2, 2), block_id=4) x = _depthwise_conv_block_classification(x, 256, alpha, depth_multiplier, block_id=5) x = _depthwise_conv_block_classification(x, 512, alpha, depth_multiplier, strides=(2, 2), block_id=6) x = _depthwise_conv_block_classification(x, 512, alpha, depth_multiplier, block_id=7) x = _depthwise_conv_block_classification(x, 512, alpha, depth_multiplier, block_id=8) x = _depthwise_conv_block_classification(x, 512, alpha, depth_multiplier, block_id=9) x = _depthwise_conv_block_classification(x, 512, alpha, depth_multiplier, block_id=10) conv4_3 = _depthwise_conv_block_classification( x, 512, alpha, depth_multiplier, block_id=11) # 11 conv4_3 (300x300)-> 19x19 x = _depthwise_conv_block_classification(conv4_3, 1024, alpha, depth_multiplier, strides=(2, 2), block_id=12) # (300x300) -> 10x10 fc7 = _depthwise_conv_block_classification( x, 1024, alpha, depth_multiplier, block_id=13) # 13 fc7 (300x300) -> 10x10 conv6_1 = bn_conv( fc7, "detection_conv6_1", 256, 1, 1, subsample=(1, 1), border_mode="same", bias=conv_has_bias, ) conv6_2 = _depthwise_conv_block_detection( input=conv6_1, layer_name="detection_conv6_2", strides=(2, 2), pointwise_conv_filters=512, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=1, ) conv7_1 = bn_conv( conv6_2, "detection_conv7_1", 128, 1, 1, subsample=(1, 1), border_mode="same", bias=conv_has_bias, ) conv7_2 = _depthwise_conv_block_detection( input=conv7_1, layer_name="detection_conv7_2", strides=(2, 2), pointwise_conv_filters=256, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=2, ) # conv7_1 = Conv2D(128, (1, 1), activation='relu', padding='same', name='detection_conv7_1')(conv6_2) # conv7_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='same', name='detection_conv7_2')(conv7_1) conv8_1 = bn_conv( conv7_2, "detection_conv8_1", 128, 1, 1, subsample=(1, 1), border_mode="same", bias=conv_has_bias, ) conv8_2 = _depthwise_conv_block_detection( input=conv8_1, layer_name="detection_conv8_2", strides=(2, 2), pointwise_conv_filters=256, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=3, ) # # conv8_2 = bn_conv(conv8_1, 'detection_conv8_2', 256, 2, 2, subsample =(1,1), border_mode ='same', bias=conv_has_bias) conv9_1 = bn_conv( conv8_2, "detection_conv9_1", 64, 1, 1, subsample=(1, 1), border_mode="same", bias=conv_has_bias, ) # conv9_2 = bn_conv(conv9_1, 'detection_conv9_2', 128, 3, 3, subsample =(2,2), border_mode ='same', bias=conv_has_bias) conv9_2 = _depthwise_conv_block_detection( input=conv9_1, layer_name="detection_conv9_2", strides=(2, 2), pointwise_conv_filters=256, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=4, ) # Feed conv4_3 into the L2 normalization layer conv4_3_norm = L2Normalization(gamma_init=20, name="detection_conv4_3_norm")(conv4_3) conv4_3_norm_mbox_conf = _depthwise_conv_block_detection( input=conv4_3_norm, layer_name="detection_conv4_3_norm_mbox_conf", strides=(1, 1), pointwise_conv_filters=n_boxes_conv4_3 * n_classes, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=1, ) fc7_mbox_conf = _depthwise_conv_block_detection( input=fc7, layer_name="detection_fc7_mbox_conf", strides=(1, 1), pointwise_conv_filters=n_boxes_fc7 * n_classes, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=2, ) conv6_2_mbox_conf = _depthwise_conv_block_detection( input=conv6_2, layer_name="detection_conv6_2_mbox_conf", strides=(1, 1), pointwise_conv_filters=n_boxes_conv6_2 * n_classes, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=3, ) conv7_2_mbox_conf = _depthwise_conv_block_detection( input=conv7_2, layer_name="detection_conv7_2_mbox_conf", strides=(1, 1), pointwise_conv_filters=n_boxes_conv7_2 * n_classes, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=4, ) conv8_2_mbox_conf = _depthwise_conv_block_detection( input=conv8_2, layer_name="detection_conv8_2_mbox_conf", strides=(1, 1), pointwise_conv_filters=n_boxes_conv8_2 * n_classes, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=5, ) conv9_2_mbox_conf = _depthwise_conv_block_detection( input=conv9_2, layer_name="detection_conv9_2_mbox_conf", strides=(1, 1), pointwise_conv_filters=n_boxes_conv9_2 * n_classes, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=6, ) # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4` # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)` conv4_3_norm_mbox_loc = _depthwise_conv_block_detection( input=conv4_3_norm, layer_name="detection_conv4_3_norm_mbox_loc", strides=(1, 1), pointwise_conv_filters=n_boxes_conv4_3 * 4, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=1, ) fc7_mbox_loc = _depthwise_conv_block_detection( input=fc7, layer_name="detection_fc7_mbox_loc", strides=(1, 1), pointwise_conv_filters=n_boxes_fc7 * 4, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=2, ) conv6_2_mbox_loc = _depthwise_conv_block_detection( input=conv6_2, layer_name="detection_conv6_2_mbox_loc", strides=(1, 1), pointwise_conv_filters=n_boxes_conv6_2 * 4, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=3, ) conv7_2_mbox_loc = _depthwise_conv_block_detection( input=conv7_2, layer_name="detection_conv7_2_mbox_loc", strides=(1, 1), pointwise_conv_filters=n_boxes_conv7_2 * 4, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=4, ) conv8_2_mbox_loc = _depthwise_conv_block_detection( input=conv8_2, layer_name="detection_conv8_2_mbox_loc", strides=(1, 1), pointwise_conv_filters=n_boxes_conv8_2 * 4, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=5, ) conv9_2_mbox_loc = _depthwise_conv_block_detection( input=conv9_2, layer_name="detection_conv9_2_mbox_loc", strides=(1, 1), pointwise_conv_filters=n_boxes_conv9_2 * 4, alpha=alpha, depth_multiplier=depth_multiplier, padding="same", use_bias=True, block_id=5, ) ### Generate the anchor boxes # Output shape of anchors: `(batch, height, width, n_boxes, 8)` conv4_3_norm_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios_conv4_3, two_boxes_for_ar1=two_boxes_for_ar1, limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name="detection_conv4_3_norm_mbox_priorbox", )(conv4_3_norm) fc7_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios_fc7, two_boxes_for_ar1=two_boxes_for_ar1, limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name="detection_fc7_mbox_priorbox", )(fc7) conv6_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios_conv6_2, two_boxes_for_ar1=two_boxes_for_ar1, limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name="detection_conv6_2_mbox_priorbox", )(conv6_2) conv7_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios_conv7_2, two_boxes_for_ar1=two_boxes_for_ar1, limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name="detection_conv7_2_mbox_priorbox", )(conv7_2) conv8_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios_conv8_2, two_boxes_for_ar1=two_boxes_for_ar1, limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name="detection_conv8_2_mbox_priorbox", )(conv8_2) conv9_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios_conv9_2, two_boxes_for_ar1=two_boxes_for_ar1, limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name="detection_conv9_2_mbox_priorbox", )(conv9_2) ### Reshape # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv4_3_norm_mbox_conf_reshape = Reshape( (-1, n_classes), name="detection_conv4_3_norm_mbox_conf_reshape")( conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape( (-1, n_classes), name="detection_fc7_mbox_conf_reshape")(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape( (-1, n_classes), name="detection_conv6_2_mbox_conf_reshape")(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape( (-1, n_classes), name="detection_conv7_2_mbox_conf_reshape")(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape( (-1, n_classes), name="detection_conv8_2_mbox_conf_reshape")(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape( (-1, n_classes), name="detection_conv9_2_mbox_conf_reshape")(conv9_2_mbox_conf) # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss conv4_3_norm_mbox_loc_reshape = Reshape( (-1, 4), name="detection_conv4_3_norm_mbox_loc_reshape")(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape( (-1, 4), name="detection_fc7_mbox_loc_reshape")(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape( (-1, 4), name="detection_conv6_2_mbox_loc_reshape")(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape( (-1, 4), name="detection_conv7_2_mbox_loc_reshape")(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape( (-1, 4), name="detection_conv8_2_mbox_loc_reshape")(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape( (-1, 4), name="detection_conv9_2_mbox_loc_reshape")(conv9_2_mbox_loc) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` conv4_3_norm_mbox_priorbox_reshape = Reshape( (-1, 8), name="detection_conv4_3_norm_mbox_priorbox_reshape")( conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape( (-1, 8), name="detection_fc7_mbox_priorbox_reshape")(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape( (-1, 8), name="detection_conv6_2_mbox_priorbox_reshape")(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape( (-1, 8), name="detection_conv7_2_mbox_priorbox_reshape")(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape( (-1, 8), name="detection_conv8_2_mbox_priorbox_reshape")(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape( (-1, 8), name="detection_conv9_2_mbox_priorbox_reshape")(conv9_2_mbox_priorbox) ### Concatenate the predictions from the different layers # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1, the number of boxes per layer # Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name="detection_mbox_conf")([ conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape, ]) # Output shape of `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name="detection_mbox_loc")([ conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape, ]) # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name="detection_mbox_priorbox")([ conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape, ]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first mbox_conf_softmax = Activation( "softmax", name="detection_mbox_conf_softmax")(mbox_conf) # Concatenate the class and box predictions and the anchors to one large predictions vector # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name="detection_predictions")( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) model = Model(inputs=img_input, outputs=predictions) # model = Model(inputs=img_input, outputs=predictions) # Get the spatial dimensions (height, width) of the predictor conv layers, we need them to # be able to generate the default boxes for the matching process outside of the model during training. # Note that the original implementation performs anchor box matching inside the loss function. We don't do that. # Instead, we'll do it in the batch generator function. # The spatial dimensions are the same for the confidence and localization predictors, so we just take those of the conf layers. predictor_sizes = np.array([ conv4_3_norm_mbox_conf._keras_shape[1:3], fc7_mbox_conf._keras_shape[1:3], conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3], ]) model_layer = dict([(layer.name, layer) for layer in model.layers]) # for key in model_layer: # model_layer[key].trainable = True # model = Model(img_input, conv9_2) # model_layer = dict([(layer.name, layer) for layer in model.layers]) # predictor_sizes = 0 return model, model_layer, img_input, predictor_sizes
def SSD_ResFPN(image_size, n_classes, l2_regularization=0, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=None, two_boxes_for_ar1=True, steps=[2, 4, 8, 32, 64], offsets=None, clip_boxes=False, variances=[0.5, 0.5, 0.5, 0.5], coords='centroids', normalize_coords=True, subtract_mean=None, divide_by_stddev=None, swap_channels=False, confidence_thresh=0.5, iou_threshold=0.45, top_k=20, nms_max_output_size=40, return_predictor_sizes=False): # Pre-processes n_predictor_layers = 6 n_classes += 1 l2_reg = l2_regularization img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2] if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError("`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified.") if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError("It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}.".format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError("Either `min_scale` and `max_scale` or `scales` need to be specified.") if scales: if len(scales) != n_predictor_layers+1: raise ValueError("It must be either scales is None or len(scales) == {}, but len(scales) == {}.".format(n_predictor_layers+1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers+1) if len(variances) != 4: raise ValueError("4 variance values must be pased, but {} values were received.".format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError("All variances must be >0, but the variances given are {}".format(variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError("You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError("You must provide at least one offset value per predictor layer.") if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]]], axis=-1) elif len(swap_channels) == 4: return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]], tensor[...,swap_channels[3]]], axis=-1) # Build the model x = Input(shape=(img_height, img_width, img_channels)) x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) base_model = ResNet50(input_shape = (256, 256, 3), include_top = False, weights = 'imagenet', input_tensor = x1, pooling = 'max') L128 = base_model.get_layer(name = 'activation_1').output # 128x128x64 L64 = base_model.get_layer(name = 'activation_10').output # 64x64x256 L32 = base_model.get_layer(name = 'activation_22').output # 32x32x512 L16 = base_model.get_layer(name = 'activation_40').output # 16x16x1024 L8 = base_model.get_layer(name = 'activation_49').output # 8x8x2048 L4_1 = MaxPooling2D(pool_size=(2,2))(L8) L4_2 = Conv2D(1024, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L4_1) L4_3 = BatchNormalization()(L4_2) L4_4 = Activation('relu')(L4_3) L4_5 = Conv2D(1024, (3,3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L4_4) L4_6 = BatchNormalization()(L4_5) L4_7 = Activation('relu')(L4_6) ### Feature pyramid FPN4_cv = Conv2D(128, (1,1), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L4_7) FPN4_p = Conv2D(256, (3,3), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(FPN4_cv) FPN8_up = UpSampling2D()(FPN4_cv) FPN8_cv = Conv2D(128, (1,1), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L8) FPN8_ad = Add()([FPN8_up, FPN8_cv]) FPN8_p = Conv2D(256, (3,3), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(FPN8_ad) FPN16_up = UpSampling2D()(FPN8_ad) FPN16_cv = Conv2D(128, (1,1), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L16) FPN16_ad = Add()([FPN16_up, FPN16_cv]) FPN16_p = Conv2D(256, (3,3), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(FPN16_ad) FPN32_up = UpSampling2D()(FPN16_ad) FPN32_cv = Conv2D(128, (1,1), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L32) FPN32_ad = Add()([FPN32_up, FPN32_cv]) FPN32_p = Conv2D(256, (3,3), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(FPN32_ad) FPN64_up = UpSampling2D()(FPN32_ad) FPN64_cv = Conv2D(128, (1,1), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L64) FPN64_ad = Add()([FPN64_up, FPN64_cv]) FPN64_p = Conv2D(256, (3,3), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(FPN64_ad) FPN128_up = UpSampling2D()(FPN64_ad) FPN128_cv = Conv2D(128, (1,1), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(L128) FPN128_ad = Add()([FPN128_up, FPN128_cv]) FPN128_p = Conv2D(256, (3,3), activation='relu', padding ='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(FPN128_ad) # Prediction layers: confidence, location, and anchor boxes L128_mbox_conf = Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L128_mbox_conf')(FPN128_p) L64_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L64_mbox_conf')(FPN64_p) L32_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L32_mbox_conf')(FPN32_p) L16_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L16_mbox_conf')(FPN16_p) L8_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L8_mbox_conf')(FPN8_p) L4_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L4_mbox_conf')(FPN4_p) L128_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L128_mbox_loc')(FPN128_p) L64_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L64_mbox_loc')(FPN64_p) L32_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L32_mbox_loc')(FPN32_p) L16_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L16_mbox_loc')(FPN16_p) L8_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L8_mbox_loc')(FPN8_p) L4_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='L4_mbox_loc')(FPN4_p) L128_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L128_mbox_priorbox')(L128_mbox_loc) L64_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L64_mbox_priorbox')(L64_mbox_loc) L32_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L32_mbox_priorbox')(L32_mbox_loc) L16_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L16_mbox_priorbox')(L16_mbox_loc) L8_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L8_mbox_priorbox')(L8_mbox_loc) L4_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='L4_mbox_priorbox')(L4_mbox_loc) # Reshape and concatenate L128_mbox_conf_reshape = Reshape((-1, n_classes), name='L128_mbox_conf_reshape')(L128_mbox_conf) L64_mbox_conf_reshape = Reshape((-1, n_classes), name='L64_mbox_conf_reshape')(L64_mbox_conf) L32_mbox_conf_reshape = Reshape((-1, n_classes), name='L32_mbox_conf_reshape')(L32_mbox_conf) L16_mbox_conf_reshape = Reshape((-1, n_classes), name='L16_mbox_conf_reshape')(L16_mbox_conf) L8_mbox_conf_reshape = Reshape((-1, n_classes), name='L8_mbox_conf_reshape')(L8_mbox_conf) L4_mbox_conf_reshape = Reshape((-1, n_classes), name='L4_mbox_conf_reshape')(L4_mbox_conf) L128_mbox_loc_reshape = Reshape((-1, 4), name='L128_mbox_loc_reshape')(L128_mbox_loc) L64_mbox_loc_reshape = Reshape((-1, 4), name='L64_mbox_loc_reshape')(L64_mbox_loc) L32_mbox_loc_reshape = Reshape((-1, 4), name='L32_mbox_loc_reshape')(L32_mbox_loc) L16_mbox_loc_reshape = Reshape((-1, 4), name='L16_mbox_loc_reshape')(L16_mbox_loc) L8_mbox_loc_reshape = Reshape((-1, 4), name='L8_mbox_loc_reshape')(L8_mbox_loc) L4_mbox_loc_reshape = Reshape((-1, 4), name='L4_mbox_loc_reshape')(L4_mbox_loc) L128_mbox_priorbox_reshape = Reshape((-1, 8), name='L128_mbox_priorbox_reshape')(L128_mbox_priorbox) L64_mbox_priorbox_reshape = Reshape((-1, 8), name='L64_mbox_priorbox_reshape')(L64_mbox_priorbox) L32_mbox_priorbox_reshape = Reshape((-1, 8), name='L32_mbox_priorbox_reshape')(L32_mbox_priorbox) L16_mbox_priorbox_reshape = Reshape((-1, 8), name='L16_mbox_priorbox_reshape')(L16_mbox_priorbox) L8_mbox_priorbox_reshape = Reshape((-1, 8), name='L8_mbox_priorbox_reshape')(L8_mbox_priorbox) L4_mbox_priorbox_reshape = Reshape((-1, 8), name='L4_mbox_priorbox_reshape')(L4_mbox_priorbox) mbox_conf = Concatenate(axis=1, name='mbox_conf')([L128_mbox_conf_reshape, L64_mbox_conf_reshape, L32_mbox_conf_reshape, L16_mbox_conf_reshape, L8_mbox_conf_reshape, L4_mbox_conf_reshape]) mbox_loc = Concatenate(axis=1, name='mbox_loc')([L128_mbox_loc_reshape, L64_mbox_loc_reshape, L32_mbox_loc_reshape, L16_mbox_loc_reshape, L8_mbox_loc_reshape, L4_mbox_loc_reshape]) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([L128_mbox_priorbox_reshape, L64_mbox_priorbox_reshape, L32_mbox_priorbox_reshape, L16_mbox_priorbox_reshape, L8_mbox_priorbox_reshape, L4_mbox_priorbox_reshape]) # Softmax loss, concatenation and model return mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) predictions = Concatenate(axis=2, name='predictions')([mbox_conf_softmax, mbox_loc, mbox_priorbox]) model = Model(inputs=x, outputs=predictions) if return_predictor_sizes: predictor_sizes = np.array([L128_mbox_conf._keras_shape[1:3], L64_mbox_conf._keras_shape[1:3], L32_mbox_conf._keras_shape[1:3], L16_mbox_conf._keras_shape[1:3], L8_mbox_conf._keras_shape[1:3], L4_mbox_conf._keras_shape[1:3]]) return model, predictor_sizes else: return model
def ssd_300(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]], two_boxes_for_ar1=True, steps=[8, 16, 32, 64, 100, 300], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): ''' 使用Keras构建SSD300模型,是VGG-16(a reduced atrous VGG-16)上进行改进 如果你正在训练网络,这里传递的参数必须与用于设置`SSDBoxEncoder`的参数相同。 如果您正在加载训练权重,则此处传递的参数必须与用于生成训练权重时的参数相同。 一些参数之后会在`SSDBoxEncoder` 类的文档中详细介绍。 提示: 需要Keras v2.0 或更新版本. 目前仅适用于TensorFlow后端(v1.0 或更新)。 Arguments: image_size (tuple): 输入图像的尺寸`(height, width, channels)`. n_classes (int): 目标类别数量(不包含背景) e.g. 20 for Pascal VOC, 80 for MS COCO. mode (str, optional): (1)training 模型输出原始预测张量 (2)inference (3)inference_fast 将原始预测解码为绝对坐标,并通过置信度阈值,非最大抑制和top-k过滤。 后两种模式之间的区别在于“inference”严格遵循原始Caffe实现过程,而'inference_fast'使用更快的预测解码过程。 l2_regularization (float, optional): L2正则化率,适用于所有卷积层,设置为零以停用L2正则化。 min_scale (float, optional), max_scale (float, optional), scales (list, optional) aspect_ratios_per_layer (list, optional): 每层比例. two_boxes_for_ar1 (bool, optional): 比例为1时是否生成两种scale的anchor steps (list, optional): 预测层每两个像素之间的距离为原图多少距离,即下采样倍数 offsets (list, optional): 分数 clip_boxes (bool, optional): 如果为`True`, 裁剪超出图像边界的default box variances (list, optional): box每个偏移值坐标会除以对应的variances coords (str, optional): centroids(cx, cy, w, h), minmax(xmin, xmax, ymin, ymax), corners(xmin, ymin, xmax, ymax). normalize_coords (bool, optional): 坐标标准化,即模型预测的box坐标在[0,1]之间(相对坐标)而不是绝对坐标。 subtract_mean (array-like, optional): 减均值 divide_by_stddev (array-like, optional): 除以标准差 swap_channels (list, optional): 'False'或整数列表,表示应交换的输入图像通道的所需顺序。 confidence_thresh (float, optional): [0,1)之间的一个浮点数, nms iou_threshold (float, optional):[0,1]之间的浮点数。与局部置信度分数最大的框之间的Jaccard相似性 大于“iou_threshold”的框,将从给定类的预测集中移除。 top_k (int, optional): 每批次数据在非最大抑制阶段之后保留的最高评分预测的数量。 nms_max_output_size (int, optional): NMS阶段后剩余的最大预测数. return_predictor_sizes (bool, optional): 如果为“True”,则此函数不仅返回模型,还返回包含用来预测的特征图的空间维度的列表。 不是必要的,因为可以通过Keras API轻松获得它们的大小,但是通过这种方式获取它们很方便且不易出错。 它们仅与训练相关(SSDBoxEncoder需要知道预测层的空间维度),inference时不需要。 Returns: model: SSD300 网络. predictor_sizes (optional): Numpy数组,包含每个预测特征图“(高度,宽度)”。 在训练期间,数据生成函数需要这样,以便将ground truth标签转换成与模型的输出张量相同结构的张量,参与损失函数计算 References: https://arxiv.org/abs/1512.02325v5 ''' n_predictor_layers = 6 # 原始SSD300中用来预测的层是6. n_classes += 1 # 考虑到背景类. l2_reg = l2_regularization # 让名字短一点. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # 一些异常. ############################################################################ if aspect_ratios_per_layer is None: raise ValueError("`aspect_ratios_per_layer` cannot both be None.") if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # 没有具体的scaling factors,那就通过`min_scale`和`max_scale`计算 scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # 计算default box的参数 ############################################################################ # 设置每一个预测层上box的比例,只有priorbox层需要这个参数。 if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer # 计算每个预测层的每个单元预测的box数量。即预测层需要有多少个通道。 if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 宽高比1的第二个框 else: n_boxes.append(len(ar)) if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # 为下面的Lambda层定义函数。 ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # 建立网络结构. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_1')(x1) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_2')(conv1_1) pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1')(conv1_2) conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_1')(pool1) conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_2')(conv2_1) pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2')(conv2_2) conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_1')(pool2) conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_2')(conv3_1) conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_3')(conv3_2) pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3')(conv3_3) conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_1')(pool3) conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_2')(conv4_1) conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3')(conv4_2) pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4')(conv4_3) conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_1')(pool4) conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_2')(conv5_1) conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_3')(conv5_2) pool5 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same', name='pool5')(conv5_3) fc6 = Conv2D(1024, (3, 3), dilation_rate=(6, 6), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc6')(pool5) fc7 = Conv2D(1024, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7')(fc6) conv6_1 = Conv2D(256, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_1')(fc7) conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv6_padding')(conv6_1) conv6_2 = Conv2D(512, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2')(conv6_1) conv7_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_1')(conv6_2) conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv7_padding')(conv7_1) conv7_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2')(conv7_1) conv8_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_1')(conv7_2) conv8_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2')(conv8_1) conv9_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_1')(conv8_2) conv9_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2')(conv9_1) # 将conv4_3传入L2 normalization层处理 conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3) ###在基础网络上建立卷积预测层 # 每个box预测`n_classes` 类别, n_boxes * n_classes # 预测类别: `(batch, height, width, n_boxes * n_classes)` conv4_3_norm_mbox_conf = Conv2D( n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_conf')(conv4_3_norm) fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_conf')(fc7) conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf')(conv6_2) conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf')(conv7_2) conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf')(conv8_2) conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf')(conv9_2) # 每个box预测4个坐标, n_boxes * 4 # 预测位置: `(batch, height, width, n_boxes * 4)` conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_loc')(conv4_3_norm) fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_loc')(fc7) conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc')(conv6_2) conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc')(conv7_2) conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(conv8_2) conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(conv9_2) ### 生成priorbox(default boxes) # priorbox: `(batch, height, width, n_boxes, 8)` conv4_3_norm_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc) fc7_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='fc7_mbox_priorbox')(fc7_mbox_loc) conv6_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) ### Reshape # Reshape类别预测, 得到三维张量`(batch, height * width * n_boxes, n_classes)` # 希望把类别分数独立到最后一维,进行softmax处理 conv4_3_norm_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape( (-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) # Reshape偏移值, 得到3维张量 `(batch, height * width * n_boxes, 4)` # 我们希望把四个坐标值拿出来计算损失 conv4_3_norm_mbox_loc_reshape = Reshape( (-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape((-1, 4), name='fc7_mbox_loc_reshape')(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) # Reshape默认框, 得到3维张量 `(batch, height * width * n_boxes, 8)` conv4_3_norm_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape( (-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox) ### 拼接不同层的预测结果 # 第0维 (batch)和第 2 维(类别数或者4)所有预测层都是相同的, # 选择沿第1维拼接,即每一层的box数量 # 类别 box_conf: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([ conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape ]) #偏移值 mbox_loc: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([ conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape ]) #默认框 mbox_priorbox: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([ conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape ]) # 这些预测值接下来就会输入损失函数参与训练 # 但是对于类别预测,我们先要经过一个softmax激活层 mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # 拼接类别,偏移值预测,以及默认框,预测: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ conv4_3_norm_mbox_conf._keras_shape[1:3], fc7_mbox_conf._keras_shape[1:3], conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def ssd_512(image_size, n_classes, mode='training', l2_regularization=0.0005, min_scale=None, max_scale=None, scales=None, aspect_ratios_global=None, aspect_ratios_per_layer=[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]], two_boxes_for_ar1=True, steps=[4, 8, 16, 32, 64, 128], offsets=None, clip_boxes=False, variances=[0.1, 0.1, 0.2, 0.2], coords='centroids', normalize_coords=True, subtract_mean=[123, 117, 104], divide_by_stddev=None, swap_channels=[2, 1, 0], confidence_thresh=0.01, iou_threshold=0.45, top_k=200, nms_max_output_size=400, return_predictor_sizes=False): ''' Build a Keras model with SSD512 architecture, see references. The base network is a reduced atrous VGG-16, extended by the SSD architecture, as described in the paper. Most of the arguments that this function takes are only needed for the anchor box layers. In case you're training the network, the parameters passed here must be the same as the ones used to set up `SSDBoxEncoder`. In case you're loading trained weights, the parameters passed here must be the same as the ones used to produce the trained weights. Some of these arguments are explained in more detail in the documentation of the `SSDBoxEncoder` class. Note: Requires Keras v2.0 or later. Currently works only with the TensorFlow backend (v1.0 or later). Arguments: image_size (tuple): The input image size in the format `(height, width, channels)`. n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO. mode (str, optional): One of 'training', 'inference' and 'inference_fast'. In 'training' mode, the model outputs the raw prediction tensor, while in 'inference' and 'inference_fast' modes, the raw predictions are decoded into absolute coordinates and filtered via confidence thresholding, non-maximum suppression, and top-k filtering. The difference between latter two modes is that 'inference' follows the exact procedure of the original Caffe implementation, while 'inference_fast' uses a faster prediction decoding procedure. l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers. Set to zero to deactivate L2-regularization. min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction of the shorter side of the input images. All scaling factors between the smallest and the largest will be linearly interpolated. Note that the second to last of the linearly interpolated scaling factors will actually be the scaling factor for the last predictor layer, while the last scaling factor is used for the second box for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer. This list must be one element longer than the number of predictor layers. The first `k` elements are the scaling factors for the `k` predictor layers, while the last element is used for the second box for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional last scaling factor must be passed either way, even if it is not being used. If a list is passed, this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero. aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be generated. This list is valid for all prediction layers. aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each prediction layer. This allows you to set the aspect ratios for each predictor layer individually, which is the case for the original SSD512 implementation. If a list is passed, it overrides `aspect_ratios_global`. two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise. If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated using the scaling factor for the respective layer, the second one will be generated using geometric mean of said scaling factor and next bigger scaling factor. steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over the image. If the list contains ints/floats, then that value will be used for both spatial dimensions. If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`. If no steps are provided, then they will be computed such that the anchor box center points will form an equidistant grid within the image dimensions. offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be either floats or tuples of two floats. These numbers represent for each predictor layer how many pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions of the step size specified in the `steps` argument. If the list contains floats, then that value will be used for both spatial dimensions. If the list contains tuples of two floats, then they represent `(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size. clip_boxes (bool, optional): If `True`, clips the anchor box coordinates to stay within image boundaries. variances (list, optional): A list of 4 floats >0. The anchor box offset for each coordinate will be divided by its respective variance value. coords (str, optional): The box coordinate format to be used internally by the model (i.e. this is not the input format of the ground truth labels). Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width, and height), 'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`. normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates, i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates. subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values of any shape that is broadcast-compatible with the image shape. The elements of this array will be subtracted from the image pixel intensity values. For example, pass a list of three integers to perform per-channel mean normalization for color images. divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or floating point values of any shape that is broadcast-compatible with the image shape. The image pixel intensity values will be divided by the elements of this array. For example, pass a list of three integers to perform per-channel standard deviation normalization for color images. swap_channels (list, optional): Either `False` or a list of integers representing the desired order in which the input image channels should be swapped. confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific positive class in order to be considered for the non-maximum suppression stage for the respective class. A lower value will result in a larger part of the selection process being done by the non-maximum suppression stage, while a larger value will result in a larger part of the selection process happening in the confidence thresholding stage. iou_threshold (float, optional): A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold` with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers to the box's confidence score. top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the non-maximum suppression stage. nms_max_output_size (int, optional): The maximal number of predictions that will be left over after the NMS stage. return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since you can always get their sizes easily via the Keras API, but it's convenient and less error-prone to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the spatial dimensions of the predictor layers), for inference you don't need them. Returns: model: The Keras SSD512 model. predictor_sizes (optional): A Numpy array containing the `(height, width)` portion of the output tensor shape for each convolutional predictor layer. During training, the generator function needs this in order to transform the ground truth labels into tensors of identical structure as the output tensors of the model, which is in turn needed for the cost function. References: https://arxiv.org/abs/1512.02325v5 ''' n_predictor_layers = 7 # The number of predictor conv layers in the network is 7 for the original SSD512 n_classes += 1 # Account for the background class. l2_reg = l2_regularization # Make the internal name shorter. img_height, img_width, img_channels = image_size[0], image_size[ 1], image_size[2] ############################################################################ # Get a few exceptions out of the way. ############################################################################ if aspect_ratios_global is None and aspect_ratios_per_layer is None: raise ValueError( "`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified." ) if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError( "It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}." .format(n_predictor_layers, len(aspect_ratios_per_layer))) if (min_scale is None or max_scale is None) and scales is None: raise ValueError( "Either `min_scale` and `max_scale` or `scales` need to be specified." ) if scales: if len(scales) != n_predictor_layers + 1: raise ValueError( "It must be either scales is None or len(scales) == {}, but len(scales) == {}." .format(n_predictor_layers + 1, len(scales))) else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale` scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if len(variances) != 4: raise ValueError( "4 variance values must be pased, but {} values were received.". format(len(variances))) variances = np.array(variances) if np.any(variances <= 0): raise ValueError( "All variances must be >0, but the variances given are {}".format( variances)) if (not (steps is None)) and (len(steps) != n_predictor_layers): raise ValueError( "You must provide at least one step value per predictor layer.") if (not (offsets is None)) and (len(offsets) != n_predictor_layers): raise ValueError( "You must provide at least one offset value per predictor layer.") ############################################################################ # Compute the anchor box parameters. ############################################################################ # Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers. if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer else: aspect_ratios = [aspect_ratios_global] * n_predictor_layers # Compute the number of boxes to be predicted per cell for each predictor layer. # We need this so that we know how many channels the predictor layers need to have. if aspect_ratios_per_layer: n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1 else: n_boxes.append(len(ar)) else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers if steps is None: steps = [None] * n_predictor_layers if offsets is None: offsets = [None] * n_predictor_layers ############################################################################ # Define functions for the Lambda layers below. ############################################################################ def identity_layer(tensor): return tensor def input_mean_normalization(tensor): return tensor - np.array(subtract_mean) def input_stddev_normalization(tensor): return tensor / np.array(divide_by_stddev) def input_channel_swap(tensor): if len(swap_channels) == 3: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]] ], axis=-1) elif len(swap_channels) == 4: return K.stack([ tensor[..., swap_channels[0]], tensor[..., swap_channels[1]], tensor[..., swap_channels[2]], tensor[..., swap_channels[3]] ], axis=-1) ############################################################################ # Build the network. ############################################################################ x = Input(shape=(img_height, img_width, img_channels)) # The following identity layer is only needed so that the subsequent lambda layers can be optional. x1 = Lambda(identity_layer, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x) if not (subtract_mean is None): x1 = Lambda(input_mean_normalization, output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1) if not (divide_by_stddev is None): x1 = Lambda(input_stddev_normalization, output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1) if swap_channels: x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1) conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_1')(x1) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1_2')(conv1_1) pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool1')(conv1_2) conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_1')(pool1) conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2_2')(conv2_1) pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool2')(conv2_2) conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_1')(pool2) conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_2')(conv3_1) conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3_3')(conv3_2) pool3 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool3')(conv3_3) conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_1')(pool3) conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_2')(conv4_1) conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3')(conv4_2) pool4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same', name='pool4')(conv4_3) conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_1')(pool4) conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_2')(conv5_1) conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5_3')(conv5_2) pool5 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same', name='pool5')(conv5_3) fc6 = Conv2D(1024, (3, 3), dilation_rate=(6, 6), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc6')(pool5) fc7 = Conv2D(1024, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7')(fc6) conv6_1 = Conv2D(256, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_1')(fc7) conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv6_padding')(conv6_1) conv6_2 = Conv2D(512, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2')(conv6_1) conv7_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_1')(conv6_2) conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv7_padding')(conv7_1) conv7_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2')(conv7_1) conv8_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_1')(conv7_2) conv8_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv8_padding')(conv8_1) conv8_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2')(conv8_1) conv9_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_1')(conv8_2) conv9_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv9_padding')(conv9_1) conv9_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2')(conv9_1) conv10_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv10_1')(conv9_2) conv10_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv10_padding')(conv10_1) conv10_2 = Conv2D(256, (4, 4), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv10_2')(conv10_1) # Feed conv4_3 into the L2 normalization layer conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3) ### Build the convolutional predictor layers on top of the base network # We precidt `n_classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * n_classes` # Output shape of the confidence layers: `(batch, height, width, n_boxes * n_classes)` conv4_3_norm_mbox_conf = Conv2D( n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_conf')(conv4_3_norm) fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_conf')(fc7) conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_conf')(conv6_2) conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_conf')(conv7_2) conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf')(conv8_2) conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf')(conv9_2) conv10_2_mbox_conf = Conv2D(n_boxes[6] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv10_2_mbox_conf')(conv10_2) # We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4` # Output shape of the localization layers: `(batch, height, width, n_boxes * 4)` conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4_3_norm_mbox_loc')(conv4_3_norm) fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='fc7_mbox_loc')(fc7) conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6_2_mbox_loc')(conv6_2) conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_2_mbox_loc')(conv7_2) conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(conv8_2) conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(conv9_2) conv10_2_mbox_loc = Conv2D(n_boxes[6] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv10_2_mbox_loc')(conv10_2) ### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names) # Output shape of anchors: `(batch, height, width, n_boxes, 8)` conv4_3_norm_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc) fc7_mbox_priorbox = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='fc7_mbox_priorbox')(fc7_mbox_loc) conv6_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc) conv7_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc) conv8_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc) conv9_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc) conv10_2_mbox_priorbox = AnchorBoxes( img_height, img_width, this_scale=scales[6], next_scale=scales[7], aspect_ratios=aspect_ratios[6], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[6], this_offsets=offsets[6], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv10_2_mbox_priorbox')(conv10_2_mbox_loc) ### Reshape # Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them conv4_3_norm_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf) fc7_mbox_conf_reshape = Reshape( (-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf) conv6_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf) conv7_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf) conv8_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) conv10_2_mbox_conf_reshape = Reshape( (-1, n_classes), name='conv10_2_mbox_conf_reshape')(conv10_2_mbox_conf) # Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)` # We want the four box coordinates isolated in the last axis to compute the smooth L1 loss conv4_3_norm_mbox_loc_reshape = Reshape( (-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc) fc7_mbox_loc_reshape = Reshape((-1, 4), name='fc7_mbox_loc_reshape')(fc7_mbox_loc) conv6_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc) conv7_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc) conv8_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) conv10_2_mbox_loc_reshape = Reshape( (-1, 4), name='conv10_2_mbox_loc_reshape')(conv10_2_mbox_loc) # Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)` conv4_3_norm_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox) fc7_mbox_priorbox_reshape = Reshape( (-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox) conv6_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox) conv7_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox) conv8_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox) conv9_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox) conv10_2_mbox_priorbox_reshape = Reshape( (-1, 8), name='conv10_2_mbox_priorbox_reshape')(conv10_2_mbox_priorbox) ### Concatenate the predictions from the different layers # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions, # so we want to concatenate along axis 1, the number of boxes per layer # Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes) mbox_conf = Concatenate(axis=1, name='mbox_conf')([ conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape, conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape, conv10_2_mbox_conf_reshape ]) # Output shape of `mbox_loc`: (batch, n_boxes_total, 4) mbox_loc = Concatenate(axis=1, name='mbox_loc')([ conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape, conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape, conv10_2_mbox_loc_reshape ]) # Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8) mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([ conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape, conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape, conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape, conv10_2_mbox_priorbox_reshape ]) # The box coordinate predictions will go into the loss function just the way they are, # but for the class predictions, we'll apply a softmax activation layer first mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate the class and box predictions and the anchors to one large predictions vector # Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8) predictions = Concatenate(axis=2, name='predictions')( [mbox_conf_softmax, mbox_loc, mbox_priorbox]) if mode == 'training': model = Model(inputs=x, outputs=predictions) elif mode == 'inference': decoded_predictions = DecodeDetections( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) elif mode == 'inference_fast': decoded_predictions = DecodeDetectionsFast( confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=x, outputs=decoded_predictions) else: raise ValueError( "`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'." .format(mode)) if return_predictor_sizes: predictor_sizes = np.array([ conv4_3_norm_mbox_conf._keras_shape[1:3], fc7_mbox_conf._keras_shape[1:3], conv6_2_mbox_conf._keras_shape[1:3], conv7_2_mbox_conf._keras_shape[1:3], conv8_2_mbox_conf._keras_shape[1:3], conv9_2_mbox_conf._keras_shape[1:3], conv10_2_mbox_conf._keras_shape[1:3] ]) return model, predictor_sizes else: return model
def ssd300(n_classes, backbone, mode='training', use_bb_layer_pred=True, input_shape=(300, 300, 3), return_predictor_sizes=False, l2_regularization=5e-4, min_scale=.2, max_scale=.9, scales=None, aspect_ratios_per_layer=None, aspect_ratios_global=None, two_boxes_for_ar1=True, steps=None, offsets=None, clip_boxes=False, variances=None, coords='centroids', normalize_coords=True, subtract_mean=None, divide_by_std=None, swap_channels=None, confidence_thresh=.01, iou_threshold=.45, top_k=200, nms_max_output_size=400): n_predictor_layers = 6 # Number of predictors conv n_classes += 1 # Anchor for the background class l2_reg = l2_regularization # Make the internal name shorter img_height, img_width, img_channels = input_shape ################################## # Set exceptions or default values ################################## if aspect_ratios_per_layer is None and aspect_ratios_global is None: aspect_ratios_per_layer = [[1., 2., .5], [1., 2., .5, 3., 1. / 3.], [1., 2., .5, 3., 1. / 3.], [1., 2., .5, 3., 1. / 3.], [1., 2., .5], [1., 2., .5]] if aspect_ratios_per_layer: if len(aspect_ratios_per_layer) != n_predictor_layers: raise ValueError(f'It must be either aspect_ratios is None or ' f'len(aspect_ratios_per_layer) == ' f'{n_predictor_layers}, but len(aspect_ratios_per' f'_layer) == {len(aspect_ratios_per_layer)}.') if scales: if len(scales) != n_predictor_layers + 1: raise ValueError(f'It must be either scales is None or len(scales)' f' == {n_predictor_layers + 1}, but len(scales) ' f'== {len(scales)}.') scales = np.array(scales) else: scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1) if variances is None: variances = [.1, .1, .2, .2] else: if len(variances) != 4: raise ValueError(f'4 variance values must be pased, ' f'but {len(variances)} values were received.') variances = np.array(variances) if np.any(variances <= 0): raise ValueError(f'All variances must be >0, ' f'but the variances given are {variances}') if steps is None: steps = [8, 16, 32, 64, 100, 300] else: if len(steps) != n_predictor_layers: raise ValueError('You must provide 4 positive float.') if offsets is None: offsets = [.5] * n_predictor_layers else: if len(offsets) != n_predictor_layers: raise ValueError('You must provide at least ' 'one offset value per predictor layer.') if aspect_ratios_per_layer: aspect_ratios = aspect_ratios_per_layer n_boxes = [] for ar in aspect_ratios_per_layer: if (1 in ar) & two_boxes_for_ar1: n_boxes.append(len(ar) + 1) else: n_boxes.append(len(ar)) else: aspect_ratios = aspect_ratios_global if (1 in aspect_ratios_global) & two_boxes_for_ar1: n_boxes = len(aspect_ratios_global) + 1 else: n_boxes = len(aspect_ratios_global) n_boxes = [n_boxes] * n_predictor_layers ################# # Build the model ################# base_model = load_backbone(name=backbone, input_shape=input_shape, l2_regularization=l2_reg, subtract_mean=subtract_mean, divide_by_std=divide_by_std, swap_channels=swap_channels) base_output = base_model.get_layer(index=-1).output if backbone == 'vgg16': bb_pred = base_model.get_layer(name='block4_conv3').output elif backbone == 'vgg19': bb_pred = base_model.get_layer(name='block4_conv4').output elif backbone == 'resnet50': bb_pred = base_model.get_layer(name='activation_22').output elif backbone == 'inception_v3': bb_pred = base_model.get_layer(name='mixed2').output # Block full Conv 6 and 7 conv6 = Conv2D(1024, (3, 3), dilation_rate=(6, 6), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6')(base_output) conv7 = Conv2D(1024, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7')(conv6) # Block 8 conv8_1 = Conv2D(256, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_1')(conv7) conv8_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv8_padding')(conv8_1) conv8_2 = Conv2D(512, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2')(conv8_1) # Block 9 conv9_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_1')(conv8_2) conv9_1 = ZeroPadding2D(padding=((1, 1), (1, 1)), name='conv9_padding')(conv9_1) conv9_2 = Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2')(conv9_1) # Block 10 conv10_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv10_1')(conv9_2) conv10_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv10_2')(conv10_1) # Block 11 conv11_1 = Conv2D(128, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv11_1')(conv10_2) conv11_2 = Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv11_2')(conv11_1) # Feed bb_pred into L2Normalization layer if use_bb_layer_pred: bb_pred_norm = L2Normalization(gamma_init=20, name='bb_pred_norm')(bb_pred) # Build convolutional predictor layers on top of the base network # Output shape of confidence: `(batch, height, width, n_boxes * n_classes)` if use_bb_layer_pred: bb_pred_norm_mbox_conf = \ Conv2D(n_boxes[0] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='bb_pred_norm_mbox_conf')(bb_pred_norm) conv7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_mbox_conf')(conv7) conv8_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_conf')(conv8_2) conv9_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_conf')(conv9_2) conv10_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv10_2_mbox_conf')(conv10_2) conv11_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv11_2_mbox_conf')(conv11_2) # Predict 4 box coordinates for each box # Output shape of localization: `(batch, height, width, n_boxes * 4)` if use_bb_layer_pred: bb_pred_norm_mbox_loc = Conv2D( n_boxes[0] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='bb_pred_norm_mbox_loc')(bb_pred_norm) conv7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7_mbox_loc')(conv7) conv8_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv8_2_mbox_loc')(conv8_2) conv9_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv9_2_mbox_loc')(conv9_2) conv10_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv10_2_mbox_loc')(conv10_2) conv11_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv11_2_mbox_loc')(conv11_2) # Generate anchor boxes # Output shape of anchors: `(batch, height, width, n_boxes, 8)` if use_bb_layer_pred: bb_pred_norm_mbox_anchor = \ AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='bb_pred_norm_mbox_anchor')(bb_pred_norm_mbox_loc) conv7_mbox_anchor = \ AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv7_mbox_anchor')(conv7_mbox_loc) conv8_2_mbox_anchor = \ AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv8_2_mbox_anchor')(conv8_2_mbox_loc) conv9_2_mbox_anchor = \ AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv9_2_mbox_anchor')(conv9_2_mbox_loc) conv10_2_mbox_anchor = \ AnchorBoxes(img_height, img_width, this_scale=scales[4], next_scale=scales[5], aspect_ratios=aspect_ratios[4], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[4], this_offsets=offsets[4], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv10_2_mbox_anchor')(conv10_2_mbox_loc) conv11_2_mbox_anchor = \ AnchorBoxes(img_height, img_width, this_scale=scales[5], next_scale=scales[6], aspect_ratios=aspect_ratios[5], two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[5], this_offsets=offsets[5], clip_boxes=clip_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='conv11_2_mbox_anchor')(conv11_2_mbox_loc) # Reshape ######### # Reshape class predictions, yielding 3D tensor of shape # `(batch, height * width * n_boxes, n_classes)` # We want the classes isolated in the last axis to perform softmax on them if use_bb_layer_pred: bb_pred_norm_mbox_conf_reshape = \ Reshape((-1, n_classes), name='bb_pred_norm_mbox_conf_reshape')(bb_pred_norm_mbox_conf) conv7_mbox_conf_reshape = \ Reshape((-1, n_classes), name='conv7_mbox_conf_reshape')(conv7_mbox_conf) conv8_2_mbox_conf_reshape = \ Reshape((-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf) conv9_2_mbox_conf_reshape = \ Reshape((-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf) conv10_2_mbox_conf_reshape = \ Reshape((-1, n_classes), name='conv10_2_mbox_conf_reshape')(conv10_2_mbox_conf) conv11_2_mbox_conf_reshape = \ Reshape((-1, n_classes), name='conv11_2_mbox_conf_reshape')(conv11_2_mbox_conf) # Reshape box predictions, yielding 3D tensor of shape # `(batch, height * width * n_boxes, 4)` # We want 4 boxes coordinates isolated in the last axis # to compute the smooth L1 loss if use_bb_layer_pred: bb_pred_norm_mbox_loc_reshape = \ Reshape((-1, 4), name='bb_pred_norm_mbox_loc_reshape')(bb_pred_norm_mbox_loc) conv7_mbox_loc_reshape = \ Reshape((-1, 4), name='conv7_mbox_loc_reshape')(conv7_mbox_loc) conv8_2_mbox_loc_reshape = \ Reshape((-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc) conv9_2_mbox_loc_reshape = \ Reshape((-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc) conv10_2_mbox_loc_reshape = \ Reshape((-1, 4), name='conv10_2_mbox_loc_reshape')(conv10_2_mbox_loc) conv11_2_mbox_loc_reshape = \ Reshape((-1, 4), name='conv11_2_mbox_loc_reshape')(conv11_2_mbox_loc) # Reshape anchor box tensors, yielding 3D tensor of shape # `(batch, height * width * n_boxes, 8)` if use_bb_layer_pred: bb_pred_norm_mbox_anchor_reshape = Reshape( (-1, 8), name='bb_pred_norm_mbox_anchor_reshape')(bb_pred_norm_mbox_anchor) conv7_mbox_anchor_reshape = \ Reshape((-1, 8), name='conv7_mbox_anchor_reshape')(conv7_mbox_anchor) conv8_2_mbox_anchor_reshape = \ Reshape((-1, 8), name='conv8_2_mbox_anchor_reshape')(conv8_2_mbox_anchor) conv9_2_mbox_anchor_reshape = \ Reshape((-1, 8), name='conv9_2_mbox_anchor_reshape')(conv9_2_mbox_anchor) conv10_2_mbox_anchor_reshape = \ Reshape((-1, 8), name='conv10_2_mbox_anchor_reshape')(conv10_2_mbox_anchor) conv11_2_mbox_anchor_reshape = \ Reshape((-1, 8), name='conv11_2_mbox_anchor_reshape')(conv11_2_mbox_anchor) # Concatenate ############# # Concatenate predictions from different layers # Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical # for all layers predictions, so we want to concatenate along axis 1 # (the number of boxes per layer) if use_bb_layer_pred: # Output shape for `mbox_conf`: `(batch, n_boxes_total, n_classes)` mbox_conf = Concatenate(axis=1, name='mbox_conf')([ bb_pred_norm_mbox_conf_reshape, conv7_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape, conv10_2_mbox_conf_reshape, conv11_2_mbox_conf_reshape ]) # Output shape for `mbox_loc`: `(batch, n_boxes_total, 4)` mbox_loc = Concatenate(axis=1, name='mbox_loc')([ bb_pred_norm_mbox_loc_reshape, conv7_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape, conv10_2_mbox_loc_reshape, conv11_2_mbox_loc_reshape ]) # Output shape for `mbox_anchor`: `(batch, n_boxes_total, 8)` mbox_anchor = Concatenate(axis=1, name='mbox_anchor')([ bb_pred_norm_mbox_anchor_reshape, conv7_mbox_anchor_reshape, conv8_2_mbox_anchor_reshape, conv9_2_mbox_anchor_reshape, conv10_2_mbox_anchor_reshape, conv11_2_mbox_anchor_reshape ]) else: # Output shape for `mbox_conf`: `(batch, n_boxes_total, n_classes)` mbox_conf = Concatenate(axis=1, name='mbox_conf')([ conv7_mbox_conf_reshape, conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape, conv10_2_mbox_conf_reshape, conv11_2_mbox_conf_reshape ]) # Output shape for `mbox_loc`: `(batch, n_boxes_total, 4)` mbox_loc = Concatenate(axis=1, name='mbox_loc')([ conv7_mbox_loc_reshape, conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape, conv10_2_mbox_loc_reshape, conv11_2_mbox_loc_reshape ]) # Output shape for `mbox_anchor`: `(batch, n_boxes_total, 8)` mbox_anchor = Concatenate(axis=1, name='mbox_anchor')([ conv7_mbox_anchor_reshape, conv8_2_mbox_anchor_reshape, conv9_2_mbox_anchor_reshape, conv10_2_mbox_anchor_reshape, conv11_2_mbox_anchor_reshape ]) # The box coordinates predictions will go into the loss function # just the way they are, but for the class prediction, # we'll apply a softmax activation layer first mbox_conf_softmax = Activation('softmax', name='mbox_conf_softmax')(mbox_conf) # Concatenate the class, box predictions and anchors to # one large predictions vector # Output shape of predictions: `(batch, n_boxes_total, n_classes + 4 + 8)` predictions = Concatenate( axis=2, name='predictions')([mbox_conf_softmax, mbox_loc, mbox_anchor]) if mode == 'training': model = Model(inputs=base_model.inputs, outputs=predictions) elif mode == 'inference': decoded_predictions = \ DecodeDetections(confidence_thresh=confidence_thresh, iou_threshold=iou_threshold, top_k=top_k, nms_max_output_size=nms_max_output_size, coords=coords, normalize_coords=normalize_coords, img_height=img_height, img_width=img_width, name='decoded_predictions')(predictions) model = Model(inputs=base_model.inputs, outputs=decoded_predictions) else: ValueError(f'Argument "mode" must be "training" or "inference", ' f'not {mode}') if return_predictor_sizes: if use_bb_layer_pred: predictor_sizes = np.array([ bb_pred_norm_mbox_conf.shape[1:3], conv7_mbox_conf.shape[1:3], conv8_2_mbox_conf.shape[1:3], conv9_2_mbox_conf.shape[1:3], conv10_2_mbox_conf.shape[1:3], conv11_2_mbox_conf.shape[1:3] ]) else: predictor_sizes = np.array([ conv7_mbox_conf.shape[1:3], conv8_2_mbox_conf.shape[1:3], conv9_2_mbox_conf.shape[1:3], conv10_2_mbox_conf.shape[1:3], conv11_2_mbox_conf.shape[1:3] ]) return model, predictor_sizes return model