示例#1
0
文件: hrnet.py 项目: Jnyle/models
def fusion_block(filters, branches_in, branches_out, activation='relu'):
    """A fusion block will fuse multi-resolution inputs.

    A typical fusion block looks like a square box with cells. For example at
    stage 3, the fusion block consists 12 cells. Each cell represents a fusion
    layer. Every cell whose row < column is a down sampling cell, whose row ==
    column is a identity cell, and the rest are up sampling cells.

             B1         B2         B3         B4
        |----------|----------|----------|----------|
    B1  | identity |    ->    |    ->    |    ->    |
        |----------|----------|----------|----------|
    B2  |    <-    | identity |    ->    |    ->    |
        |----------|----------|----------|----------|
    B3  |    <-    |    <-    | identity |    ->    |
        |----------|----------|----------|----------|
    """
    # Construct the fusion layers.
    _fusion_grid = []

    rows = branches_in
    columns = branches_out

    for row in range(rows):
        _fusion_layers = []
        for column in range(columns):
            if column == row:
                _fusion_layers.append(tf.identity)
            elif column > row:
                # Down sampling.
                _fusion_layers.append(
                    fusion_layer(filters * pow(2, column), False, activation))
            else:
                # Up sampling.
                _fusion_layers.append(
                    fusion_layer(filters * pow(2, column), True, activation))

        _fusion_grid.append(_fusion_layers)

    if len(_fusion_grid) > 1:
        _add_layers_group = [layers.Add() for _ in range(branches_out)]

    def forward(inputs):
        rows = len(_fusion_grid)
        columns = len(_fusion_grid[0])

        # Every cell in the fusion grid has an output value.
        fusion_values = [[None for _ in range(columns)] for _ in range(rows)]

        for row in range(rows):
            # The down sampling operation excutes from left to right.
            for column in range(columns):
                # The input will be different for different cells.
                if column < row:
                    # Skip all up samping cells.
                    continue
                elif column == row:
                    # The input is the branch output.
                    x = inputs[row]
                elif column > row:
                    # Down sampling, the input is the fusion value of the left cell.
                    x = fusion_values[row][column - 1]

                fusion_values[row][column] = _fusion_grid[row][column](x)

            # The upsampling operation excutes in the opposite direction.
            for column in reversed(range(columns)):
                if column >= row:
                    # Skip all down samping and identity cells.
                    continue

                x = fusion_values[row][column + 1]
                fusion_values[row][column] = _fusion_grid[row][column](x)

        # The fused value for each branch.
        if rows == 1:
            outputs = [fusion_values[0][0], fusion_values[0][1]]
        else:
            outputs = []
            fusion_values = [list(v) for v in zip(*fusion_values)]

            for index, values in enumerate(fusion_values):
                outputs.append(_add_layers_group[index](values))

        return outputs

    return forward
示例#2
0
def block1(x,
           filters,
           kernel_size=3,
           stride=1,
           conv_shortcut=True,
           dilation=1,
           name=None):
    """A residual block.

    # Arguments
        x: input tensor.
        filters: integer, filters of the bottleneck layer.
        kernel_size: default 3, kernel size of the bottleneck layer.
        stride: default 1, stride of the first layer.
        conv_shortcut: default True, use convolution shortcut if True,
            otherwise identity shortcut.
        name: string, block label.

    # Returns
        Output tensor for the residual block.
    """
    bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1

    if conv_shortcut is True:
        if stride == 1:
            shortcut = layers.Conv2D(4 * filters,
                                     1,
                                     strides=stride,
                                     use_bias=False,
                                     name=name + '_0_conv')(x)
            shortcut = layers.BatchNormalization(axis=bn_axis,
                                                 epsilon=1.001e-5,
                                                 name=name + '_0_bn')(shortcut)
        else:
            shortcut = layers.Conv2D(4 * filters,
                                     3,
                                     strides=stride,
                                     use_bias=False,
                                     name=name + '_0_conv')(x)
            shortcut = layers.BatchNormalization(axis=bn_axis,
                                                 epsilon=1.001e-5,
                                                 name=name + '_0_bn')(shortcut)
    else:
        shortcut = x

    x = layers.Conv2D(filters, 1, use_bias=False, name=name + '_1_conv')(x)
    x = layers.BatchNormalization(axis=bn_axis,
                                  epsilon=1.001e-5,
                                  name=name + '_1_bn')(x)
    x = layers.Activation('relu', name=name + '_1_relu')(x)

    padding = 'SAME' if stride == 1 else 'VALID'
    x = layers.Conv2D(filters,
                      kernel_size,
                      strides=stride,
                      padding=padding,
                      dilation_rate=dilation,
                      use_bias=False,
                      name=name + '_2_conv')(x)
    x = layers.BatchNormalization(axis=bn_axis,
                                  epsilon=1.001e-5,
                                  name=name + '_2_bn')(x)
    x = layers.Activation('relu', name=name + '_2_relu')(x)

    x = layers.Conv2D(4 * filters, 1, use_bias=False, name=name + '_3_conv')(x)
    x = layers.BatchNormalization(axis=bn_axis,
                                  epsilon=1.001e-5,
                                  name=name + '_3_bn')(x)

    x = layers.Add(name=name + '_add')([shortcut, x])
    x = layers.Activation('relu', name=name + '_out')(x)
    return x
示例#3
0
def block3(x,
           filters,
           kernel_size=3,
           stride=1,
           groups=32,
           conv_shortcut=True,
           name=None):
    """A residual block.

    Args:
        x: input tensor.
        filters: integer, filters of the bottleneck layer.
        kernel_size: default 3, kernel size of the bottleneck layer.
        stride: default 1, stride of the first layer.
        groups: default 32, group size for grouped convolution.
        conv_shortcut: default True, use convolution shortcut if True,
            otherwise identity shortcut.
        name: string, block label.

    Returns:
        Output tensor for the residual block.
    """
    bn_axis = 3 if backend.image_data_format() == "channels_last" else 1

    if conv_shortcut is True:
        shortcut = layers.Conv2D(
            (64 // groups) * filters,
            1,
            strides=stride,
            use_bias=False,
            name=name + "_0_conv",
        )(x)
        shortcut = layers.BatchNormalization(axis=bn_axis,
                                             epsilon=1.001e-5,
                                             name=name + "_0_bn")(shortcut)
    else:
        shortcut = x

    x = layers.Conv2D(filters, 1, use_bias=False, name=name + "_1_conv")(x)
    x = layers.BatchNormalization(axis=bn_axis,
                                  epsilon=1.001e-5,
                                  name=name + "_1_bn")(x)
    x = layers.Activation("relu", name=name + "_1_relu")(x)

    c = filters // groups
    x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + "_2_pad")(x)
    x = layers.DepthwiseConv2D(
        kernel_size,
        strides=stride,
        depth_multiplier=c,
        use_bias=False,
        name=name + "_2_conv",
    )(x)
    x_shape = backend.int_shape(x)[1:-1]
    x = layers.Reshape(x_shape + (groups, c, c))(x)
    output_shape = x_shape + (groups,
                              c) if backend.backend() == "theano" else None

    x = layers.Lambda(
        lambda x: sum([x[:, :, :, :, i] for i in range(c)]),
        output_shape=output_shape,
        name=name + "_2_reduce",
    )(x)

    x = layers.Reshape(x_shape + (filters, ))(x)

    x = layers.BatchNormalization(axis=bn_axis,
                                  epsilon=1.001e-5,
                                  name=name + "_2_bn")(x)

    x = layers.Activation("relu", name=name + "_2_relu")(x)

    x = layers.Conv2D((64 // groups) * filters,
                      1,
                      use_bias=False,
                      name=name + "_3_conv")(x)

    x = layers.BatchNormalization(axis=bn_axis,
                                  epsilon=1.001e-5,
                                  name=name + "_3_bn")(x)

    x = layers.Add(name=name + "_add")([shortcut, x])
    x = layers.Activation("relu", name=name + "_out")(x)
    return x
示例#4
0
 def call(self, inputs):
     x = self.conv1(inputs)
     x = self.pixel_conv(x)
     x = self.conv2(x)
     return layers.Add([inputs, x])
示例#5
0
def build_BiFPN(features, num_channels, id, freeze_bn=False):
    if id == 0:
        _, _, C3, C4, C5 = features
        P3_in = ConvBlock(num_channels,
                          kernel_size=1,
                          strides=1,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P3'.format(id))(C3)
        P4_in = ConvBlock(num_channels,
                          kernel_size=1,
                          strides=1,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P4'.format(id))(C4)
        P5_in = ConvBlock(num_channels,
                          kernel_size=1,
                          strides=1,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P5'.format(id))(C5)
        P6_in = ConvBlock(num_channels,
                          kernel_size=3,
                          strides=2,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P6'.format(id))(C5)
        P7_in = ConvBlock(num_channels,
                          kernel_size=3,
                          strides=2,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P7'.format(id))(P6_in)
    else:
        P3_in, P4_in, P5_in, P6_in, P7_in = features
        P3_in = ConvBlock(num_channels,
                          kernel_size=1,
                          strides=1,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P3'.format(id))(P3_in)
        P4_in = ConvBlock(num_channels,
                          kernel_size=1,
                          strides=1,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P4'.format(id))(P4_in)
        P5_in = ConvBlock(num_channels,
                          kernel_size=1,
                          strides=1,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P5'.format(id))(P5_in)
        P6_in = ConvBlock(num_channels,
                          kernel_size=1,
                          strides=1,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P6'.format(id))(P6_in)
        P7_in = ConvBlock(num_channels,
                          kernel_size=1,
                          strides=1,
                          freeze_bn=freeze_bn,
                          name='BiFPN_{}_P7'.format(id))(P7_in)

    # upsample
    P7_U = layers.UpSampling2D()(P7_in)
    P6_td = layers.Add()([P7_U, P6_in])
    P6_td = DepthwiseConvBlock(kernel_size=3,
                               strides=1,
                               freeze_bn=freeze_bn,
                               name='BiFPN_{}_U_P6'.format(id))(P6_td)
    P6_U = layers.UpSampling2D()(P6_td)
    P5_td = layers.Add()([P6_U, P5_in])
    P5_td = DepthwiseConvBlock(kernel_size=3,
                               strides=1,
                               freeze_bn=freeze_bn,
                               name='BiFPN_{}_U_P5'.format(id))(P5_td)
    P5_U = layers.UpSampling2D()(P5_td)
    P4_td = layers.Add()([P5_U, P4_in])
    P4_td = DepthwiseConvBlock(kernel_size=3,
                               strides=1,
                               freeze_bn=freeze_bn,
                               name='BiFPN_{}_U_P4'.format(id))(P4_td)
    P4_U = layers.UpSampling2D()(P4_td)
    P3_out = layers.Add()([P4_U, P3_in])
    P3_out = DepthwiseConvBlock(kernel_size=3,
                                strides=1,
                                freeze_bn=freeze_bn,
                                name='BiFPN_{}_U_P3'.format(id))(P3_out)
    # downsample
    P3_D = layers.MaxPooling2D(strides=(2, 2))(P3_out)
    P4_out = layers.Add()([P3_D, P4_td, P4_in])
    P4_out = DepthwiseConvBlock(kernel_size=3,
                                strides=1,
                                freeze_bn=freeze_bn,
                                name='BiFPN_{}_D_P4'.format(id))(P4_out)
    P4_D = layers.MaxPooling2D(strides=(2, 2))(P4_out)
    P5_out = layers.Add()([P4_D, P5_td, P5_in])
    P5_out = DepthwiseConvBlock(kernel_size=3,
                                strides=1,
                                freeze_bn=freeze_bn,
                                name='BiFPN_{}_D_P5'.format(id))(P5_out)
    P5_D = layers.MaxPooling2D(strides=(2, 2))(P5_out)
    P6_out = layers.Add()([P5_D, P6_td, P6_in])
    P6_out = DepthwiseConvBlock(kernel_size=3,
                                strides=1,
                                freeze_bn=freeze_bn,
                                name='BiFPN_{}_D_P6'.format(id))(P6_out)
    P6_D = layers.MaxPooling2D(strides=(2, 2))(P6_out)
    P7_out = layers.Add()([P6_D, P7_in])
    P7_out = DepthwiseConvBlock(kernel_size=3,
                                strides=1,
                                freeze_bn=freeze_bn,
                                name='BiFPN_{}_D_P7'.format(id))(P7_out)

    return P3_out, P4_out, P5_out, P6_out, P7_out
    def build(self, mode, config):

        tf.compat.v1.disable_eager_execution()

        #input
        input_image = keras.Input(shape=config.IMAGE_SHAPE.tolist(),
                                  name="input_image")
        input_image_meta = keras.Input(shape=[None], name="input_image_meta")

        if mode == "train":
            #RPN
            input_rpn_match = keras.Input(shape=[None, 1],
                                          name="input_rpn_match",
                                          dtype=tf.int32)  #match
            input_rpn_bbox = keras.Input(shape=[None, 4],
                                         name="input_rpn_bbox",
                                         dtype=tf.float32)  #bounding box

            #ground truth
            input_gt_ids = keras.Input(shape=[None],
                                       name="input_gt_class_ids",
                                       dtype=tf.int32)  #GT class IDs
            input_gt_boxes = keras.Input(shape=[None, 4],
                                         name="input_gt_boxes",
                                         dtype=tf.float32)

            h, w = K.shape(input_image)[1], K.shape(input_image)[2]
            image_scale = K.cast(K.stack([h, w, h, w], axis=0), tf.float32)

            #normalize ground truth boxes
            gt_boxes = layers.Lambda(lambda x: x / image_scale)(input_gt_boxes)

            #mini_mask
            #input_gt_masks = keras.Input(shape = [config.MINI_MASK_SHAPE[0],
            #                                      config.MINI_MASK_SHAPE[1], None],
            #                             name = "input_gt_masks", dtype=bool)

        #resnet layer
        _, C2, C3, C4, C5 = resnet101.build_layers(input_image)
        #FPN
        P5 = layers.Conv2D(256, (1, 1), name='fpn_c5p5')(C5)
        P4 = layers.Add(name="fpn_p4add")([
            layers.UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
            layers.Conv2D(256, (1, 1), name='fpn_c4p4')(C4)
        ])
        P3 = layers.Add(name="fpn_p3add")([
            layers.UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
            layers.Conv2D(256, (1, 1), name='fpn_c3p3')(C3)
        ])
        P2 = layers.Add(name="fpn_p2add")([
            layers.UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
            layers.Conv2D(256, (1, 1), name='fpn_c2p2')(C2)
        ])
        # Attach 3x3 conv to all P layers to get the final feature maps.
        P2 = layers.Conv2D(256, (3, 3), padding="SAME", name="fpn_p2")(P2)
        P3 = layers.Conv2D(256, (3, 3), padding="SAME", name="fpn_p3")(P3)
        P4 = layers.Conv2D(256, (3, 3), padding="SAME", name="fpn_p4")(P4)
        P5 = layers.Conv2D(256, (3, 3), padding="SAME", name="fpn_p5")(P5)
        # P6 is used for the 5th anchor scale in RPN. Generated by
        # subsampling from P5 with stride of 2.
        P6 = layers.MaxPooling2D(pool_size=(1, 1), strides=2,
                                 name="fpn_p6")(P5)

        RPN_feature = [P2, P3, P4, P5, P6]
        RCNN_feature = [P2, P3, P4, P5]

        self.anchors = utils.generate_anchors(self.config.ANCHOR_SCALES,
                                              self.config.ANCHOR_RATIOS,
                                              self.config.ANCHOR_STRIDE,
                                              self.config.BACKBONE_SHAPES,
                                              self.config.BACKBONE_STRIDES)

        #RPN Keras model
        input_feature = keras.Input(shape=[None, None, config.PIRAMID_SIZE],
                                    name="input_rpn_feature_map")
        """
        rpn_class_cls: anchor class classifier
        rpn_probs: anchor classifier probability
        rpn_bbox_offset: anchor bounding box offset
        """
        outputs = RPN.build_graph(input_feature, len(config.ANCHOR_RATIOS),
                                  config.ANCHOR_STRIDE)

        RPN_model = keras.Model([input_feature], outputs, name="rpn_model")
        """
        In FPN, we generate a pyramid of feature maps. We apply the RPN (described in the previous section)
        to generate ROIs. Based on the size of the ROI,
        we select the feature map layer in the most proper scale to extract the feature patches.
        """
        layer_outputs = []
        for x in RPN_feature:
            layer_outputs.append(RPN_model([x]))

        output_names = ["rpn_class_logits", "rpn_class", "rpn_bbox"]
        outputs = list(zip(*layer_outputs))
        outputs = [
            layers.Concatenate(axis=1, name=n)(list(o))
            for o, n in zip(outputs, output_names)
        ]

        #rpn_class_cls, rpn_probs, rpn_bbox = layer_outputs
        rpn_class_ids, rpn_probs, rpn_bbox_offset = outputs

        #Proposal layer
        if mode == "train":
            num_proposal = config.NUM_ROI_TRAINING
        else:
            num_proposal = config.NUM_ROI_INFERENCE

        ROIS_proposals = ProposalLayer(
            num_proposal=num_proposal,
            nms_threshold=config.NMS_THRESHOLD,
            anchors=self.anchors,
            config=config)([rpn_probs, rpn_bbox_offset])

        #combine together
        if mode == "train":
            #class ids
            total_class_ids = layers.Lambda(lambda x: utils.parse_image_meta(x)
                                            ["class_ids"])(input_image_meta)

            #Subsamples proposals and generates target box refinement, class_ids 1.7
            #ratio postive/negative rois = 1/3 (threshold = 0.5)
            #target_ids: class ids of gt boxes closest to positive roi
            #target_bbox = offset from positive rois to it's closest gt_box

            target_rois = ROIS_proposals

            #rois, target_ids, target_bbox, target_mask =\
            # TrainingDetectionLayer(config, name="proposal_targets")([target_rois,
            #                                                          input_gt_ids,gt_boxes, input_gt_masks])

            rois, target_ids, target_bbox =\
             TrainingDetectionLayer(config, name="proposal_targets")([target_rois,
                                                                      input_gt_ids,gt_boxes])

            #classification and regression ROIs after RPN through FPN
            rcnn_class_ids, rcnn_class_probs, rcnn_bbox = fpn_classifier(
                rois, RCNN_feature, config.IMAGE_SHAPE, config.POOL_SIZE,
                config.NUM_CLASSES)

            #rcnn_mask = fpn_mask(rois, RCNN_feature, config.IMAGE_SHAPE, config.MASK_POOL_SIZE, config.NUM_CLASSES)

            output_rois = layers.Lambda(lambda x: x * 1,
                                        name="output_rois")(rois)

            #rpn losses
            rpn_class_loss = layers.Lambda(
                lambda x: losses.rpn_class_loss_func(*x),
                name="rpn_class_loss")([input_rpn_match, rpn_class_ids])

            rpn_bbox_loss = layers.Lambda(
                lambda x: losses.rpn_bbox_loss_func(config, *x),
                name="rpn_bbox_loss")(
                    [input_rpn_bbox, input_rpn_match, rpn_bbox_offset])
            #rcnn losses

            rcnn_class_loss = layers.Lambda(
                lambda x: losses.rcnn_class_loss_func(*x),
                name="mrcnn_class_loss")(
                    [target_ids, rcnn_class_ids, total_class_ids])

            rcnn_bbox_loss = layers.Lambda(
                lambda x: losses.rcnn_bbox_loss_func(*x),
                name="mrcnn_bbox_loss")([target_bbox, target_ids, rcnn_bbox])

            #rcnn_mask_loss = layers.Lambda(lambda x : losses.rcnn_mask_loss_func(*x), name="mrcnn_mask_loss")(
            #                 [target_mask, target_ids, rcnn_mask])

            #MODEL
            """
            inputs = [input_image, input_image_meta, input_rpn_match, input_rpn_bbox,
                      input_gt_ids, input_gt_boxes, input_gt_masks]
            outputs = [rpn_class_ids, rpn_probs, rpn_bbox_offset,
                       rcnn_class_ids,rcnn_class_probs, rcnn_bbox, rcnn_mask,
                       ROIS_proposals, output_rois,
                       rpn_class_loss, rpn_bbox_loss, rcnn_class_loss, rcnn_bbox_loss, rcnn_mask_loss]
            """
            inputs = [
                input_image, input_image_meta, input_rpn_match, input_rpn_bbox,
                input_gt_ids, input_gt_boxes
            ]
            outputs = [
                rpn_class_ids, rpn_probs, rpn_bbox_offset, rcnn_class_ids,
                rcnn_class_probs, rcnn_bbox, ROIS_proposals, output_rois,
                rpn_class_loss, rpn_bbox_loss, rcnn_class_loss, rcnn_bbox_loss
            ]

            model = keras.Model(inputs, outputs, name='mask_rcnn')

        else:
            rcnn_class_ids,rcnn_class_probs, rcnn_bbox =\
                fpn_classifier(ROIS_proposals, RCNN_feature, config.IMAGE_SHAPE,
                               config.POOL_SIZE,config.NUM_CLASSES)

            #[N, (y1, x1, y2, x2, class_id, score)]
            detections = InferenceDetectionLayer(config,
                                                 name="mrcnn_detection")([
                                                     ROIS_proposals,
                                                     rcnn_class_probs,
                                                     rcnn_bbox,
                                                     input_image_meta
                                                 ])

            #h,w = config.IMAGE_SHAPE[:2]

            #detection_boxes = layers.Lambda(
            #                  lambda x: x[..., :4] / np.array([h,w,h,w]))(detections)

            inputs = [input_image, input_image_meta]
            outputs = [
                detections, rcnn_class_probs, rcnn_bbox, ROIS_proposals,
                rpn_probs, rpn_bbox_offset
            ]

            model = keras.Model(inputs, outputs, name='mask_rcnn')

        #print(model.layers)
        return model
示例#7
0
def build_BiFPN(features, num_channels, id, freeze_bn=False):
    if id == 0:
        _, _, C3, C4, C5 = features
        P3_in = C3
        P4_in = C4
        P5_in = C5
        P6_in = layers.Conv2D(num_channels,
                              kernel_size=1,
                              padding='same',
                              name='resample_p6/conv2d')(C5)
        P6_in = layers.BatchNormalization(momentum=MOMENTUM,
                                          epsilon=EPSILON,
                                          name='resample_p6/bn')(P6_in)
        # P6_in = BatchNormalization(freeze=freeze_bn, name='resample_p6/bn')(P6_in)
        P6_in = layers.MaxPooling2D(pool_size=3,
                                    strides=2,
                                    padding='same',
                                    name='resample_p6/maxpool')(P6_in)
        P7_in = layers.MaxPooling2D(pool_size=3,
                                    strides=2,
                                    padding='same',
                                    name='resample_p7/maxpool')(P6_in)
        P7_U = layers.UpSampling2D()(P7_in)
        P6_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode0/add')(
            [P6_in, P7_U])
        P6_td = layers.Activation(lambda x: tf.nn.swish(x))(P6_td)
        P6_td = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode0/op_after_combine5')(P6_td)
        P5_in_1 = layers.Conv2D(
            num_channels,
            kernel_size=1,
            padding='same',
            name=f'fpn_cells/cell_{id}/fnode1/resample_0_2_6/conv2d')(P5_in)
        P5_in_1 = layers.BatchNormalization(
            momentum=MOMENTUM,
            epsilon=EPSILON,
            name=f'fpn_cells/cell_{id}/fnode1/resample_0_2_6/bn')(P5_in_1)
        # P5_in_1 = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode1/resample_0_2_6/bn')(P5_in_1)
        P6_U = layers.UpSampling2D()(P6_td)
        P5_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode1/add')(
            [P5_in_1, P6_U])
        P5_td = layers.Activation(lambda x: tf.nn.swish(x))(P5_td)
        P5_td = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode1/op_after_combine6')(P5_td)
        P4_in_1 = layers.Conv2D(
            num_channels,
            kernel_size=1,
            padding='same',
            name=f'fpn_cells/cell_{id}/fnode2/resample_0_1_7/conv2d')(P4_in)
        P4_in_1 = layers.BatchNormalization(
            momentum=MOMENTUM,
            epsilon=EPSILON,
            name=f'fpn_cells/cell_{id}/fnode2/resample_0_1_7/bn')(P4_in_1)
        # P4_in_1 = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode2/resample_0_1_7/bn')(P4_in_1)
        P5_U = layers.UpSampling2D()(P5_td)
        P4_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode2/add')(
            [P4_in_1, P5_U])
        P4_td = layers.Activation(lambda x: tf.nn.swish(x))(P4_td)
        P4_td = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode2/op_after_combine7')(P4_td)
        P3_in = layers.Conv2D(
            num_channels,
            kernel_size=1,
            padding='same',
            name=f'fpn_cells/cell_{id}/fnode3/resample_0_0_8/conv2d')(P3_in)
        P3_in = layers.BatchNormalization(
            momentum=MOMENTUM,
            epsilon=EPSILON,
            name=f'fpn_cells/cell_{id}/fnode3/resample_0_0_8/bn')(P3_in)
        # P3_in = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode3/resample_0_0_8/bn')(P3_in)
        P4_U = layers.UpSampling2D()(P4_td)
        P3_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode3/add')(
            [P3_in, P4_U])
        P3_out = layers.Activation(lambda x: tf.nn.swish(x))(P3_out)
        P3_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode3/op_after_combine8')(P3_out)
        P4_in_2 = layers.Conv2D(
            num_channels,
            kernel_size=1,
            padding='same',
            name=f'fpn_cells/cell_{id}/fnode4/resample_0_1_9/conv2d')(P4_in)
        P4_in_2 = layers.BatchNormalization(
            momentum=MOMENTUM,
            epsilon=EPSILON,
            name=f'fpn_cells/cell_{id}/fnode4/resample_0_1_9/bn')(P4_in_2)
        # P4_in_2 = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode4/resample_0_1_9/bn')(P4_in_2)
        P3_D = layers.MaxPooling2D(pool_size=3, strides=2,
                                   padding='same')(P3_out)
        P4_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode4/add')(
            [P4_in_2, P4_td, P3_D])
        P4_out = layers.Activation(lambda x: tf.nn.swish(x))(P4_out)
        P4_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode4/op_after_combine9')(P4_out)

        P5_in_2 = layers.Conv2D(
            num_channels,
            kernel_size=1,
            padding='same',
            name=f'fpn_cells/cell_{id}/fnode5/resample_0_2_10/conv2d')(P5_in)
        P5_in_2 = layers.BatchNormalization(
            momentum=MOMENTUM,
            epsilon=EPSILON,
            name=f'fpn_cells/cell_{id}/fnode5/resample_0_2_10/bn')(P5_in_2)
        # P5_in_2 = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode5/resample_0_2_10/bn')(P5_in_2)
        P4_D = layers.MaxPooling2D(pool_size=3, strides=2,
                                   padding='same')(P4_out)
        P5_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode5/add')(
            [P5_in_2, P5_td, P4_D])
        P5_out = layers.Activation(lambda x: tf.nn.swish(x))(P5_out)
        P5_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode5/op_after_combine10')(P5_out)

        P5_D = layers.MaxPooling2D(pool_size=3, strides=2,
                                   padding='same')(P5_out)
        P6_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode6/add')(
            [P6_in, P6_td, P5_D])
        P6_out = layers.Activation(lambda x: tf.nn.swish(x))(P6_out)
        P6_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode6/op_after_combine11')(P6_out)

        P6_D = layers.MaxPooling2D(pool_size=3, strides=2,
                                   padding='same')(P6_out)
        P7_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode7/add')(
            [P7_in, P6_D])
        P7_out = layers.Activation(lambda x: tf.nn.swish(x))(P7_out)
        P7_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode7/op_after_combine12')(P7_out)

    else:
        P3_in, P4_in, P5_in, P6_in, P7_in = features
        P7_U = layers.UpSampling2D()(P7_in)
        P6_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode0/add')(
            [P6_in, P7_U])
        P6_td = layers.Activation(lambda x: tf.nn.swish(x))(P6_td)
        P6_td = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode0/op_after_combine5')(P6_td)
        P6_U = layers.UpSampling2D()(P6_td)
        P5_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode1/add')(
            [P5_in, P6_U])
        P5_td = layers.Activation(lambda x: tf.nn.swish(x))(P5_td)
        P5_td = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode1/op_after_combine6')(P5_td)
        P5_U = layers.UpSampling2D()(P5_td)
        P4_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode2/add')(
            [P4_in, P5_U])
        P4_td = layers.Activation(lambda x: tf.nn.swish(x))(P4_td)
        P4_td = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode2/op_after_combine7')(P4_td)
        P4_U = layers.UpSampling2D()(P4_td)
        P3_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode3/add')(
            [P3_in, P4_U])
        P3_out = layers.Activation(lambda x: tf.nn.swish(x))(P3_out)
        P3_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode3/op_after_combine8')(P3_out)
        P3_D = layers.MaxPooling2D(pool_size=3, strides=2,
                                   padding='same')(P3_out)
        P4_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode4/add')(
            [P4_in, P4_td, P3_D])
        P4_out = layers.Activation(lambda x: tf.nn.swish(x))(P4_out)
        P4_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode4/op_after_combine9')(P4_out)

        P4_D = layers.MaxPooling2D(pool_size=3, strides=2,
                                   padding='same')(P4_out)
        P5_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode5/add')(
            [P5_in, P5_td, P4_D])
        P5_out = layers.Activation(lambda x: tf.nn.swish(x))(P5_out)
        P5_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode5/op_after_combine10')(P5_out)

        P5_D = layers.MaxPooling2D(pool_size=3, strides=2,
                                   padding='same')(P5_out)
        P6_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode6/add')(
            [P6_in, P6_td, P5_D])
        P6_out = layers.Activation(lambda x: tf.nn.swish(x))(P6_out)
        P6_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode6/op_after_combine11')(P6_out)

        P6_D = layers.MaxPooling2D(pool_size=3, strides=2,
                                   padding='same')(P6_out)
        P7_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode7/add')(
            [P7_in, P6_D])
        P7_out = layers.Activation(lambda x: tf.nn.swish(x))(P7_out)
        P7_out = SeparableConvBlock(
            num_channels=num_channels,
            kernel_size=3,
            strides=1,
            name=f'fpn_cells/cell_{id}/fnode7/op_after_combine12')(P7_out)
    return P3_out, P4_td, P5_td, P6_td, P7_out
示例#8
0
    def build(self, mode, config):
        """Build Mask R-CNN architecture.
            input_shape: The shape of the input image.
            mode: Either "training" or "inference". The inputs and
                outputs of the model differ accordingly.
        """
        assert mode in ['training', 'inference']

        # Image size must be dividable by 2 multiple times
        h, w = config.IMAGE_SHAPE[:2]
        if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6):
            raise Exception(
                "Image size must be dividable by 2 at least 6 times "
                "to avoid fractions when downscaling and upscaling."
                "For example, use 256, 320, 384, 448, 512, ... etc. ")

        # Inputs
        input_image = KL.Input(shape=[None, None, config.IMAGE_SHAPE[2]],
                               name="input_image")
        input_image_meta = KL.Input(shape=[config.IMAGE_META_SIZE],
                                    name="input_image_meta")
        if mode == "training":
            # RPN GT
            input_rpn_match = KL.Input(shape=[None, 1],
                                       name="input_rpn_match",
                                       dtype=tf.int32)
            input_rpn_bbox = KL.Input(shape=[None, 4],
                                      name="input_rpn_bbox",
                                      dtype=tf.float32)

            # Detection GT (class IDs, bounding boxes, and masks)
            # 1. GT Class IDs (zero padded)
            input_gt_class_ids = KL.Input(shape=[None],
                                          name="input_gt_class_ids",
                                          dtype=tf.int32)
            # 2. GT Boxes in pixels (zero padded)
            # [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] in image coordinates
            input_gt_boxes = KL.Input(shape=[None, 4],
                                      name="input_gt_boxes",
                                      dtype=tf.float32)
            # Normalize coordinates
            gt_boxes = KL.Lambda(lambda x: norm_boxes_graph(
                x,
                K.shape(input_image)[1:3]))(input_gt_boxes)
            # 3. GT Masks (zero padded)
            # [batch, height, width, MAX_GT_INSTANCES]
            if config.USE_MINI_MASK:
                input_gt_masks = KL.Input(shape=[
                    config.MINI_MASK_SHAPE[0], config.MINI_MASK_SHAPE[1], None
                ],
                                          name="input_gt_masks",
                                          dtype=bool)
            else:
                input_gt_masks = KL.Input(
                    shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None],
                    name="input_gt_masks",
                    dtype=bool)
        elif mode == "inference":
            # Anchors in normalized coordinates
            input_anchors = KL.Input(shape=[None, 4], name="input_anchors")

        # Build the shared convolutional layers.
        # Bottom-up Layers
        # Returns a list of the last layers of each stage, 5 in total.
        # Don't create the thead (stage 5), so we pick the 4th item in the list.
        if callable(config.BACKBONE):
            _, C2, C3, C4, C5 = config.BACKBONE(input_image,
                                                stage5=True,
                                                train_bn=config.TRAIN_BN)
        else:
            _, C2, C3, C4, C5 = resnet_graph(input_image,
                                             config.BACKBONE,
                                             stage5=True,
                                             train_bn=config.TRAIN_BN)
        # Top-down Layers
        # TODO: add assert to varify feature map sizes match what's in config
        P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
                       name='fpn_c5p5')(C5)
        P4 = KL.Add(name="fpn_p4add")([
            KL.UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
            KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
                      name='fpn_c4p4')(C4)
        ])
        P3 = KL.Add(name="fpn_p3add")([
            KL.UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
            KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
                      name='fpn_c3p3')(C3)
        ])
        P2 = KL.Add(name="fpn_p2add")([
            KL.UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
            KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
                      name='fpn_c2p2')(C2)
        ])
        # Attach 3x3 conv to all P layers to get the final feature maps.
        P2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                       padding="SAME",
                       name="fpn_p2")(P2)
        P3 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                       padding="SAME",
                       name="fpn_p3")(P3)
        P4 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                       padding="SAME",
                       name="fpn_p4")(P4)
        P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                       padding="SAME",
                       name="fpn_p5")(P5)
        # P6 is used for the 5th anchor scale in RPN. Generated by
        # # subsampling from P5 with stride of 2.
        P6 = KL.MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5)

        # Bottom-up Layers
        N3 = KL.Add(name="fpn_n3add")([
            P3,
            KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                      strides=(2, 2),
                      padding="SAME",
                      name='fpn_n2conv')(P2)
        ])
        N4 = KL.Add(name="fpn_n4add")([
            P4,
            KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                      strides=(2, 2),
                      padding="SAME",
                      name='fpn_n4conv')(N3)
        ])
        N5 = KL.Add(name="fpn_n5add")([
            P5,
            KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                      strides=(2, 2),
                      padding="SAME",
                      name='fpn_n5conv')(N4)
        ])

        N3 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                       strides=(1, 1),
                       padding="SAME",
                       name="fpn_n3")(N3)
        N4 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                       strides=(1, 1),
                       padding="SAME",
                       name="fpn_n4")(N4)
        N5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
                       strides=(1, 1),
                       padding="SAME",
                       name="fpn_n5")(N5)
        N6 = KL.MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_n6")(N5)

        # Note that P6 is used in RPN, but not in the classifier heads.
        rpn_feature_maps = [P2, N3, N4, N5, N6]
        mrcnn_feature_maps = [P2, N3, N4, N5]

        # Anchors
        if mode == "training":
            anchors = self.get_anchors(config.IMAGE_SHAPE)
            # Duplicate across the batch dimension because Keras requires it
            # TODO: can this be optimized to avoid duplicating the anchors?
            anchors = np.broadcast_to(anchors,
                                      (config.BATCH_SIZE, ) + anchors.shape)
            # A hack to get around Keras's bad support for constants
            anchors = KL.Lambda(lambda x: tf.Variable(anchors),
                                name="anchors")(input_image)
        else:
            anchors = input_anchors

        # RPN Model
        rpn = build_rpn_model(config.RPN_ANCHOR_STRIDE,
                              len(config.RPN_ANCHOR_RATIOS),
                              config.TOP_DOWN_PYRAMID_SIZE)
        # Loop through pyramid layers
        layer_outputs = []  # list of lists
        for p in rpn_feature_maps:
            layer_outputs.append(rpn([p]))
        # Concatenate layer outputs
        # Convert from list of lists of level outputs to list of lists
        # of outputs across levels.
        # e.g. [[a1, b1, c1], [a2, b2, c2]] => [[a1, a2], [b1, b2], [c1, c2]]
        output_names = ["rpn_class_logits", "rpn_class", "rpn_bbox"]
        outputs = list(zip(*layer_outputs))
        outputs = [
            KL.Concatenate(axis=1, name=n)(list(o))
            for o, n in zip(outputs, output_names)
        ]

        rpn_class_logits, rpn_class, rpn_bbox = outputs

        # Generate proposals
        # Proposals are [batch, N, (y1, x1, y2, x2)] in normalized coordinates
        # and zero padded.
        proposal_count = config.POST_NMS_ROIS_TRAINING if mode == "training" \
            else config.POST_NMS_ROIS_INFERENCE
        rpn_rois = ProposalLayer(proposal_count=proposal_count,
                                 nms_threshold=config.RPN_NMS_THRESHOLD,
                                 name="ROI",
                                 config=config)([rpn_class, rpn_bbox, anchors])

        if mode == "training":
            # Class ID mask to mark class IDs supported by the dataset the image
            # came from.
            active_class_ids = KL.Lambda(lambda x: parse_image_meta_graph(x)[
                "active_class_ids"])(input_image_meta)

            if not config.USE_RPN_ROIS:
                # Ignore predicted ROIs and use ROIs provided as an input.
                input_rois = KL.Input(shape=[config.POST_NMS_ROIS_TRAINING, 4],
                                      name="input_roi",
                                      dtype=np.int32)
                # Normalize coordinates
                target_rois = KL.Lambda(lambda x: norm_boxes_graph(
                    x,
                    K.shape(input_image)[1:3]))(input_rois)
            else:
                target_rois = rpn_rois

            # Generate detection targets
            # Subsamples proposals and generates target outputs for training
            # Note that proposal class IDs, gt_boxes, and gt_masks are zero
            # padded. Equally, returned rois and targets are zero padded.
            rois, target_class_ids, target_bbox, target_mask = \
                DetectionTargetLayer(config, name="proposal_targets")([
                    target_rois, input_gt_class_ids, gt_boxes, input_gt_masks])

            # Network Heads
            # TODO: verify that this handles zero padded ROIs
            mrcnn_class_logits, mrcnn_class, mrcnn_bbox = \
                panet_fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta,
                                           config.POOL_SIZE, config.NUM_CLASSES,
                                           train_bn=config.TRAIN_BN,
                                           fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)

            mrcnn_mask = panet_build_fpn_mask_graph(rois,
                                                    mrcnn_feature_maps,
                                                    input_image_meta,
                                                    config.MASK_POOL_SIZE,
                                                    config.NUM_CLASSES,
                                                    train_bn=config.TRAIN_BN)

            # TODO: clean up (use tf.identify if necessary)
            output_rois = KL.Lambda(lambda x: x * 1, name="output_rois")(rois)

            # Losses
            rpn_class_loss = KL.Lambda(lambda x: rpn_class_loss_graph(*x),
                                       name="rpn_class_loss")(
                                           [input_rpn_match, rpn_class_logits])
            rpn_bbox_loss = KL.Lambda(
                lambda x: rpn_bbox_loss_graph(config, *x),
                name="rpn_bbox_loss")(
                    [input_rpn_bbox, input_rpn_match, rpn_bbox])
            class_loss = KL.Lambda(lambda x: mrcnn_class_loss_graph(*x),
                                   name="mrcnn_class_loss")([
                                       target_class_ids, mrcnn_class_logits,
                                       active_class_ids
                                   ])
            bbox_loss = KL.Lambda(lambda x: mrcnn_bbox_loss_graph(*x),
                                  name="mrcnn_bbox_loss")([
                                      target_bbox, target_class_ids, mrcnn_bbox
                                  ])
            mask_loss = KL.Lambda(lambda x: mrcnn_mask_loss_graph(*x),
                                  name="mrcnn_mask_loss")([
                                      target_mask, target_class_ids, mrcnn_mask
                                  ])

            # Model
            inputs = [
                input_image, input_image_meta, input_rpn_match, input_rpn_bbox,
                input_gt_class_ids, input_gt_boxes, input_gt_masks
            ]
            if not config.USE_RPN_ROIS:
                inputs.append(input_rois)
            outputs = [
                rpn_class_logits, rpn_class, rpn_bbox, mrcnn_class_logits,
                mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, output_rois,
                rpn_class_loss, rpn_bbox_loss, class_loss, bbox_loss, mask_loss
            ]
            model = KM.Model(inputs, outputs, name='mask_rcnn')
        else:
            # Network Heads
            # Proposal classifier and BBox regressor heads
            mrcnn_class_logits, mrcnn_class, mrcnn_bbox = \
                panet_fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta,
                                           config.POOL_SIZE, config.NUM_CLASSES,
                                           train_bn=config.TRAIN_BN,
                                           fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)

            # Detections
            # output is [batch, num_detections, (y1, x1, y2, x2, class_id, score)] in
            # normalized coordinates
            detections = DetectionLayer(config, name="mrcnn_detection")(
                [rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta])

            # Create masks for detections
            detection_boxes = KL.Lambda(lambda x: x[..., :4])(detections)
            mrcnn_mask = panet_build_fpn_mask_graph(detection_boxes,
                                                    mrcnn_feature_maps,
                                                    input_image_meta,
                                                    config.MASK_POOL_SIZE,
                                                    config.NUM_CLASSES,
                                                    train_bn=config.TRAIN_BN)

            model = KM.Model([input_image, input_image_meta, input_anchors], [
                detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois,
                rpn_class, rpn_bbox
            ],
                             name='mask_rcnn')

        # Add multi-GPU support.
        if config.GPU_COUNT > 1:
            from .parallel_model import ParallelModel
            model = ParallelModel(model, config.GPU_COUNT)

        return model
示例#9
0
def block2(x,
           filters,
           kernel_size=3,
           stride=1,
           conv_shortcut=False,
           name=None,
           norm_use="bn"):
    """A residual block.
    # Arguments
        x: input tensor.
        filters: integer, filters of the bottleneck layer.
        kernel_size: default 3, kernel size of the bottleneck layer.
        stride: default 1, stride of the first layer.
        conv_shortcut: default False, use convolution shortcut if True,
            otherwise identity shortcut.
        name: string, block label.
    # Returns
        Output tensor for the residual block.
    """
    bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1

    #preact = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_preact_bn')(x)
    preact = normalize_layer(x, norm_use=norm_use, name=name + '_preact_')
    preact = layers.Activation('relu', name=name + '_preact_relu')(preact)

    if conv_shortcut is True:
        shortcut = layers.Conv2D(4 * filters,
                                 1,
                                 strides=stride,
                                 kernel_initializer='he_normal',
                                 name=name + '_0_conv')(preact)
    else:
        shortcut = layers.MaxPooling2D(1,
                                       strides=stride)(x) if stride > 1 else x

    x = layers.Conv2D(filters,
                      1,
                      strides=1,
                      use_bias=False,
                      kernel_initializer='he_normal',
                      name=name + '_1_conv')(preact)
    x = normalize_layer(x, norm_use=norm_use, name=name + '_1_')
    #x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_1_bn')(x)
    x = layers.Activation('relu', name=name + '_1_relu')(x)

    x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x)
    x = layers.Conv2D(filters,
                      kernel_size,
                      strides=stride,
                      kernel_initializer='he_normal',
                      use_bias=False,
                      name=name + '_2_conv')(x)
    x = normalize_layer(x, norm_use=norm_use, name=name + '_2_')
    #x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_2_bn')(x)
    x = layers.Activation('relu', name=name + '_2_relu')(x)

    x = layers.Conv2D(4 * filters,
                      1,
                      name=name + '_3_conv',
                      kernel_initializer='he_normal')(x)
    x = layers.Add(name=name + '_out')([shortcut, x])
    return x
示例#10
0
    def build_generator(self):
        def residual_block(layer_input, filters):
            #"""Residual block described in paper"""
            d = layers.Conv2D(filters,
                              kernel_size=3,
                              strides=1,
                              padding='same')(layer_input)
            d = layers.Activation('relu')(d)
            d = layers.BatchNormalization(momentum=0.8)(d)
            d = layers.Conv2D(filters,
                              kernel_size=3,
                              strides=1,
                              padding='same')(d)
            d = layers.BatchNormalization(momentum=0.8)(d)
            d = layers.Add()([d, layer_input])
            return d

        def deconv2d(layer_input):
            #"""Layers used during upsampling"""
            u = layers.UpSampling2D(size=2)(layer_input)
            u = layers.Conv2D(256, kernel_size=3, strides=1, padding='same')(u)
            u = layers.Activation('relu')(u)
            return u

        img_lr_shape = (config.input_width, config.input_height, 3)

        model = Sequential()
        img_lr = Input(shape=img_lr_shape)

        c1 = layers.Conv2D(64, kernel_size=9, strides=1,
                           padding='same')(img_lr * 2 - 1)
        c1 = layers.Activation('relu')(c1)

        # Local variables
        self.gf = 64
        self.n_residual_blocks = 16

        # Propogate through residual blocks
        r = residual_block(c1, self.gf)
        for _ in range(self.n_residual_blocks - 1):
            r = residual_block(r, self.gf)

        # Post-residual block
        c2 = layers.Conv2D(64, kernel_size=3, strides=1, padding='same')(r)
        c2 = layers.BatchNormalization(momentum=0.8)(c2)
        c2 = layers.Add()([c2, c1])

        # Upsampling
        u1 = deconv2d(c2)
        u2 = deconv2d(u1)
        u3 = deconv2d(u2)

        # Generate high resolution output
        gen_hr = layers.Conv2D(self.channels,
                               kernel_size=9,
                               strides=1,
                               padding='same',
                               activation='tanh')(u3)
        gen_hr = gen_hr * 0.5 + 0.5

        return Model(img_lr, gen_hr)
    shape=(time_sound, nfreqs,
           1))  # define input (rows, columns, channels (only one in my case))
model_l_conv1 = layers.Conv2D(32, (3, 7), activation='relu', padding='same')(
    in1)  # define first layer and input to the layer
model_l_conv1_mp = layers.MaxPooling2D(pool_size=(1, 2))(model_l_conv1)
model_l_conv1_mp_do = layers.Dropout(0.2)(model_l_conv1_mp)

# CNN 1 - right channel
in2 = layers.Input(shape=(time_sound, nfreqs, 1))  # define input
model_r_conv1 = layers.Conv2D(32, (3, 7), activation='relu', padding='same')(
    in2)  # define first layer and input to the layer
model_r_conv1_mp = layers.MaxPooling2D(pool_size=(1, 2))(model_r_conv1)
model_r_conv1_mp_do = layers.Dropout(0.2)(model_r_conv1_mp)

# CNN 2 - merged
model_final_merge = layers.Add()([model_l_conv1_mp_do, model_r_conv1_mp_do])
model_final_conv1 = layers.Conv2D(32, (3, 7),
                                  activation='relu',
                                  padding='same')(model_final_merge)
model_final_conv1_mp = layers.MaxPooling2D(pool_size=(2, 2))(model_final_conv1)
model_final_conv1_mp_do = layers.Dropout(0.2)(model_final_conv1_mp)

# CNN 3 - merged
model_final_conv2 = layers.Conv2D(64, (3, 7),
                                  activation='relu',
                                  padding='same')(model_final_conv1_mp_do)
model_final_conv2_mp = layers.MaxPooling2D(pool_size=(2, 2))(model_final_conv2)
model_final_conv2_mp_do = layers.Dropout(0.2)(model_final_conv2_mp)

# CNN 4 - merged
model_final_conv3 = layers.Conv2D(128, (3, 7),
示例#12
0
def add_layer(inputs, **kwargs):
    """
    Simple wrapper around add layer for convenience
    """
    return layers.Add(**kwargs)(inputs)
def Smi2Smi():

    #product
    l_in = layers.Input(shape=(None, ))
    l_mask = layers.Input(shape=(None, ))

    #reagents
    l_dec = layers.Input(shape=(None, ))
    l_dmask = layers.Input(shape=(None, ))

    #positional encodings for product and reagents, respectively
    l_pos = PositionLayer(EMBEDDING_SIZE)(l_mask)
    l_dpos = PositionLayer(EMBEDDING_SIZE)(l_dmask)

    l_emask = MaskLayerRight()([l_dmask, l_mask])
    l_right_mask = MaskLayerTriangular()(l_dmask)
    l_left_mask = MaskLayerLeft()(l_mask)

    #encoder
    l_voc = layers.Embedding(input_dim=vocab_size,
                             output_dim=EMBEDDING_SIZE,
                             input_length=None)

    l_embed = layers.Add()([l_voc(l_in), l_pos])
    l_embed = layers.Dropout(rate=0.1)(l_embed)

    for layer in range(n_block):

        #self attention
        l_o = [
            SelfLayer(EMBEDDING_SIZE,
                      KEY_SIZE)([l_embed, l_embed, l_embed, l_left_mask])
            for i in range(n_self)
        ]

        l_con = layers.Concatenate()(l_o)
        l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con)
        l_drop = layers.Dropout(rate=0.1)(l_dense)
        l_add = layers.Add()([l_drop, l_embed])
        l_att = LayerNormalization()(l_add)

        #position-wise
        l_c1 = layers.Conv1D(N_HIDDEN, 1, activation='relu')(l_att)
        l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1)(l_c1)
        l_drop = layers.Dropout(rate=0.1)(l_c2)
        l_ff = layers.Add()([l_att, l_drop])
        l_embed = LayerNormalization()(l_ff)

    #bottleneck
    l_encoder = l_embed

    l_embed = layers.Add()([l_voc(l_dec), l_dpos])
    l_embed = layers.Dropout(rate=0.1)(l_embed)

    for layer in range(n_block):

        #self attention
        l_o = [
            SelfLayer(EMBEDDING_SIZE,
                      KEY_SIZE)([l_embed, l_embed, l_embed, l_right_mask])
            for i in range(n_self)
        ]

        l_con = layers.Concatenate()(l_o)
        l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con)
        l_drop = layers.Dropout(rate=0.1)(l_dense)
        l_add = layers.Add()([l_drop, l_embed])
        l_att = LayerNormalization()(l_add)

        #attention to the encoder
        l_o = [
            SelfLayer(EMBEDDING_SIZE,
                      KEY_SIZE)([l_att, l_encoder, l_encoder, l_emask])
            for i in range(n_self)
        ]
        l_con = layers.Concatenate()(l_o)
        l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con)
        l_drop = layers.Dropout(rate=0.1)(l_dense)
        l_add = layers.Add()([l_drop, l_att])
        l_att = LayerNormalization()(l_add)

        #position-wise
        l_c1 = layers.Conv1D(N_HIDDEN, 1, activation='relu')(l_att)
        l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1)(l_c1)
        l_drop = layers.Dropout(rate=0.1)(l_c2)
        l_ff = layers.Add()([l_att, l_drop])
        l_embed = LayerNormalization()(l_ff)

    l_out = layers.TimeDistributed(layers.Dense(vocab_size,
                                                use_bias=False))(l_embed)

    mdl = tf.keras.Model([l_in, l_mask, l_dec, l_dmask], l_out)

    def masked_loss(y_true, y_pred):
        loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_true,
                                                          logits=y_pred)
        mask = tf.cast(tf.not_equal(tf.reduce_sum(y_true, -1), 0), 'float32')
        loss = tf.reduce_sum(loss * mask, -1) / tf.reduce_sum(mask, -1)
        loss = K.mean(loss)
        return loss

    def masked_acc(y_true, y_pred):
        mask = tf.cast(tf.not_equal(tf.reduce_sum(y_true, -1), 0), 'float32')
        eq = K.cast(
            K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)),
            'float32')
        eq = tf.reduce_sum(eq * mask, -1) / tf.reduce_sum(mask, -1)
        eq = K.mean(eq)
        return eq

    mdl.compile(optimizer='adam',
                loss=masked_loss,
                metrics=['accuracy', masked_acc])

    mdl_enc = tf.keras.Model([l_in, l_mask], l_encoder)
    mdl_enc.compile(optimizer="adam", loss="categorical_crossentropy")

    #mdl.summary();

    return mdl, mdl_enc
def buildNetwork():

    unfreeze = False

    l_in = layers.Input(shape=(None, ))
    l_mask = layers.Input(shape=(None, ))

    l_ymask = []
    for i in range(len(props)):
        l_ymask.append(layers.Input(shape=(1, )))

    #transformer part
    #positional encodings for product and reagents, respectively
    l_pos = PositionLayer(EMBEDDING_SIZE)(l_mask)
    l_left_mask = MaskLayerLeft()(l_mask)

    #encoder
    l_voc = layers.Embedding(input_dim=vocab_size,
                             output_dim=EMBEDDING_SIZE,
                             input_length=None,
                             trainable=unfreeze)
    l_embed = layers.Add()([l_voc(l_in), l_pos])

    for layer in range(n_block):

        #self attention
        l_o = [
            SelfLayer(EMBEDDING_SIZE, KEY_SIZE, trainable=unfreeze)(
                [l_embed, l_embed, l_embed, l_left_mask])
            for i in range(n_self)
        ]

        l_con = layers.Concatenate()(l_o)
        l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE,
                                                      trainable=unfreeze),
                                         trainable=unfreeze)(l_con)
        if unfreeze == True: l_dense = layers.Dropout(rate=0.1)(l_dense)
        l_add = layers.Add()([l_dense, l_embed])
        l_att = LayerNormalization(trainable=unfreeze)(l_add)

        #position-wise
        l_c1 = layers.Conv1D(N_HIDDEN,
                             1,
                             activation='relu',
                             trainable=unfreeze)(l_att)
        l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1, trainable=unfreeze)(l_c1)
        if unfreeze == True: l_c2 = layers.Dropout(rate=0.1)(l_c2)
        l_ff = layers.Add()([l_att, l_c2])
        l_embed = LayerNormalization(trainable=unfreeze)(l_ff)

    #end of Transformer's part
    l_encoder = l_embed

    #text-cnn part
    #https://github.com/deepchem/deepchem/blob/b7a6d3d759145d238eb8abaf76183e9dbd7b683c/deepchem/models/tensorgraph/models/text_cnn.py

    l_in2 = layers.Input(shape=(None, EMBEDDING_SIZE))

    kernel_sizes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20]
    num_filters = [100, 200, 200, 200, 200, 100, 100, 100, 100, 100, 160, 160]

    l_pool = []
    for i in range(len(kernel_sizes)):
        l_conv = layers.Conv1D(num_filters[i],
                               kernel_size=kernel_sizes[i],
                               padding='valid',
                               kernel_initializer='normal',
                               activation='relu')(l_in2)
        l_maxpool = layers.Lambda(lambda x: tf.reduce_max(x, axis=1))(l_conv)
        l_pool.append(l_maxpool)

    l_cnn = layers.Concatenate(axis=1)(l_pool)
    l_cnn_drop = layers.Dropout(rate=0.25)(l_cnn)

    #dense part
    l_dense = layers.Dense(N_HIDDEN_CNN, activation='relu')(l_cnn_drop)

    #https://github.com/ParikhKadam/Highway-Layer-Keras
    transform_gate = layers.Dense(
        units=N_HIDDEN_CNN,
        activation="sigmoid",
        bias_initializer=tf.keras.initializers.Constant(-1))(l_dense)

    carry_gate = layers.Lambda(lambda x: 1.0 - x,
                               output_shape=(N_HIDDEN_CNN, ))(transform_gate)
    transformed_data = layers.Dense(units=N_HIDDEN_CNN,
                                    activation="relu")(l_dense)
    transformed_gated = layers.Multiply()([transform_gate, transformed_data])
    identity_gated = layers.Multiply()([carry_gate, l_dense])

    l_highway = layers.Add()([transformed_gated, identity_gated])

    #Because of multitask we have here a few different outputs and a custom loss.

    def mse_loss(prop):
        def loss(y_true, y_pred):
            y2 = y_true * l_ymask[prop] + y_pred * (1 - l_ymask[prop])
            return tf.keras.losses.mse(y2, y_pred)

        return loss

    def binary_loss(prop):
        def loss(y_true, y_pred):
            y_pred = tf.clip_by_value(y_pred, K.epsilon(), 1.0 - K.epsilon())
            r = y_true * K.log(y_pred) + (1.0 - y_true) * K.log(1.0 - y_pred)
            r = -tf.reduce_mean(r * l_ymask[prop])
            return r

        return loss

    l_out = []
    losses = []
    for prop in props:
        if props[prop][2] == "regression":
            l_out.append(
                layers.Dense(1,
                             activation='linear',
                             name="Regression-" + props[prop][1])(l_highway))
            losses.append(mse_loss(prop))
        else:
            l_out.append(
                layers.Dense(1,
                             activation='sigmoid',
                             name="Classification-" +
                             props[prop][1])(l_highway))
            losses.append(binary_loss(prop))

    l_input = [l_in2]
    l_input.extend(l_ymask)

    mdl = tf.keras.Model(l_input, l_out)
    mdl.compile(optimizer='adam', loss=losses)

    #mdl.summary();

    K.set_value(mdl.optimizer.lr, 1.0e-4)

    #so far we do not train the encoder part of the model.
    encoder = tf.keras.Model([l_in, l_mask], l_encoder)
    encoder.compile(optimizer='adam', loss='mse')
    encoder.set_weights(np.load("embeddings.npy", allow_pickle=True))

    #encoder.summary();

    return mdl, encoder
示例#15
0
def fpn(encoder, input_shape, num_heads=1):

    image_input = layers.Input(shape=input_shape)
    # mask = layers.Input(shape=input_shape)

    image_input, levels = encoder(image_input)
    [f0, f1, f2, f3, f4] = levels
    # paper says: f0 should be disregarded due to high memory footprint.

    # ***** FPN *****
    # 7x7 to 14x14
    P4 = f4
    f4_prime = layers.Conv2D(256, (1, 1))(P4)
    x = layers.UpSampling2D((2, 2))(f4_prime)
    f3_prime = layers.Conv2D(256, (1, 1))(f3)
    x = layers.Add()([x, f3_prime])
    x = layers.Conv2D(256, (3, 3), padding='same')(x)
    x = layers.BatchNormalization(axis=-1)(x)
    P3 = layers.Activation("relu")(x)

    # 14x14 to 28x28
    x = layers.UpSampling2D((2, 2))(P3)
    f2_prime = layers.Conv2D(256, (1, 1))(f2)
    x = layers.Add()([x, f2_prime])
    x = layers.Conv2D(256, (3, 3), padding='same')(x)
    x = layers.BatchNormalization(axis=-1)(x)
    P2 = layers.Activation("relu")(x)

    # 28x28 to 56x56
    x = layers.UpSampling2D((2, 2))(P2)
    f1_prime = layers.Conv2D(256, (1, 1))(f1)
    x = layers.Add()([x, f1_prime])
    x = layers.Conv2D(256, (3, 3), padding='same')(x)
    x = layers.BatchNormalization(axis=-1)(x)
    P1 = layers.Activation("relu")(x)

    # ***** UPSAMPLE *****
    # upsample to 1/4 org. image
    # P4 - 1
    up_p4 = layers.Conv2D(128, (3, 3), padding='same')(P4)
    up_p4 = layers.BatchNormalization(axis=-1)(
        up_p4)  # tfa.layers.GroupNormalization()
    up_p4 = layers.Activation("relu")(up_p4)
    up_p4 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p4)

    # P4 - 2
    up_p4 = layers.Conv2D(128, (3, 3), padding='same')(up_p4)
    up_p4 = layers.BatchNormalization(axis=-1)(
        up_p4)  # tfa.layers.GroupNormalization()
    up_p4 = layers.Activation("relu")(up_p4)
    up_p4 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p4)

    # P4 - 3
    up_p4 = layers.Conv2D(128, (3, 3), padding='same')(up_p4)
    up_p4 = layers.BatchNormalization(axis=-1)(
        up_p4)  # tfa.layers.GroupNormalization()
    up_p4 = layers.Activation("relu")(up_p4)
    up_p4 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p4)

    # P3 - 1
    up_p3 = layers.Conv2D(128, (3, 3), padding='same')(P3)
    up_p3 = layers.BatchNormalization(axis=-1)(
        up_p3)  # tfa.layers.GroupNormalization()
    up_p3 = layers.Activation("relu")(up_p3)
    up_p3 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p3)

    # P3 - 2
    up_p3 = layers.Conv2D(128, (3, 3), padding='same')(up_p3)
    up_p3 = layers.BatchNormalization(axis=-1)(
        up_p3)  # tfa.layers.GroupNormalization()
    up_p3 = layers.Activation("relu")(up_p3)
    up_p3 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p3)

    # P2 - 1
    up_p2 = layers.Conv2D(128, (3, 3), padding='same')(P2)
    up_p2 = layers.BatchNormalization(axis=-1)(
        up_p2)  # tfa.layers.GroupNormalization()
    up_p2 = layers.Activation("relu")(up_p2)
    up_p2 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p2)

    # P2
    up_p1 = layers.Conv2D(128, (3, 3), padding='same')(P1)
    up_p1 = layers.BatchNormalization(axis=-1)(
        up_p1)  # tfa.layers.GroupNormalization()
    up_p1 = layers.Activation("relu")(up_p1)

    # SUM ELEMENT WISE
    y = layers.Add()([up_p4, up_p3, up_p2, up_p1])
    y = layers.BatchNormalization(axis=-1)(
        y)  # tfa.layers.GroupNormalization()
    y = layers.UpSampling2D((4, 4), interpolation="bilinear")(y)
    y = layers.Conv2D(64, (3, 3), padding='same')(y)

    y = layers.BatchNormalization(axis=-1)(
        y)  # tfa.layers.GroupNormalization()
    y = layers.Activation("relu")(y)

    # REGRESSION
    stacked_many_heads_output = None
    for i in range(num_heads):  # get multiple heads working
        output_reg = layers.Conv2D(3, (3, 3), padding='same')(y)
        # output_masked_reg = layers.Multiply()([output_reg, mask])

        if stacked_many_heads_output == None:
            stacked_many_heads_output = output_reg
        else:
            stacked_many_heads_output = layers.Concatenate(axis=-1)(
                [stacked_many_heads_output, output_reg])

    # model = Model(inputs=[image_input, mask], outputs=stacked_many_heads_output)
    model = Model(inputs=image_input, outputs=stacked_many_heads_output)

    #     model.summary()

    return model
示例#16
0
def block1(x,
           filters,
           kernel_size=3,
           stride=1,
           conv_shortcut=True,
           name=None,
           norm_use="bn",
           use_deformable=False):
    """A residual block.
    # Arguments
        x: input tensor.
        filters: integer, filters of the bottleneck layer.
        kernel_size: default 3, kernel size of the bottleneck layer.
        stride: default 1, stride of the first layer.
        conv_shortcut: default True, use convolution shortcut if True,
            otherwise identity shortcut.
        name: string, block label.
    # Returns
        Output tensor for the residual block.
    """
    bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1

    if conv_shortcut is True:
        shortcut = layers.Conv2D(4 * filters,
                                 1,
                                 strides=stride,
                                 kernel_initializer='he_normal',
                                 name=name + '_0_conv')(x)
        shortcut = normalize_layer(shortcut,
                                   norm_use=norm_use,
                                   name=name + '_0_')
        #shortcut = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_0_bn')(shortcut)
    else:
        shortcut = x

    x = layers.Conv2D(filters,
                      1,
                      strides=stride,
                      name=name + '_1_conv',
                      kernel_initializer='he_normal')(x)
    #x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_1_bn')(x)
    x = normalize_layer(x, norm_use=norm_use, name=name + '_1_')
    x = layers.Activation('relu', name=name + '_1_relu')(x)

    if use_deformable:
        x = DeformableConvLayer(filters=filters,
                                kernel_size=kernel_size,
                                strides=(1, 1),
                                padding="same",
                                num_deformable_group=filters // 8)(x)
    else:
        x = layers.Conv2D(filters,
                          kernel_size,
                          padding='SAME',
                          kernel_initializer='he_normal',
                          name=name + '_2_conv')(x)
    x = normalize_layer(x, norm_use=norm_use, name=name + '_2_')
    #x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_2_bn')(x)
    x = layers.Activation('relu', name=name + '_2_relu')(x)

    x = layers.Conv2D(4 * filters,
                      1,
                      name=name + '_3_conv',
                      kernel_initializer='he_normal')(x)
    x = normalize_layer(x, norm_use=norm_use, name=name + '_3_')
    #x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_3_bn')(x)

    x = layers.Add(name=name + '_add')([shortcut, x])
    x = layers.Activation('relu', name=name + '_out')(x)
    return x
示例#17
0
def transfer_model_7x7_14x14(backbone_model,
                             input_shape,
                             dims_list,
                             num_aspect_ratios,
                             wt_decay,
                             model_name='transfer-objdet-model-7x7-14x14'):
    inputs = keras.Input(shape=input_shape)
    intermediate_layer_model = keras.Model(
        inputs=backbone_model.input,
        outputs=backbone_model.get_layer('conv4_block6_out').output)
    intermediate_output = intermediate_layer_model(inputs)  #14
    backbone_output = backbone_model(inputs)  #7

    x = layers.Conv2D(512, 1, padding='same',
                      kernel_regularizer=l2(wt_decay))(backbone_output)  #7
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(1024, 3, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #7
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(512, 1, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #7
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(1024, 3, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #7
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(512, 1, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #7
    x = layers.BatchNormalization()(x)
    upsample = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(2048, 3, padding='same',
                      kernel_regularizer=l2(wt_decay))(upsample)  #7
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    tens_7x7 = layers.Add()([x, backbone_output])

    x = layers.Conv2D(256, 1, padding='same',
                      kernel_regularizer=l2(wt_decay))(upsample)  #7
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2DTranspose(512, 5, strides=(2, 2), padding='same')(x)  #14
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)

    x = layers.Concatenate()([x, intermediate_output])

    x = layers.Conv2D(256, 1, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #14
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(512, 3, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #14
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(256, 1, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #14
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(512, 3, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #14
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(256, 1, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #14
    x = layers.BatchNormalization()(x)
    x = layers.LeakyReLU(0.01)(x)
    x = layers.Conv2D(512, 3, padding='same',
                      kernel_regularizer=l2(wt_decay))(x)  #14
    x = layers.BatchNormalization()(x)
    tens_14x14 = layers.LeakyReLU(0.01)(x)

    dim_tensor_map = {'7x7': tens_7x7, '14x14': tens_14x14}

    # For each dimension, construct a predictions tensor. Accumulate them into a dictionary for keras to understand multiple labels.
    preds_dict = {}
    for dims in dims_list:
        dimkey = '{}x{}'.format(*dims)
        tens = dim_tensor_map[dimkey]
        ar_preds = []
        for _ in range(num_aspect_ratios):
            objectness_preds = layers.Conv2D(
                1, 1, kernel_regularizer=l2(wt_decay))(tens)
            class_preds = layers.Conv2D(len(cat_list),
                                        1,
                                        kernel_regularizer=l2(wt_decay))(tens)
            bbox_preds = layers.Conv2D(4, 1,
                                       kernel_regularizer=l2(wt_decay))(tens)
            ar_preds.append(layers.Concatenate()(
                [objectness_preds, class_preds, bbox_preds]))

        if num_aspect_ratios > 1:
            predictions = layers.Concatenate()(ar_preds)
        elif num_aspect_ratios == 1:
            predictions = ar_preds[0]

        predictions = layers.Reshape(
            (*dims, num_aspect_ratios, 5 + len(cat_list)),
            name=dimkey)(predictions)
        preds_dict[dimkey] = predictions

    model = keras.Model(inputs, preds_dict, name=model_name)

    model.compile(optimizer=tf.keras.optimizers.Adam(1e-5), loss=custom_loss)
    return model
Cname = [None] * 6
C = [None] * 6
for i in range(2, 6):
    substr = 'conv' + str(i)
    res = [x for x in layernames if substr in x]
    Cname[i] = res[-1]
    C[i] = basemodel.get_layer(name=res[-1])

P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
               name='fpn_c5p5')(C[5].output)

T1 = KL.UpSampling2D(size=(2, 2), name='fpn_p5upsampled')(P5)
T2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
               name='fpn_c4p4')(C[4].output)
P4 = KL.Add(name='fpn_p4add')([T1, T2])

T1 = KL.UpSampling2D(size=(2, 2), name='fpn_p4upsampled')(P4)
T2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
               name='fpn_c3p3')(C[3].output)
P3 = KL.Add(name='fpn_p3add')([T1, T2])

T1 = KL.UpSampling2D(size=(2, 2), name='fpn_p4upsampled')(P4)
T2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
               name='fpn_c3p3')(C[3].output)
P3 = KL.Add(name='fpn_p3add')([T1, T2])

T1 = KL.UpSampling2D(size=(2, 2), name='fpn_p3upsampled')(P3)
T2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
               name='fpn_c2p2')(C[2].output)
P2 = KL.Add(name='fpn_p2add')([T1, T2])
示例#19
0
oof = np.zeros(len(data))
train_preds = np.zeros(len(data))
models = []

cv = KFold(n_splits=5, random_state=6, shuffle=True)

for fold_, (train_idx, val_idx) in enumerate(cv.split(data, targets), 1):
    print(f'Training with fold {fold_} started.')
    input_layer = layers.Input(len(features))
    layer_0 = layers.Dense(1000)(input_layer)
    layer_1 = layers.Dense(1000, activation='sigmoid')(layer_0)
    layer_2 = layers.Dense(1000, activation='relu')(layer_0)
    layer_3 = layers.Dense(500, activation='sigmoid')(layer_1)
    layer_4 = layers.Dense(500, activation='relu')(layer_2)
    layer_5 = layers.Add()([layer_3, layer_4])
    layer_6 = layers.Dense(300, activation='relu')(layer_5)
    out_layer = layers.Dense(2, activation='softmax')(layer_6)
    model = keras.Model(inputs=input_layer, outputs=out_layer)

    model.compile(optimizer=keras.optimizers.Adam(0.0001),
                  loss=keras.losses.categorical_crossentropy,
                  metrics=[keras.metrics.AUC(name='auc'), 'acc'])

    model.summary()

    checkpoint = keras.callbacks.ModelCheckpoint(f"{fold_}model.hdf5",
                                                 monitor='val_auc',
                                                 verbose=0,
                                                 save_best_only=True,
                                                 mode='max')
示例#20
0
 def build(self, input_shape):
     super().build(input_shape)
     dense_units = self.units * self.num_heads  # N*H
     self.query_layer = layers.Dense(self.num_heads, name='query')
     self.value_layer = layers.Dense(dense_units, name='value')
     self.add = layers.Add()
示例#21
0
    def __init__(self, in_channels, out_channels, stride, dropout, name, trainable):
        super(BasicBlock, self).__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.stride = stride
        self.dropout = dropout
        # name = name
        self.trainable = trainable

        self.bn1 = layers.BatchNormalization(
            momentum=0.999,
            trainable=self.trainable,
            name=name+'_bn1'
        )
        self.relu1 = layers.LeakyReLU(alpha=0.1)
        # self.relu1 = keras.activations.relu
        self.conv1 = layers.Conv2D(
            filters=self.out_channels,
            kernel_size=3,
            strides=self.stride,
            padding='same',
            use_bias=False,
            kernel_initializer='he_normal',
            # kernel_regularizer=regularizers.l2(config.WEIGHT_DECAY),
            trainable=self.trainable,
            name=name+'_conv1',
        )
        self.bn2 = layers.BatchNormalization(
            momentum=0.999,
            trainable=self.trainable,
            name=name+'_bn2'
        )
        self.relu2 = layers.LeakyReLU(alpha=0.1)
        # self.relu2 = keras.activations.relu
        self.drop_layer = layers.Dropout(
            rate=self.dropout,
            trainable=self.trainable,
            name=name+'_dropout',
        )
        self.conv2 = layers.Conv2D(
            filters=self.out_channels,
            kernel_size=3,
            strides=1,
            padding='same',
            use_bias=False,
            kernel_initializer='he_normal',
            # kernel_regularizer=regularizers.l2(config.WEIGHT_DECAY),
            trainable=self.trainable,
            name=name+'_conv2',
        )
        if self.stride != 1 or self.in_channels != self.out_channels:
            self.short_cut = layers.Conv2D(
                filters=self.out_channels,
                kernel_size=1,
                strides=self.stride,
                padding='same',
                use_bias=False,
                kernel_initializer='he_normal',
                # kernel_regularizer=regularizers.l2(config.WEIGHT_DECAY),
                trainable=self.trainable,
                name=name+'_shortcut'
            )
        self.add = layers.Add(name=name+'_add')
示例#22
0
文件: BAFA.py 项目: junhahyung/paf
        def boundary_generator(input_img):
            # per image standardization
            img = layers.Lambda(
                lambda x: tf.image.per_image_standardization(x))(input_img)

            # downsample a bit
            net = layers.Conv2D(
                filters=16,
                kernel_size=3,
                strides=2,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(img)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            net = layers.Dropout(rate=self.dropout_rate)(net)
            # (48 48 16)
            net = layers.Conv2D(
                filters=32,
                kernel_size=3,
                strides=2,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            net = layers.Dropout(rate=self.dropout_rate)(net)
            # (24 24 32)

            shortcut = net
            net = layers.DepthwiseConv2D(
                kernel_size=3,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Conv2D(
                64,
                kernel_size=1,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            # net = layers.Add()([shortcut, net])
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            net = layers.Dropout(rate=self.dropout_rate)(net)

            shortcut = net
            net = layers.DepthwiseConv2D(
                kernel_size=3,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Conv2D(
                128,
                kernel_size=1,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            # net = layers.Add()([shortcut, net])
            # net = layers.ReLU()(net)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            net = layers.Dropout(rate=self.dropout_rate)(net)

            shortcut = net
            net = layers.DepthwiseConv2D(
                kernel_size=3,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Conv2D(
                128,
                kernel_size=1,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Add()([shortcut, net])
            net = layers.ReLU()(net)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            net = layers.Dropout(rate=self.dropout_rate)(net)

            shortcut = net
            net = layers.DepthwiseConv2D(
                kernel_size=3,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Conv2D(
                128,
                kernel_size=1,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Add()([shortcut, net])
            net = layers.ReLU()(net)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            net = layers.Dropout(rate=self.dropout_rate)(net)

            shortcut = net
            net = layers.DepthwiseConv2D(
                kernel_size=3,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Conv2D(
                128,
                kernel_size=1,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Add()([shortcut, net])
            net = layers.ReLU()(net)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            net = layers.Dropout(rate=self.dropout_rate)(net)

            shortcut = net
            net = layers.DepthwiseConv2D(
                kernel_size=3,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Conv2D(
                128,
                kernel_size=1,
                strides=1,
                padding='same',
                kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            net = layers.Add()([shortcut, net])
            # net = layers.ReLU()(net)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            net = layers.Dropout(rate=self.dropout_rate)(net)

            # shortcut = net
            # net = layers.DepthwiseConv2D(kernel_size=3, strides=1, padding='same',
            #                              kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            # net = layers.Conv2D(128, kernel_size=1, strides=1, padding='same',
            #                     kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            # net = layers.Add()([shortcut, net])
            # net = layers.ReLU()(net)
            # net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1, 2])(net)
            # net = layers.Dropout(rate=self.dropout_rate)(net)
            #
            # shortcut = net
            # net = layers.DepthwiseConv2D(kernel_size=3, strides=1, padding='same',
            #                              kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            # net = layers.Conv2D(128, kernel_size=1, strides=1, padding='same',
            #                     kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
            # net = layers.Add()([shortcut, net])
            # # net = layers.ReLU()(net)
            # net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1, 2])(net)
            # net = layers.Dropout(rate=self.dropout_rate)(net)

            # transposes
            net = layers.Conv2DTranspose(filters=64,
                                         kernel_size=3,
                                         strides=2,
                                         padding='same',
                                         output_padding=1,
                                         use_bias=False)(net)
            net = layers.BatchNormalization(momentum=0.99)(net)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)
            # net = layers.ReLU()(net)

            net = layers.Conv2DTranspose(filters=32,
                                         kernel_size=3,
                                         strides=2,
                                         padding='same',
                                         output_padding=1,
                                         use_bias=False)(net)
            net = layers.BatchNormalization(momentum=0.99)(net)
            net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
                                                                       2])(net)

            out = layers.Conv2D(self.num_boundaries,
                                kernel_size=1,
                                strides=1,
                                padding='same')(net)
            # out = kernel_convolution(out)

            return out
示例#23
0
def ResNetModel(Img, ImageSize, MiniBatchSize):
	"""
	Inputs: 
	Img is a MiniBatch of the current image
	ImageSize - Size of the Image
	Outputs:
	prLogits - logits output of the network
	prSoftMax - softmax output of the network
	"""
	
	#############################
	# Fill your network here!
	#############################
	training = tf.compat.v1.placeholder_with_default(False, shape=(), name='training')

	#Batch_norm 1
	batch_norm1 = tf.layers.batch_normalization(Img, training=training, momentum=0.9)

	# RelU layer 1
	layer_relu1 = new_relu_layer(batch_norm1, name="relu1")
	
	# Convolutional Layer 1
	layer_conv1, weights_conv1 = new_conv_layer(layer_relu1, num_input_channels=3, filter_size=5, num_filters=20, name ="conv1")

	#Batch_norm 2
	batch_norm2 = tf.layers.batch_normalization(layer_conv1, training=training, momentum=0.9)

	# RelU layer 2
	layer_relu2 = new_relu_layer(batch_norm2, name="relu2")

	# Convolutional Layer 2
	layer_conv2, weights_conv2 = new_conv_layer(input=layer_relu2, num_input_channels=20, filter_size=3, num_filters=40, name= "conv2")

	#Skip-Connection 1
	#paddings = tf.constant([[0, 0], [8, 8], [8, 8], [0, 0]])
	#batch_norm1pad = tf.pad(batch_norm1, paddings, "CONSTANT")
	unit_conv1, weights_unit_conv1 = new_conv_layer(input=Img, num_input_channels=3, filter_size=1, num_filters=40, name ="unitconv1")
	skip_conn1 = layers.Add()([unit_conv1, layer_conv2])

	#Batch_norm 3
	batch_norm3 = tf.layers.batch_normalization(skip_conn1, training=training, momentum=0.9)

	# RelU layer 3
	layer_relu3 = new_relu_layer(batch_norm3, name="relu3")

	# Convolutional Layer 3
	layer_conv3, weights_conv3 = new_conv_layer(input=layer_relu3, num_input_channels=40, filter_size=3, num_filters=80, name= "conv3")

	#Batch_norm 4
	batch_norm4 = tf.layers.batch_normalization(layer_conv3, training=training, momentum=0.9)

	# RelU layer 4
	layer_relu4 = new_relu_layer(batch_norm4, name="relu3")

	# Convolutional Layer 4
	layer_conv4, weights_conv4 = new_conv_layer(input=layer_relu4, num_input_channels=80, filter_size=3, num_filters=100, name= "conv4")

	#Skip-Connection 2
	unit_conv2, weights_unit_conv2 = new_conv_layer(input=skip_conn1, num_input_channels=40, filter_size=1, num_filters=100, name ="unitconv2")
	skip_conn2 = layers.Add()([unit_conv2, layer_conv4])

	#Batch_norm 5
	batch_norm5 = tf.layers.batch_normalization(skip_conn2, training=training, momentum=0.9)

	# RelU layer 5
	layer_relu5 = new_relu_layer(batch_norm5, name="relu5")

	# Pooling Layer 1
	layer_pool1 = new_pool_layer(layer_relu5, name="pool1")

	# Flatten Layer
	num_features = layer_pool1.get_shape()[1:4].num_elements()
	layer_flat = tf.reshape(layer_pool1, [-1, num_features])

	# Fully-Connected Layer 1
	layer_fc1 = new_fc_layer(layer_flat, num_inputs=num_features, num_outputs=10, name="fc1")

	# Use Softmax function to normalize the output
	with tf.compat.v1.variable_scope("Softmax"):
		y_pred = tf.nn.softmax(layer_fc1)
		y_pred_cls = tf.argmax(y_pred, axis=1)

	prLogits = layer_fc1
	prSoftMax = y_pred
	
	return prLogits, prSoftMax
示例#24
0
def dumbelXL():

    inL = layers.Input(shape=(256, 256, 2))
    
    conv01 = cl.convBlock(32, 3)(inL)
    conv02 = cl.convBlock(32, 3)(conv01)
    sConv01 = cl.convBlock(64, 3, 2)(conv02)
    
    conv03 = cl.convBlock(64, 3)(sConv01)
    conv04 = cl.convBlock(64, 3)(conv03)
    sConv02 = cl.convBlock(128, 3, 2)(conv04)
    
    conv05 = cl.convBlock(128, 3)(sConv02)
    conv06 = cl.convBlock(128, 3)(conv05)
    sConv03 = cl.convBlock(256, 3, 2)(conv06)
    
    conv07 = cl.convBlock(256, 3)(sConv03)
    conv08 = cl.convBlock(256, 3)(conv07)
    sConv04 = cl.convBlock(512, 3, 2)(conv08)



    
    res01 = cl.resBlock(512, 3)(sConv04)
    res02 = cl.resBlock(512, 3)(res01)
    res03 = cl.resBlock(512, 3)(res02)
    res04 = cl.resBlock(512, 3)(res03)
    res05 = cl.resBlock(512, 3)(res04)
    res06 = cl.resBlock(512, 3)(res05)
    
    

    tConv01 = cl.convTransBlock(256, 3, 2)(res06)
    add01 = layers.Add()([tConv01, conv08])
    conv09 = cl.convBlock(256, 3)(add01)
    conv10 = cl.convBlock(256, 3)(conv09)
    
    tConv02 = cl.convTransBlock(128, 3, 2)(conv10)
    add02 = layers.Add()([tConv02, conv06])
    conv11 = cl.convBlock(128, 3)(add02)
    conv12 = cl.convBlock(128, 3)(conv11)
    
    

    tConv03 = cl.convTransBlock(64, 3, 2)(conv12)
    add03 = layers.Add()([tConv03, conv04])
    conv13 = cl.convBlock(64, 3)(add03)
    conv14 = cl.convBlock(64, 3)(conv13)
    

    tConv04 = cl.convTransBlock(32, 3, 2)(conv14)
    add04 = layers.Add()([tConv04, conv02])
    conv15 = cl.convBlock(32, 3)(add04)
    conv16 = cl.convBlock(32, 3)(conv15)

    synt = layers.Conv2D(2, 3, padding="same", use_bias=False)(conv16)

    
    
    model = models.Model(inputs=[inL], outputs=[synt])
    
    return(model)
示例#25
0
def SVBRDF(num_classes):
    #=============== first layer ==================

    inputs = keras.Input(shape=(256, 256) + (3, ))
    x = layers.LeakyReLU()(inputs)
    GF = layers.AveragePooling2D((x.shape[1], x.shape[1]))(x)
    GF = layers.Dense(128)(GF)
    GF = layers.Activation('selu')(GF)
    x = layers.SeparableConv2D(128, 4, 2, padding="same")(x)
    #previous_block_activation = x  # Set aside residual

    #========== define filters for unet ===================

    downfilters = np.array([128, 256, 512, 512, 512, 512, 512, 512])
    Upfilters = np.flip(np.copy(downfilters))
    downfilters = np.delete(downfilters, 0)
    #print(downfilters)
    prefilter = 128

    #===================== upsampling =======================

    for filters in downfilters:
        #print(x.shape)
        #print(filters)
        GFdown = layers.AveragePooling2D((x.shape[1], x.shape[1]))(x)
        GFup = layers.Dense(prefilter)(x)
        GF = layers.Concatenate()([GF, GFdown])
        GF = layers.Dense(filters)(GF)
        GF = layers.Activation('selu')(GF)

        x = layers.Add()([x, GFup])
        x = layers.LeakyReLU()(x)
        x = layers.SeparableConv2D(filters, 4, 2, padding="same")(x)
        prefilter = filters
        #x = layers.BatchNormalization()(x)
        # Project residual

        #residual = layers.Conv2D(filters, 4,2, padding="same")(previous_block_activation)
        #x = layers.add([x, residual])  # Add back residual
        #previous_block_activation = x  # Set aside next residual

    #====================== downsampling ============================

    for filters in Upfilters:

        GFdown = layers.AveragePooling2D((x.shape[1], x.shape[1]))(x)
        GFup = layers.Dense(prefilter)(x)
        GF = layers.Concatenate()([GF, GFdown])
        GF = layers.Dense(filters)(GF)
        GF = layers.Activation('selu')(GF)

        x = layers.Add()([x, GFup])
        x = layers.LeakyReLU()(x)
        x = layers.Conv2DTranspose(filters, 4, 2, padding="same")(x)
        prefilter = filters

        # Project residual
        #residual = layers.UpSampling2D(2)(previous_block_activation)
        #residual = layers.Conv2D(filters, 4, padding="same")(residual)
        #x = layers.add([x, residual])  # Add back residual
        #previous_block_activation = x  # Set aside next residual

    #====================== last connection =====================

    GFup = layers.Dense(prefilter)(x)
    x = layers.Add()([x, GFup])
    outputs = layers.Conv2D(num_classes,
                            3,
                            activation="softmax",
                            padding="same")(x)
    model = keras.Model(inputs, outputs)
    return model
示例#26
0
def get_generator_model(name="generator",
                        num_blocks=20,
                        num_filters=32,
                        input_dtypes=None,
                        output_dtype=None):
    """
    Create generator model.

    Inputs:
    - input_image (N x H x W x 3) - input frame
    - pre_warp (N x H * 2 x W * 2 x 3) - warped previously generated frame

    Outputs:
    - (N x H * 2 x W * 2 x 3) - upscaled frame

    Parameters
    ----------
    name: str
        Model name
    num_blocks: int
        Number of residual blocks
    num_filters: int
        Number of filters
    input_dtypes : array of str or None
        Data types for input images and warped previous images
    output_dtype : str or None
        Output dtype

    Returns
    -------
    keras.Model
        Model
    """
    images = keras.Input(shape=[None, None, 3],
                         name="input_image",
                         dtype=input_dtypes[0] if input_dtypes else None)
    pre_warp = keras.Input(shape=[None, None, 3],
                           name="pre_warp",
                           dtype=input_dtypes[1] if input_dtypes else None)
    inputs = layers.Concatenate()([
        images,
        layers.Lambda(lambda x: tf.nn.space_to_depth(x, 2))(pre_warp)
    ])
    net = layers.Conv2D(filters=num_filters,
                        kernel_size=3,
                        strides=1,
                        padding="SAME",
                        use_bias=False)(inputs)
    net = layers.BatchNormalization()(net)
    net = layers.LeakyReLU()(net)

    def res_block(x):  # pylint: disable=invalid-name
        shortcut = x
        x = layers.Conv2D(filters=num_filters,
                          kernel_size=3,
                          strides=1,
                          padding="SAME",
                          use_bias=False)(x)
        x = layers.BatchNormalization()(x)
        x = layers.LeakyReLU()(x)
        x = layers.Conv2D(filters=num_filters,
                          kernel_size=3,
                          strides=1,
                          padding="SAME",
                          use_bias=False)(x)
        x = layers.BatchNormalization()(x)
        x = layers.Add()([x, shortcut])
        x = layers.LeakyReLU()(x)
        return x

    for _ in range(num_blocks):
        net = res_block(net)

    net = layers.Conv2DTranspose(filters=num_filters,
                                 kernel_size=3,
                                 strides=2,
                                 padding="SAME")(net)
    net = layers.LeakyReLU()(net)
    net = layers.Conv2D(filters=3, kernel_size=3, strides=1,
                        padding="SAME")(net)
    net = layers.Activation(K.tanh)(net)
    upscaled = UpscaleLayer(dtype="float32")(images)
    output = layers.Add(dtype=output_dtype)([upscaled, net])
    model = keras.Model(inputs=[images, pre_warp], outputs=output, name=name)
    return model
示例#27
0
def create_tabtransformer_classifier(
    num_transformer_blocks,
    num_heads,
    embedding_dims,
    mlp_hidden_units_factors,
    dropout_rate,
    use_column_embedding=False,
):

    # Create model inputs.
    inputs = create_model_inputs()
    # encode features.
    encoded_categorical_feature_list, numerical_feature_list = encode_inputs(
        inputs, embedding_dims)
    # Stack categorical feature embeddings for the Tansformer.
    encoded_categorical_features = tf.stack(encoded_categorical_feature_list,
                                            axis=1)
    # Concatenate numerical features.
    numerical_features = layers.concatenate(numerical_feature_list)

    # Add column embedding to categorical feature embeddings.
    if use_column_embedding:
        num_columns = encoded_categorical_features.shape[1]
        column_embedding = layers.Embedding(input_dim=num_columns,
                                            output_dim=embedding_dims)
        column_indices = tf.range(start=0, limit=num_columns, delta=1)
        encoded_categorical_features = encoded_categorical_features + column_embedding(
            column_indices)

    # Create multiple layers of the Transformer block.
    for block_idx in range(num_transformer_blocks):
        # Create a multi-head attention layer.
        attention_output = layers.MultiHeadAttention(
            num_heads=num_heads,
            key_dim=embedding_dims,
            dropout=dropout_rate,
            name=f"multihead_attention_{block_idx}",
        )(encoded_categorical_features, encoded_categorical_features)
        # Skip connection 1.
        x = layers.Add(name=f"skip_connection1_{block_idx}")(
            [attention_output, encoded_categorical_features])
        # Layer normalization 1.
        x = layers.LayerNormalization(name=f"layer_norm1_{block_idx}",
                                      epsilon=1e-6)(x)
        # Feedforward.
        feedforward_output = create_mlp(
            hidden_units=[embedding_dims],
            dropout_rate=dropout_rate,
            activation=keras.activations.gelu,
            normalization_layer=layers.LayerNormalization(epsilon=1e-6),
            name=f"feedforward_{block_idx}",
        )(x)
        # Skip connection 2.
        x = layers.Add(name=f"skip_connection2_{block_idx}")(
            [feedforward_output, x])
        # Layer normalization 2.
        encoded_categorical_features = layers.LayerNormalization(
            name=f"layer_norm2_{block_idx}", epsilon=1e-6)(x)

    # Flatten the "contextualized" embeddings of the categorical features.
    categorical_features = layers.Flatten()(encoded_categorical_features)
    # Apply layer normalization to the numerical features.
    numerical_features = layers.LayerNormalization(
        epsilon=1e-6)(numerical_features)
    # Prepare the input for the final MLP block.
    features = layers.concatenate([categorical_features, numerical_features])

    # Compute MLP hidden_units.
    mlp_hidden_units = [
        factor * features.shape[-1] for factor in mlp_hidden_units_factors
    ]
    # Create final MLP.
    features = create_mlp(
        hidden_units=mlp_hidden_units,
        dropout_rate=dropout_rate,
        activation=keras.activations.selu,
        normalization_layer=layers.BatchNormalization(),
        name="MLP",
    )(features)

    # Add a sigmoid as a binary classifer.
    outputs = layers.Dense(units=1, activation="sigmoid",
                           name="sigmoid")(features)
    model = keras.Model(inputs=inputs, outputs=outputs)
    return model
示例#28
0
                            name='conv_layer2',
                            padding='same')(conv_layer1)

# add1=layers.Add()([ concat_layer,conv_layer2 ])
conv_layer3 = layers.Conv2D(filters=64,
                            kernel_size=(3, 3),
                            activation='relu',
                            name='conv_layer3',
                            padding='same')(conv_layer2)
conv_layer4 = layers.Conv2D(filters=64,
                            kernel_size=(3, 3),
                            activation='relu',
                            name='conv_layer4',
                            padding='same')(conv_layer3)

add2 = layers.Add()([conv_layer2, conv_layer4])
conv_layer5 = layers.Conv2D(filters=64,
                            kernel_size=(3, 3),
                            activation='relu',
                            name='conv_layer5',
                            padding='same')(add2)
conv_layer6 = layers.Conv2D(filters=64,
                            kernel_size=(3, 3),
                            activation='relu',
                            name='conv_layer6',
                            padding='same')(conv_layer5)

add3 = layers.Add()([conv_layer4, conv_layer6])
conv_layer7 = layers.Conv2D(filters=64,
                            kernel_size=(3, 3),
                            activation='relu',
示例#29
0
def residual_dense_unit(x, units, dropout=0.45):
    _x = dense_unit(x, units, dropout=dropout)
    x = KL.Add()([x, _x])
    # if dropout > 0:
    #     x = KL.Dropout(dropout)(x)
    return x
示例#30
0
    def __init__(self,
                 inshape,
                 pheno_input_shape,
                 nb_unet_features=None,
                 src_feats=1,
                 conv_image_shape=None,
                 conv_size=3,
                 conv_nb_levels=0,
                 conv_nb_features=32,
                 extra_conv_layers=3,
                 use_mean_stream=True,
                 mean_cap=100,
                 templcondsi=False,
                 templcondsi_init=None,
                 **kwargs):
        """ 
        Parameters:
            inshape: Input shape. e.g. (192, 192, 192)
            pheno_input_shape: Pheno data input shape. e.g. (2)
            nb_unet_features: Unet convolutional features. See VxmDense documentation for more information.
            src_feats: Number of source (atlas) features. Default is 1.
            conv_image_shape: Intermediate phenotype image shape. Default is inshape with conv_nb_features.
            conv_size: Atlas generator convolutional kernel size. Default is 3.
            conv_nb_levels: Number of levels in atlas generator unet. Default is 0.
            conv_nb_features: Number of features in atlas generator convolutions. Default is 32.
            extra_conv_layers: Number of extra convolutions after unet in atlas generator. Default is 3.
            use_mean_stream: Return mean stream layer for training. Default is True.
            mean_cap: Cap for mean stream. Default is 100.
            templcondsi: Default is False.
            templcondsi_init: Default is None.
            kwargs: Forwarded to the internal VxmDense model.
        """

        if conv_image_shape is None:
            conv_image_shape = (*inshape, conv_nb_features)

        # build initial dense pheno to image shape model
        pheno_input = KL.Input(pheno_input_shape, name='pheno_input')
        pheno_dense = KL.Dense(np.prod(conv_image_shape),
                               activation='elu')(pheno_input)
        pheno_reshaped = KL.Reshape(conv_image_shape,
                                    name='pheno_reshape')(pheno_dense)
        pheno_init_model = tf.keras.models.Model(pheno_input, pheno_reshaped)

        # build model to decode reshaped pheno
        pheno_decoder_model = ne.models.conv_dec(
            conv_nb_features,
            conv_image_shape,
            conv_nb_levels,
            conv_size,
            nb_labels=conv_nb_features,
            final_pred_activation='linear',
            input_model=pheno_init_model,
            name='atlas_decoder')

        # add extra convolutions
        Conv = getattr(KL, 'Conv%dD' % len(inshape))
        last = pheno_decoder_model.output
        for n in range(extra_conv_layers):
            last = Conv(conv_nb_features,
                        kernel_size=conv_size,
                        padding='same',
                        name='atlas_extra_conv_%d' % n)(last)

        # final convolution to get atlas features
        atlas_gen = Conv(src_feats,
                         kernel_size=3,
                         padding='same',
                         name='atlas_gen',
                         kernel_initializer=KI.RandomNormal(mean=0.0,
                                                            stddev=1e-7),
                         bias_initializer=KI.RandomNormal(mean=0.0,
                                                          stddev=1e-7))(last)

        # image input layers
        atlas_input = tf.keras.Input((*inshape, src_feats), name='atlas_input')
        source_input = tf.keras.Input((*inshape, src_feats),
                                      name='source_input')

        if templcondsi:
            atlas_tensor = KL.Add(name='atlas_tmp')([atlas_input, pout])
            # change first channel to be result from seg with another add layer
            tmp_layer = KL.Lambda(lambda x: K.softmax(x[..., 1:]))(
                atlas_tensor)
            conv_layer = Conv(1,
                              kernel_size=1,
                              padding='same',
                              use_bias=False,
                              name='atlas_gen',
                              kernel_initializer=KI.RandomNormal(mean=0,
                                                                 stddev=1e-5))
            x_img = conv_layer(tmp_layer)
            if templcondsi_init is not None:
                weights = conv_layer.get_weights()
                weights[0] = templcondsi_init.reshape(weights[0].shape)
                conv_layer.set_weights(weights)
            atlas_tensor = KL.Lambda(
                lambda x: K.concatenate([x[0], x[1][..., 1:]]),
                name='atlas')([x_img, atlas_tensor])
        else:
            atlas_tensor = KL.Add(name='atlas')([atlas_input, atlas_gen])

        # build complete pheno to atlas model
        pheno_model = tf.keras.models.Model(
            [pheno_decoder_model.input, atlas_input], atlas_tensor)

        inputs = [pheno_decoder_model.input, atlas_input, source_input]
        warp_input_model = tf.keras.Model(inputs=inputs,
                                          outputs=[atlas_tensor, source_input])

        # warp model
        vxm_model = VxmDense(inshape,
                             nb_unet_features=nb_unet_features,
                             bidir=True,
                             input_model=warp_input_model,
                             **kwargs)

        # extract tensors from stacked model
        y_source = vxm_model.references.y_source
        pos_flow = vxm_model.references.pos_flow
        neg_flow = vxm_model.references.neg_flow

        if use_mean_stream:
            # get mean stream from negative flow
            mean_stream = ne.layers.MeanStream(name='mean_stream',
                                               cap=mean_cap)(neg_flow)
            outputs = [y_source, mean_stream, pos_flow, pos_flow]
        else:
            outputs = [y_source, pos_flow, pos_flow]

        # initialize the keras model
        super().__init__(inputs=inputs, outputs=outputs)