def fusion_block(filters, branches_in, branches_out, activation='relu'):
"""A fusion block will fuse multi-resolution inputs.
A typical fusion block looks like a square box with cells. For example at
stage 3, the fusion block consists 12 cells. Each cell represents a fusion
layer. Every cell whose row < column is a down sampling cell, whose row ==
column is a identity cell, and the rest are up sampling cells.
B1 B2 B3 B4
|----------|----------|----------|----------|
B1 | identity | -> | -> | -> |
|----------|----------|----------|----------|
B2 | <- | identity | -> | -> |
|----------|----------|----------|----------|
B3 | <- | <- | identity | -> |
|----------|----------|----------|----------|
"""
# Construct the fusion layers.
_fusion_grid = []
rows = branches_in
columns = branches_out
for row in range(rows):
_fusion_layers = []
for column in range(columns):
if column == row:
_fusion_layers.append(tf.identity)
elif column > row:
# Down sampling.
_fusion_layers.append(
fusion_layer(filters * pow(2, column), False, activation))
else:
# Up sampling.
_fusion_layers.append(
fusion_layer(filters * pow(2, column), True, activation))
_fusion_grid.append(_fusion_layers)
if len(_fusion_grid) > 1:
_add_layers_group = [layers.Add() for _ in range(branches_out)]
def forward(inputs):
rows = len(_fusion_grid)
columns = len(_fusion_grid[0])
# Every cell in the fusion grid has an output value.
fusion_values = [[None for _ in range(columns)] for _ in range(rows)]
for row in range(rows):
# The down sampling operation excutes from left to right.
for column in range(columns):
# The input will be different for different cells.
if column < row:
# Skip all up samping cells.
continue
elif column == row:
# The input is the branch output.
x = inputs[row]
elif column > row:
# Down sampling, the input is the fusion value of the left cell.
x = fusion_values[row][column - 1]
fusion_values[row][column] = _fusion_grid[row][column](x)
# The upsampling operation excutes in the opposite direction.
for column in reversed(range(columns)):
if column >= row:
# Skip all down samping and identity cells.
continue
x = fusion_values[row][column + 1]
fusion_values[row][column] = _fusion_grid[row][column](x)
# The fused value for each branch.
if rows == 1:
outputs = [fusion_values[0][0], fusion_values[0][1]]
else:
outputs = []
fusion_values = [list(v) for v in zip(*fusion_values)]
for index, values in enumerate(fusion_values):
outputs.append(_add_layers_group[index](values))
return outputs
return forward
def block1(x,
filters,
kernel_size=3,
stride=1,
conv_shortcut=True,
dilation=1,
name=None):
"""A residual block.
# Arguments
x: input tensor.
filters: integer, filters of the bottleneck layer.
kernel_size: default 3, kernel size of the bottleneck layer.
stride: default 1, stride of the first layer.
conv_shortcut: default True, use convolution shortcut if True,
otherwise identity shortcut.
name: string, block label.
# Returns
Output tensor for the residual block.
"""
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
if conv_shortcut is True:
if stride == 1:
shortcut = layers.Conv2D(4 * filters,
1,
strides=stride,
use_bias=False,
name=name + '_0_conv')(x)
shortcut = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + '_0_bn')(shortcut)
else:
shortcut = layers.Conv2D(4 * filters,
3,
strides=stride,
use_bias=False,
name=name + '_0_conv')(x)
shortcut = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + '_0_bn')(shortcut)
else:
shortcut = x
x = layers.Conv2D(filters, 1, use_bias=False, name=name + '_1_conv')(x)
x = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + '_1_bn')(x)
x = layers.Activation('relu', name=name + '_1_relu')(x)
padding = 'SAME' if stride == 1 else 'VALID'
x = layers.Conv2D(filters,
kernel_size,
strides=stride,
padding=padding,
dilation_rate=dilation,
use_bias=False,
name=name + '_2_conv')(x)
x = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + '_2_bn')(x)
x = layers.Activation('relu', name=name + '_2_relu')(x)
x = layers.Conv2D(4 * filters, 1, use_bias=False, name=name + '_3_conv')(x)
x = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + '_3_bn')(x)
x = layers.Add(name=name + '_add')([shortcut, x])
x = layers.Activation('relu', name=name + '_out')(x)
return x
def block3(x,
filters,
kernel_size=3,
stride=1,
groups=32,
conv_shortcut=True,
name=None):
"""A residual block.
Args:
x: input tensor.
filters: integer, filters of the bottleneck layer.
kernel_size: default 3, kernel size of the bottleneck layer.
stride: default 1, stride of the first layer.
groups: default 32, group size for grouped convolution.
conv_shortcut: default True, use convolution shortcut if True,
otherwise identity shortcut.
name: string, block label.
Returns:
Output tensor for the residual block.
"""
bn_axis = 3 if backend.image_data_format() == "channels_last" else 1
if conv_shortcut is True:
shortcut = layers.Conv2D(
(64 // groups) * filters,
1,
strides=stride,
use_bias=False,
name=name + "_0_conv",
)(x)
shortcut = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + "_0_bn")(shortcut)
else:
shortcut = x
x = layers.Conv2D(filters, 1, use_bias=False, name=name + "_1_conv")(x)
x = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + "_1_bn")(x)
x = layers.Activation("relu", name=name + "_1_relu")(x)
c = filters // groups
x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + "_2_pad")(x)
x = layers.DepthwiseConv2D(
kernel_size,
strides=stride,
depth_multiplier=c,
use_bias=False,
name=name + "_2_conv",
)(x)
x_shape = backend.int_shape(x)[1:-1]
x = layers.Reshape(x_shape + (groups, c, c))(x)
output_shape = x_shape + (groups,
c) if backend.backend() == "theano" else None
x = layers.Lambda(
lambda x: sum([x[:, :, :, :, i] for i in range(c)]),
output_shape=output_shape,
name=name + "_2_reduce",
)(x)
x = layers.Reshape(x_shape + (filters, ))(x)
x = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + "_2_bn")(x)
x = layers.Activation("relu", name=name + "_2_relu")(x)
x = layers.Conv2D((64 // groups) * filters,
1,
use_bias=False,
name=name + "_3_conv")(x)
x = layers.BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + "_3_bn")(x)
x = layers.Add(name=name + "_add")([shortcut, x])
x = layers.Activation("relu", name=name + "_out")(x)
return x
def call(self, inputs):
x = self.conv1(inputs)
x = self.pixel_conv(x)
x = self.conv2(x)
return layers.Add([inputs, x])
def build_BiFPN(features, num_channels, id, freeze_bn=False):
if id == 0:
_, _, C3, C4, C5 = features
P3_in = ConvBlock(num_channels,
kernel_size=1,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_P3'.format(id))(C3)
P4_in = ConvBlock(num_channels,
kernel_size=1,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_P4'.format(id))(C4)
P5_in = ConvBlock(num_channels,
kernel_size=1,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_P5'.format(id))(C5)
P6_in = ConvBlock(num_channels,
kernel_size=3,
strides=2,
freeze_bn=freeze_bn,
name='BiFPN_{}_P6'.format(id))(C5)
P7_in = ConvBlock(num_channels,
kernel_size=3,
strides=2,
freeze_bn=freeze_bn,
name='BiFPN_{}_P7'.format(id))(P6_in)
else:
P3_in, P4_in, P5_in, P6_in, P7_in = features
P3_in = ConvBlock(num_channels,
kernel_size=1,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_P3'.format(id))(P3_in)
P4_in = ConvBlock(num_channels,
kernel_size=1,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_P4'.format(id))(P4_in)
P5_in = ConvBlock(num_channels,
kernel_size=1,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_P5'.format(id))(P5_in)
P6_in = ConvBlock(num_channels,
kernel_size=1,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_P6'.format(id))(P6_in)
P7_in = ConvBlock(num_channels,
kernel_size=1,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_P7'.format(id))(P7_in)
# upsample
P7_U = layers.UpSampling2D()(P7_in)
P6_td = layers.Add()([P7_U, P6_in])
P6_td = DepthwiseConvBlock(kernel_size=3,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_U_P6'.format(id))(P6_td)
P6_U = layers.UpSampling2D()(P6_td)
P5_td = layers.Add()([P6_U, P5_in])
P5_td = DepthwiseConvBlock(kernel_size=3,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_U_P5'.format(id))(P5_td)
P5_U = layers.UpSampling2D()(P5_td)
P4_td = layers.Add()([P5_U, P4_in])
P4_td = DepthwiseConvBlock(kernel_size=3,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_U_P4'.format(id))(P4_td)
P4_U = layers.UpSampling2D()(P4_td)
P3_out = layers.Add()([P4_U, P3_in])
P3_out = DepthwiseConvBlock(kernel_size=3,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_U_P3'.format(id))(P3_out)
# downsample
P3_D = layers.MaxPooling2D(strides=(2, 2))(P3_out)
P4_out = layers.Add()([P3_D, P4_td, P4_in])
P4_out = DepthwiseConvBlock(kernel_size=3,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_D_P4'.format(id))(P4_out)
P4_D = layers.MaxPooling2D(strides=(2, 2))(P4_out)
P5_out = layers.Add()([P4_D, P5_td, P5_in])
P5_out = DepthwiseConvBlock(kernel_size=3,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_D_P5'.format(id))(P5_out)
P5_D = layers.MaxPooling2D(strides=(2, 2))(P5_out)
P6_out = layers.Add()([P5_D, P6_td, P6_in])
P6_out = DepthwiseConvBlock(kernel_size=3,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_D_P6'.format(id))(P6_out)
P6_D = layers.MaxPooling2D(strides=(2, 2))(P6_out)
P7_out = layers.Add()([P6_D, P7_in])
P7_out = DepthwiseConvBlock(kernel_size=3,
strides=1,
freeze_bn=freeze_bn,
name='BiFPN_{}_D_P7'.format(id))(P7_out)
return P3_out, P4_out, P5_out, P6_out, P7_out
def build(self, mode, config):
tf.compat.v1.disable_eager_execution()
#input
input_image = keras.Input(shape=config.IMAGE_SHAPE.tolist(),
name="input_image")
input_image_meta = keras.Input(shape=[None], name="input_image_meta")
if mode == "train":
#RPN
input_rpn_match = keras.Input(shape=[None, 1],
name="input_rpn_match",
dtype=tf.int32) #match
input_rpn_bbox = keras.Input(shape=[None, 4],
name="input_rpn_bbox",
dtype=tf.float32) #bounding box
#ground truth
input_gt_ids = keras.Input(shape=[None],
name="input_gt_class_ids",
dtype=tf.int32) #GT class IDs
input_gt_boxes = keras.Input(shape=[None, 4],
name="input_gt_boxes",
dtype=tf.float32)
h, w = K.shape(input_image)[1], K.shape(input_image)[2]
image_scale = K.cast(K.stack([h, w, h, w], axis=0), tf.float32)
#normalize ground truth boxes
gt_boxes = layers.Lambda(lambda x: x / image_scale)(input_gt_boxes)
#mini_mask
#input_gt_masks = keras.Input(shape = [config.MINI_MASK_SHAPE[0],
# config.MINI_MASK_SHAPE[1], None],
# name = "input_gt_masks", dtype=bool)
#resnet layer
_, C2, C3, C4, C5 = resnet101.build_layers(input_image)
#FPN
P5 = layers.Conv2D(256, (1, 1), name='fpn_c5p5')(C5)
P4 = layers.Add(name="fpn_p4add")([
layers.UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
layers.Conv2D(256, (1, 1), name='fpn_c4p4')(C4)
])
P3 = layers.Add(name="fpn_p3add")([
layers.UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
layers.Conv2D(256, (1, 1), name='fpn_c3p3')(C3)
])
P2 = layers.Add(name="fpn_p2add")([
layers.UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
layers.Conv2D(256, (1, 1), name='fpn_c2p2')(C2)
])
# Attach 3x3 conv to all P layers to get the final feature maps.
P2 = layers.Conv2D(256, (3, 3), padding="SAME", name="fpn_p2")(P2)
P3 = layers.Conv2D(256, (3, 3), padding="SAME", name="fpn_p3")(P3)
P4 = layers.Conv2D(256, (3, 3), padding="SAME", name="fpn_p4")(P4)
P5 = layers.Conv2D(256, (3, 3), padding="SAME", name="fpn_p5")(P5)
# P6 is used for the 5th anchor scale in RPN. Generated by
# subsampling from P5 with stride of 2.
P6 = layers.MaxPooling2D(pool_size=(1, 1), strides=2,
name="fpn_p6")(P5)
RPN_feature = [P2, P3, P4, P5, P6]
RCNN_feature = [P2, P3, P4, P5]
self.anchors = utils.generate_anchors(self.config.ANCHOR_SCALES,
self.config.ANCHOR_RATIOS,
self.config.ANCHOR_STRIDE,
self.config.BACKBONE_SHAPES,
self.config.BACKBONE_STRIDES)
#RPN Keras model
input_feature = keras.Input(shape=[None, None, config.PIRAMID_SIZE],
name="input_rpn_feature_map")
"""
rpn_class_cls: anchor class classifier
rpn_probs: anchor classifier probability
rpn_bbox_offset: anchor bounding box offset
"""
outputs = RPN.build_graph(input_feature, len(config.ANCHOR_RATIOS),
config.ANCHOR_STRIDE)
RPN_model = keras.Model([input_feature], outputs, name="rpn_model")
"""
In FPN, we generate a pyramid of feature maps. We apply the RPN (described in the previous section)
to generate ROIs. Based on the size of the ROI,
we select the feature map layer in the most proper scale to extract the feature patches.
"""
layer_outputs = []
for x in RPN_feature:
layer_outputs.append(RPN_model([x]))
output_names = ["rpn_class_logits", "rpn_class", "rpn_bbox"]
outputs = list(zip(*layer_outputs))
outputs = [
layers.Concatenate(axis=1, name=n)(list(o))
for o, n in zip(outputs, output_names)
]
#rpn_class_cls, rpn_probs, rpn_bbox = layer_outputs
rpn_class_ids, rpn_probs, rpn_bbox_offset = outputs
#Proposal layer
if mode == "train":
num_proposal = config.NUM_ROI_TRAINING
else:
num_proposal = config.NUM_ROI_INFERENCE
ROIS_proposals = ProposalLayer(
num_proposal=num_proposal,
nms_threshold=config.NMS_THRESHOLD,
anchors=self.anchors,
config=config)([rpn_probs, rpn_bbox_offset])
#combine together
if mode == "train":
#class ids
total_class_ids = layers.Lambda(lambda x: utils.parse_image_meta(x)
["class_ids"])(input_image_meta)
#Subsamples proposals and generates target box refinement, class_ids 1.7
#ratio postive/negative rois = 1/3 (threshold = 0.5)
#target_ids: class ids of gt boxes closest to positive roi
#target_bbox = offset from positive rois to it's closest gt_box
target_rois = ROIS_proposals
#rois, target_ids, target_bbox, target_mask =\
# TrainingDetectionLayer(config, name="proposal_targets")([target_rois,
# input_gt_ids,gt_boxes, input_gt_masks])
rois, target_ids, target_bbox =\
TrainingDetectionLayer(config, name="proposal_targets")([target_rois,
input_gt_ids,gt_boxes])
#classification and regression ROIs after RPN through FPN
rcnn_class_ids, rcnn_class_probs, rcnn_bbox = fpn_classifier(
rois, RCNN_feature, config.IMAGE_SHAPE, config.POOL_SIZE,
config.NUM_CLASSES)
#rcnn_mask = fpn_mask(rois, RCNN_feature, config.IMAGE_SHAPE, config.MASK_POOL_SIZE, config.NUM_CLASSES)
output_rois = layers.Lambda(lambda x: x * 1,
name="output_rois")(rois)
#rpn losses
rpn_class_loss = layers.Lambda(
lambda x: losses.rpn_class_loss_func(*x),
name="rpn_class_loss")([input_rpn_match, rpn_class_ids])
rpn_bbox_loss = layers.Lambda(
lambda x: losses.rpn_bbox_loss_func(config, *x),
name="rpn_bbox_loss")(
[input_rpn_bbox, input_rpn_match, rpn_bbox_offset])
#rcnn losses
rcnn_class_loss = layers.Lambda(
lambda x: losses.rcnn_class_loss_func(*x),
name="mrcnn_class_loss")(
[target_ids, rcnn_class_ids, total_class_ids])
rcnn_bbox_loss = layers.Lambda(
lambda x: losses.rcnn_bbox_loss_func(*x),
name="mrcnn_bbox_loss")([target_bbox, target_ids, rcnn_bbox])
#rcnn_mask_loss = layers.Lambda(lambda x : losses.rcnn_mask_loss_func(*x), name="mrcnn_mask_loss")(
# [target_mask, target_ids, rcnn_mask])
#MODEL
"""
inputs = [input_image, input_image_meta, input_rpn_match, input_rpn_bbox,
input_gt_ids, input_gt_boxes, input_gt_masks]
outputs = [rpn_class_ids, rpn_probs, rpn_bbox_offset,
rcnn_class_ids,rcnn_class_probs, rcnn_bbox, rcnn_mask,
ROIS_proposals, output_rois,
rpn_class_loss, rpn_bbox_loss, rcnn_class_loss, rcnn_bbox_loss, rcnn_mask_loss]
"""
inputs = [
input_image, input_image_meta, input_rpn_match, input_rpn_bbox,
input_gt_ids, input_gt_boxes
]
outputs = [
rpn_class_ids, rpn_probs, rpn_bbox_offset, rcnn_class_ids,
rcnn_class_probs, rcnn_bbox, ROIS_proposals, output_rois,
rpn_class_loss, rpn_bbox_loss, rcnn_class_loss, rcnn_bbox_loss
]
model = keras.Model(inputs, outputs, name='mask_rcnn')
else:
rcnn_class_ids,rcnn_class_probs, rcnn_bbox =\
fpn_classifier(ROIS_proposals, RCNN_feature, config.IMAGE_SHAPE,
config.POOL_SIZE,config.NUM_CLASSES)
#[N, (y1, x1, y2, x2, class_id, score)]
detections = InferenceDetectionLayer(config,
name="mrcnn_detection")([
ROIS_proposals,
rcnn_class_probs,
rcnn_bbox,
input_image_meta
])
#h,w = config.IMAGE_SHAPE[:2]
#detection_boxes = layers.Lambda(
# lambda x: x[..., :4] / np.array([h,w,h,w]))(detections)
inputs = [input_image, input_image_meta]
outputs = [
detections, rcnn_class_probs, rcnn_bbox, ROIS_proposals,
rpn_probs, rpn_bbox_offset
]
model = keras.Model(inputs, outputs, name='mask_rcnn')
#print(model.layers)
return model
def build_BiFPN(features, num_channels, id, freeze_bn=False):
if id == 0:
_, _, C3, C4, C5 = features
P3_in = C3
P4_in = C4
P5_in = C5
P6_in = layers.Conv2D(num_channels,
kernel_size=1,
padding='same',
name='resample_p6/conv2d')(C5)
P6_in = layers.BatchNormalization(momentum=MOMENTUM,
epsilon=EPSILON,
name='resample_p6/bn')(P6_in)
# P6_in = BatchNormalization(freeze=freeze_bn, name='resample_p6/bn')(P6_in)
P6_in = layers.MaxPooling2D(pool_size=3,
strides=2,
padding='same',
name='resample_p6/maxpool')(P6_in)
P7_in = layers.MaxPooling2D(pool_size=3,
strides=2,
padding='same',
name='resample_p7/maxpool')(P6_in)
P7_U = layers.UpSampling2D()(P7_in)
P6_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode0/add')(
[P6_in, P7_U])
P6_td = layers.Activation(lambda x: tf.nn.swish(x))(P6_td)
P6_td = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode0/op_after_combine5')(P6_td)
P5_in_1 = layers.Conv2D(
num_channels,
kernel_size=1,
padding='same',
name=f'fpn_cells/cell_{id}/fnode1/resample_0_2_6/conv2d')(P5_in)
P5_in_1 = layers.BatchNormalization(
momentum=MOMENTUM,
epsilon=EPSILON,
name=f'fpn_cells/cell_{id}/fnode1/resample_0_2_6/bn')(P5_in_1)
# P5_in_1 = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode1/resample_0_2_6/bn')(P5_in_1)
P6_U = layers.UpSampling2D()(P6_td)
P5_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode1/add')(
[P5_in_1, P6_U])
P5_td = layers.Activation(lambda x: tf.nn.swish(x))(P5_td)
P5_td = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode1/op_after_combine6')(P5_td)
P4_in_1 = layers.Conv2D(
num_channels,
kernel_size=1,
padding='same',
name=f'fpn_cells/cell_{id}/fnode2/resample_0_1_7/conv2d')(P4_in)
P4_in_1 = layers.BatchNormalization(
momentum=MOMENTUM,
epsilon=EPSILON,
name=f'fpn_cells/cell_{id}/fnode2/resample_0_1_7/bn')(P4_in_1)
# P4_in_1 = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode2/resample_0_1_7/bn')(P4_in_1)
P5_U = layers.UpSampling2D()(P5_td)
P4_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode2/add')(
[P4_in_1, P5_U])
P4_td = layers.Activation(lambda x: tf.nn.swish(x))(P4_td)
P4_td = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode2/op_after_combine7')(P4_td)
P3_in = layers.Conv2D(
num_channels,
kernel_size=1,
padding='same',
name=f'fpn_cells/cell_{id}/fnode3/resample_0_0_8/conv2d')(P3_in)
P3_in = layers.BatchNormalization(
momentum=MOMENTUM,
epsilon=EPSILON,
name=f'fpn_cells/cell_{id}/fnode3/resample_0_0_8/bn')(P3_in)
# P3_in = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode3/resample_0_0_8/bn')(P3_in)
P4_U = layers.UpSampling2D()(P4_td)
P3_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode3/add')(
[P3_in, P4_U])
P3_out = layers.Activation(lambda x: tf.nn.swish(x))(P3_out)
P3_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode3/op_after_combine8')(P3_out)
P4_in_2 = layers.Conv2D(
num_channels,
kernel_size=1,
padding='same',
name=f'fpn_cells/cell_{id}/fnode4/resample_0_1_9/conv2d')(P4_in)
P4_in_2 = layers.BatchNormalization(
momentum=MOMENTUM,
epsilon=EPSILON,
name=f'fpn_cells/cell_{id}/fnode4/resample_0_1_9/bn')(P4_in_2)
# P4_in_2 = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode4/resample_0_1_9/bn')(P4_in_2)
P3_D = layers.MaxPooling2D(pool_size=3, strides=2,
padding='same')(P3_out)
P4_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode4/add')(
[P4_in_2, P4_td, P3_D])
P4_out = layers.Activation(lambda x: tf.nn.swish(x))(P4_out)
P4_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode4/op_after_combine9')(P4_out)
P5_in_2 = layers.Conv2D(
num_channels,
kernel_size=1,
padding='same',
name=f'fpn_cells/cell_{id}/fnode5/resample_0_2_10/conv2d')(P5_in)
P5_in_2 = layers.BatchNormalization(
momentum=MOMENTUM,
epsilon=EPSILON,
name=f'fpn_cells/cell_{id}/fnode5/resample_0_2_10/bn')(P5_in_2)
# P5_in_2 = BatchNormalization(freeze=freeze_bn, name=f'fpn_cells/cell_{id}/fnode5/resample_0_2_10/bn')(P5_in_2)
P4_D = layers.MaxPooling2D(pool_size=3, strides=2,
padding='same')(P4_out)
P5_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode5/add')(
[P5_in_2, P5_td, P4_D])
P5_out = layers.Activation(lambda x: tf.nn.swish(x))(P5_out)
P5_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode5/op_after_combine10')(P5_out)
P5_D = layers.MaxPooling2D(pool_size=3, strides=2,
padding='same')(P5_out)
P6_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode6/add')(
[P6_in, P6_td, P5_D])
P6_out = layers.Activation(lambda x: tf.nn.swish(x))(P6_out)
P6_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode6/op_after_combine11')(P6_out)
P6_D = layers.MaxPooling2D(pool_size=3, strides=2,
padding='same')(P6_out)
P7_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode7/add')(
[P7_in, P6_D])
P7_out = layers.Activation(lambda x: tf.nn.swish(x))(P7_out)
P7_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode7/op_after_combine12')(P7_out)
else:
P3_in, P4_in, P5_in, P6_in, P7_in = features
P7_U = layers.UpSampling2D()(P7_in)
P6_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode0/add')(
[P6_in, P7_U])
P6_td = layers.Activation(lambda x: tf.nn.swish(x))(P6_td)
P6_td = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode0/op_after_combine5')(P6_td)
P6_U = layers.UpSampling2D()(P6_td)
P5_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode1/add')(
[P5_in, P6_U])
P5_td = layers.Activation(lambda x: tf.nn.swish(x))(P5_td)
P5_td = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode1/op_after_combine6')(P5_td)
P5_U = layers.UpSampling2D()(P5_td)
P4_td = layers.Add(name=f'fpn_cells/cell_{id}/fnode2/add')(
[P4_in, P5_U])
P4_td = layers.Activation(lambda x: tf.nn.swish(x))(P4_td)
P4_td = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode2/op_after_combine7')(P4_td)
P4_U = layers.UpSampling2D()(P4_td)
P3_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode3/add')(
[P3_in, P4_U])
P3_out = layers.Activation(lambda x: tf.nn.swish(x))(P3_out)
P3_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode3/op_after_combine8')(P3_out)
P3_D = layers.MaxPooling2D(pool_size=3, strides=2,
padding='same')(P3_out)
P4_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode4/add')(
[P4_in, P4_td, P3_D])
P4_out = layers.Activation(lambda x: tf.nn.swish(x))(P4_out)
P4_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode4/op_after_combine9')(P4_out)
P4_D = layers.MaxPooling2D(pool_size=3, strides=2,
padding='same')(P4_out)
P5_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode5/add')(
[P5_in, P5_td, P4_D])
P5_out = layers.Activation(lambda x: tf.nn.swish(x))(P5_out)
P5_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode5/op_after_combine10')(P5_out)
P5_D = layers.MaxPooling2D(pool_size=3, strides=2,
padding='same')(P5_out)
P6_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode6/add')(
[P6_in, P6_td, P5_D])
P6_out = layers.Activation(lambda x: tf.nn.swish(x))(P6_out)
P6_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode6/op_after_combine11')(P6_out)
P6_D = layers.MaxPooling2D(pool_size=3, strides=2,
padding='same')(P6_out)
P7_out = layers.Add(name=f'fpn_cells/cell_{id}/fnode7/add')(
[P7_in, P6_D])
P7_out = layers.Activation(lambda x: tf.nn.swish(x))(P7_out)
P7_out = SeparableConvBlock(
num_channels=num_channels,
kernel_size=3,
strides=1,
name=f'fpn_cells/cell_{id}/fnode7/op_after_combine12')(P7_out)
return P3_out, P4_td, P5_td, P6_td, P7_out
def build(self, mode, config):
"""Build Mask R-CNN architecture.
input_shape: The shape of the input image.
mode: Either "training" or "inference". The inputs and
outputs of the model differ accordingly.
"""
assert mode in ['training', 'inference']
# Image size must be dividable by 2 multiple times
h, w = config.IMAGE_SHAPE[:2]
if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6):
raise Exception(
"Image size must be dividable by 2 at least 6 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# Inputs
input_image = KL.Input(shape=[None, None, config.IMAGE_SHAPE[2]],
name="input_image")
input_image_meta = KL.Input(shape=[config.IMAGE_META_SIZE],
name="input_image_meta")
if mode == "training":
# RPN GT
input_rpn_match = KL.Input(shape=[None, 1],
name="input_rpn_match",
dtype=tf.int32)
input_rpn_bbox = KL.Input(shape=[None, 4],
name="input_rpn_bbox",
dtype=tf.float32)
# Detection GT (class IDs, bounding boxes, and masks)
# 1. GT Class IDs (zero padded)
input_gt_class_ids = KL.Input(shape=[None],
name="input_gt_class_ids",
dtype=tf.int32)
# 2. GT Boxes in pixels (zero padded)
# [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] in image coordinates
input_gt_boxes = KL.Input(shape=[None, 4],
name="input_gt_boxes",
dtype=tf.float32)
# Normalize coordinates
gt_boxes = KL.Lambda(lambda x: norm_boxes_graph(
x,
K.shape(input_image)[1:3]))(input_gt_boxes)
# 3. GT Masks (zero padded)
# [batch, height, width, MAX_GT_INSTANCES]
if config.USE_MINI_MASK:
input_gt_masks = KL.Input(shape=[
config.MINI_MASK_SHAPE[0], config.MINI_MASK_SHAPE[1], None
],
name="input_gt_masks",
dtype=bool)
else:
input_gt_masks = KL.Input(
shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None],
name="input_gt_masks",
dtype=bool)
elif mode == "inference":
# Anchors in normalized coordinates
input_anchors = KL.Input(shape=[None, 4], name="input_anchors")
# Build the shared convolutional layers.
# Bottom-up Layers
# Returns a list of the last layers of each stage, 5 in total.
# Don't create the thead (stage 5), so we pick the 4th item in the list.
if callable(config.BACKBONE):
_, C2, C3, C4, C5 = config.BACKBONE(input_image,
stage5=True,
train_bn=config.TRAIN_BN)
else:
_, C2, C3, C4, C5 = resnet_graph(input_image,
config.BACKBONE,
stage5=True,
train_bn=config.TRAIN_BN)
# Top-down Layers
# TODO: add assert to varify feature map sizes match what's in config
P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c5p5')(C5)
P4 = KL.Add(name="fpn_p4add")([
KL.UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c4p4')(C4)
])
P3 = KL.Add(name="fpn_p3add")([
KL.UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c3p3')(C3)
])
P2 = KL.Add(name="fpn_p2add")([
KL.UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c2p2')(C2)
])
# Attach 3x3 conv to all P layers to get the final feature maps.
P2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p2")(P2)
P3 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p3")(P3)
P4 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p4")(P4)
P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p5")(P5)
# P6 is used for the 5th anchor scale in RPN. Generated by
# # subsampling from P5 with stride of 2.
P6 = KL.MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5)
# Bottom-up Layers
N3 = KL.Add(name="fpn_n3add")([
P3,
KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
strides=(2, 2),
padding="SAME",
name='fpn_n2conv')(P2)
])
N4 = KL.Add(name="fpn_n4add")([
P4,
KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
strides=(2, 2),
padding="SAME",
name='fpn_n4conv')(N3)
])
N5 = KL.Add(name="fpn_n5add")([
P5,
KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
strides=(2, 2),
padding="SAME",
name='fpn_n5conv')(N4)
])
N3 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
strides=(1, 1),
padding="SAME",
name="fpn_n3")(N3)
N4 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
strides=(1, 1),
padding="SAME",
name="fpn_n4")(N4)
N5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
strides=(1, 1),
padding="SAME",
name="fpn_n5")(N5)
N6 = KL.MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_n6")(N5)
# Note that P6 is used in RPN, but not in the classifier heads.
rpn_feature_maps = [P2, N3, N4, N5, N6]
mrcnn_feature_maps = [P2, N3, N4, N5]
# Anchors
if mode == "training":
anchors = self.get_anchors(config.IMAGE_SHAPE)
# Duplicate across the batch dimension because Keras requires it
# TODO: can this be optimized to avoid duplicating the anchors?
anchors = np.broadcast_to(anchors,
(config.BATCH_SIZE, ) + anchors.shape)
# A hack to get around Keras's bad support for constants
anchors = KL.Lambda(lambda x: tf.Variable(anchors),
name="anchors")(input_image)
else:
anchors = input_anchors
# RPN Model
rpn = build_rpn_model(config.RPN_ANCHOR_STRIDE,
len(config.RPN_ANCHOR_RATIOS),
config.TOP_DOWN_PYRAMID_SIZE)
# Loop through pyramid layers
layer_outputs = [] # list of lists
for p in rpn_feature_maps:
layer_outputs.append(rpn([p]))
# Concatenate layer outputs
# Convert from list of lists of level outputs to list of lists
# of outputs across levels.
# e.g. [[a1, b1, c1], [a2, b2, c2]] => [[a1, a2], [b1, b2], [c1, c2]]
output_names = ["rpn_class_logits", "rpn_class", "rpn_bbox"]
outputs = list(zip(*layer_outputs))
outputs = [
KL.Concatenate(axis=1, name=n)(list(o))
for o, n in zip(outputs, output_names)
]
rpn_class_logits, rpn_class, rpn_bbox = outputs
# Generate proposals
# Proposals are [batch, N, (y1, x1, y2, x2)] in normalized coordinates
# and zero padded.
proposal_count = config.POST_NMS_ROIS_TRAINING if mode == "training" \
else config.POST_NMS_ROIS_INFERENCE
rpn_rois = ProposalLayer(proposal_count=proposal_count,
nms_threshold=config.RPN_NMS_THRESHOLD,
name="ROI",
config=config)([rpn_class, rpn_bbox, anchors])
if mode == "training":
# Class ID mask to mark class IDs supported by the dataset the image
# came from.
active_class_ids = KL.Lambda(lambda x: parse_image_meta_graph(x)[
"active_class_ids"])(input_image_meta)
if not config.USE_RPN_ROIS:
# Ignore predicted ROIs and use ROIs provided as an input.
input_rois = KL.Input(shape=[config.POST_NMS_ROIS_TRAINING, 4],
name="input_roi",
dtype=np.int32)
# Normalize coordinates
target_rois = KL.Lambda(lambda x: norm_boxes_graph(
x,
K.shape(input_image)[1:3]))(input_rois)
else:
target_rois = rpn_rois
# Generate detection targets
# Subsamples proposals and generates target outputs for training
# Note that proposal class IDs, gt_boxes, and gt_masks are zero
# padded. Equally, returned rois and targets are zero padded.
rois, target_class_ids, target_bbox, target_mask = \
DetectionTargetLayer(config, name="proposal_targets")([
target_rois, input_gt_class_ids, gt_boxes, input_gt_masks])
# Network Heads
# TODO: verify that this handles zero padded ROIs
mrcnn_class_logits, mrcnn_class, mrcnn_bbox = \
panet_fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta,
config.POOL_SIZE, config.NUM_CLASSES,
train_bn=config.TRAIN_BN,
fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)
mrcnn_mask = panet_build_fpn_mask_graph(rois,
mrcnn_feature_maps,
input_image_meta,
config.MASK_POOL_SIZE,
config.NUM_CLASSES,
train_bn=config.TRAIN_BN)
# TODO: clean up (use tf.identify if necessary)
output_rois = KL.Lambda(lambda x: x * 1, name="output_rois")(rois)
# Losses
rpn_class_loss = KL.Lambda(lambda x: rpn_class_loss_graph(*x),
name="rpn_class_loss")(
[input_rpn_match, rpn_class_logits])
rpn_bbox_loss = KL.Lambda(
lambda x: rpn_bbox_loss_graph(config, *x),
name="rpn_bbox_loss")(
[input_rpn_bbox, input_rpn_match, rpn_bbox])
class_loss = KL.Lambda(lambda x: mrcnn_class_loss_graph(*x),
name="mrcnn_class_loss")([
target_class_ids, mrcnn_class_logits,
active_class_ids
])
bbox_loss = KL.Lambda(lambda x: mrcnn_bbox_loss_graph(*x),
name="mrcnn_bbox_loss")([
target_bbox, target_class_ids, mrcnn_bbox
])
mask_loss = KL.Lambda(lambda x: mrcnn_mask_loss_graph(*x),
name="mrcnn_mask_loss")([
target_mask, target_class_ids, mrcnn_mask
])
# Model
inputs = [
input_image, input_image_meta, input_rpn_match, input_rpn_bbox,
input_gt_class_ids, input_gt_boxes, input_gt_masks
]
if not config.USE_RPN_ROIS:
inputs.append(input_rois)
outputs = [
rpn_class_logits, rpn_class, rpn_bbox, mrcnn_class_logits,
mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, output_rois,
rpn_class_loss, rpn_bbox_loss, class_loss, bbox_loss, mask_loss
]
model = KM.Model(inputs, outputs, name='mask_rcnn')
else:
# Network Heads
# Proposal classifier and BBox regressor heads
mrcnn_class_logits, mrcnn_class, mrcnn_bbox = \
panet_fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta,
config.POOL_SIZE, config.NUM_CLASSES,
train_bn=config.TRAIN_BN,
fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)
# Detections
# output is [batch, num_detections, (y1, x1, y2, x2, class_id, score)] in
# normalized coordinates
detections = DetectionLayer(config, name="mrcnn_detection")(
[rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta])
# Create masks for detections
detection_boxes = KL.Lambda(lambda x: x[..., :4])(detections)
mrcnn_mask = panet_build_fpn_mask_graph(detection_boxes,
mrcnn_feature_maps,
input_image_meta,
config.MASK_POOL_SIZE,
config.NUM_CLASSES,
train_bn=config.TRAIN_BN)
model = KM.Model([input_image, input_image_meta, input_anchors], [
detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois,
rpn_class, rpn_bbox
],
name='mask_rcnn')
# Add multi-GPU support.
if config.GPU_COUNT > 1:
from .parallel_model import ParallelModel
model = ParallelModel(model, config.GPU_COUNT)
return model
def block2(x,
filters,
kernel_size=3,
stride=1,
conv_shortcut=False,
name=None,
norm_use="bn"):
"""A residual block.
# Arguments
x: input tensor.
filters: integer, filters of the bottleneck layer.
kernel_size: default 3, kernel size of the bottleneck layer.
stride: default 1, stride of the first layer.
conv_shortcut: default False, use convolution shortcut if True,
otherwise identity shortcut.
name: string, block label.
# Returns
Output tensor for the residual block.
"""
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
#preact = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_preact_bn')(x)
preact = normalize_layer(x, norm_use=norm_use, name=name + '_preact_')
preact = layers.Activation('relu', name=name + '_preact_relu')(preact)
if conv_shortcut is True:
shortcut = layers.Conv2D(4 * filters,
1,
strides=stride,
kernel_initializer='he_normal',
name=name + '_0_conv')(preact)
else:
shortcut = layers.MaxPooling2D(1,
strides=stride)(x) if stride > 1 else x
x = layers.Conv2D(filters,
1,
strides=1,
use_bias=False,
kernel_initializer='he_normal',
name=name + '_1_conv')(preact)
x = normalize_layer(x, norm_use=norm_use, name=name + '_1_')
#x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_1_bn')(x)
x = layers.Activation('relu', name=name + '_1_relu')(x)
x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x)
x = layers.Conv2D(filters,
kernel_size,
strides=stride,
kernel_initializer='he_normal',
use_bias=False,
name=name + '_2_conv')(x)
x = normalize_layer(x, norm_use=norm_use, name=name + '_2_')
#x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_2_bn')(x)
x = layers.Activation('relu', name=name + '_2_relu')(x)
x = layers.Conv2D(4 * filters,
1,
name=name + '_3_conv',
kernel_initializer='he_normal')(x)
x = layers.Add(name=name + '_out')([shortcut, x])
return x
def build_generator(self):
def residual_block(layer_input, filters):
#"""Residual block described in paper"""
d = layers.Conv2D(filters,
kernel_size=3,
strides=1,
padding='same')(layer_input)
d = layers.Activation('relu')(d)
d = layers.BatchNormalization(momentum=0.8)(d)
d = layers.Conv2D(filters,
kernel_size=3,
strides=1,
padding='same')(d)
d = layers.BatchNormalization(momentum=0.8)(d)
d = layers.Add()([d, layer_input])
return d
def deconv2d(layer_input):
#"""Layers used during upsampling"""
u = layers.UpSampling2D(size=2)(layer_input)
u = layers.Conv2D(256, kernel_size=3, strides=1, padding='same')(u)
u = layers.Activation('relu')(u)
return u
img_lr_shape = (config.input_width, config.input_height, 3)
model = Sequential()
img_lr = Input(shape=img_lr_shape)
c1 = layers.Conv2D(64, kernel_size=9, strides=1,
padding='same')(img_lr * 2 - 1)
c1 = layers.Activation('relu')(c1)
# Local variables
self.gf = 64
self.n_residual_blocks = 16
# Propogate through residual blocks
r = residual_block(c1, self.gf)
for _ in range(self.n_residual_blocks - 1):
r = residual_block(r, self.gf)
# Post-residual block
c2 = layers.Conv2D(64, kernel_size=3, strides=1, padding='same')(r)
c2 = layers.BatchNormalization(momentum=0.8)(c2)
c2 = layers.Add()([c2, c1])
# Upsampling
u1 = deconv2d(c2)
u2 = deconv2d(u1)
u3 = deconv2d(u2)
# Generate high resolution output
gen_hr = layers.Conv2D(self.channels,
kernel_size=9,
strides=1,
padding='same',
activation='tanh')(u3)
gen_hr = gen_hr * 0.5 + 0.5
return Model(img_lr, gen_hr)
shape=(time_sound, nfreqs,
1)) # define input (rows, columns, channels (only one in my case))
model_l_conv1 = layers.Conv2D(32, (3, 7), activation='relu', padding='same')(
in1) # define first layer and input to the layer
model_l_conv1_mp = layers.MaxPooling2D(pool_size=(1, 2))(model_l_conv1)
model_l_conv1_mp_do = layers.Dropout(0.2)(model_l_conv1_mp)
# CNN 1 - right channel
in2 = layers.Input(shape=(time_sound, nfreqs, 1)) # define input
model_r_conv1 = layers.Conv2D(32, (3, 7), activation='relu', padding='same')(
in2) # define first layer and input to the layer
model_r_conv1_mp = layers.MaxPooling2D(pool_size=(1, 2))(model_r_conv1)
model_r_conv1_mp_do = layers.Dropout(0.2)(model_r_conv1_mp)
# CNN 2 - merged
model_final_merge = layers.Add()([model_l_conv1_mp_do, model_r_conv1_mp_do])
model_final_conv1 = layers.Conv2D(32, (3, 7),
activation='relu',
padding='same')(model_final_merge)
model_final_conv1_mp = layers.MaxPooling2D(pool_size=(2, 2))(model_final_conv1)
model_final_conv1_mp_do = layers.Dropout(0.2)(model_final_conv1_mp)
# CNN 3 - merged
model_final_conv2 = layers.Conv2D(64, (3, 7),
activation='relu',
padding='same')(model_final_conv1_mp_do)
model_final_conv2_mp = layers.MaxPooling2D(pool_size=(2, 2))(model_final_conv2)
model_final_conv2_mp_do = layers.Dropout(0.2)(model_final_conv2_mp)
# CNN 4 - merged
model_final_conv3 = layers.Conv2D(128, (3, 7),
def add_layer(inputs, **kwargs):
"""
Simple wrapper around add layer for convenience
"""
return layers.Add(**kwargs)(inputs)
def Smi2Smi():
#product
l_in = layers.Input(shape=(None, ))
l_mask = layers.Input(shape=(None, ))
#reagents
l_dec = layers.Input(shape=(None, ))
l_dmask = layers.Input(shape=(None, ))
#positional encodings for product and reagents, respectively
l_pos = PositionLayer(EMBEDDING_SIZE)(l_mask)
l_dpos = PositionLayer(EMBEDDING_SIZE)(l_dmask)
l_emask = MaskLayerRight()([l_dmask, l_mask])
l_right_mask = MaskLayerTriangular()(l_dmask)
l_left_mask = MaskLayerLeft()(l_mask)
#encoder
l_voc = layers.Embedding(input_dim=vocab_size,
output_dim=EMBEDDING_SIZE,
input_length=None)
l_embed = layers.Add()([l_voc(l_in), l_pos])
l_embed = layers.Dropout(rate=0.1)(l_embed)
for layer in range(n_block):
#self attention
l_o = [
SelfLayer(EMBEDDING_SIZE,
KEY_SIZE)([l_embed, l_embed, l_embed, l_left_mask])
for i in range(n_self)
]
l_con = layers.Concatenate()(l_o)
l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con)
l_drop = layers.Dropout(rate=0.1)(l_dense)
l_add = layers.Add()([l_drop, l_embed])
l_att = LayerNormalization()(l_add)
#position-wise
l_c1 = layers.Conv1D(N_HIDDEN, 1, activation='relu')(l_att)
l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1)(l_c1)
l_drop = layers.Dropout(rate=0.1)(l_c2)
l_ff = layers.Add()([l_att, l_drop])
l_embed = LayerNormalization()(l_ff)
#bottleneck
l_encoder = l_embed
l_embed = layers.Add()([l_voc(l_dec), l_dpos])
l_embed = layers.Dropout(rate=0.1)(l_embed)
for layer in range(n_block):
#self attention
l_o = [
SelfLayer(EMBEDDING_SIZE,
KEY_SIZE)([l_embed, l_embed, l_embed, l_right_mask])
for i in range(n_self)
]
l_con = layers.Concatenate()(l_o)
l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con)
l_drop = layers.Dropout(rate=0.1)(l_dense)
l_add = layers.Add()([l_drop, l_embed])
l_att = LayerNormalization()(l_add)
#attention to the encoder
l_o = [
SelfLayer(EMBEDDING_SIZE,
KEY_SIZE)([l_att, l_encoder, l_encoder, l_emask])
for i in range(n_self)
]
l_con = layers.Concatenate()(l_o)
l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con)
l_drop = layers.Dropout(rate=0.1)(l_dense)
l_add = layers.Add()([l_drop, l_att])
l_att = LayerNormalization()(l_add)
#position-wise
l_c1 = layers.Conv1D(N_HIDDEN, 1, activation='relu')(l_att)
l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1)(l_c1)
l_drop = layers.Dropout(rate=0.1)(l_c2)
l_ff = layers.Add()([l_att, l_drop])
l_embed = LayerNormalization()(l_ff)
l_out = layers.TimeDistributed(layers.Dense(vocab_size,
use_bias=False))(l_embed)
mdl = tf.keras.Model([l_in, l_mask, l_dec, l_dmask], l_out)
def masked_loss(y_true, y_pred):
loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_true,
logits=y_pred)
mask = tf.cast(tf.not_equal(tf.reduce_sum(y_true, -1), 0), 'float32')
loss = tf.reduce_sum(loss * mask, -1) / tf.reduce_sum(mask, -1)
loss = K.mean(loss)
return loss
def masked_acc(y_true, y_pred):
mask = tf.cast(tf.not_equal(tf.reduce_sum(y_true, -1), 0), 'float32')
eq = K.cast(
K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)),
'float32')
eq = tf.reduce_sum(eq * mask, -1) / tf.reduce_sum(mask, -1)
eq = K.mean(eq)
return eq
mdl.compile(optimizer='adam',
loss=masked_loss,
metrics=['accuracy', masked_acc])
mdl_enc = tf.keras.Model([l_in, l_mask], l_encoder)
mdl_enc.compile(optimizer="adam", loss="categorical_crossentropy")
#mdl.summary();
return mdl, mdl_enc
def buildNetwork():
unfreeze = False
l_in = layers.Input(shape=(None, ))
l_mask = layers.Input(shape=(None, ))
l_ymask = []
for i in range(len(props)):
l_ymask.append(layers.Input(shape=(1, )))
#transformer part
#positional encodings for product and reagents, respectively
l_pos = PositionLayer(EMBEDDING_SIZE)(l_mask)
l_left_mask = MaskLayerLeft()(l_mask)
#encoder
l_voc = layers.Embedding(input_dim=vocab_size,
output_dim=EMBEDDING_SIZE,
input_length=None,
trainable=unfreeze)
l_embed = layers.Add()([l_voc(l_in), l_pos])
for layer in range(n_block):
#self attention
l_o = [
SelfLayer(EMBEDDING_SIZE, KEY_SIZE, trainable=unfreeze)(
[l_embed, l_embed, l_embed, l_left_mask])
for i in range(n_self)
]
l_con = layers.Concatenate()(l_o)
l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE,
trainable=unfreeze),
trainable=unfreeze)(l_con)
if unfreeze == True: l_dense = layers.Dropout(rate=0.1)(l_dense)
l_add = layers.Add()([l_dense, l_embed])
l_att = LayerNormalization(trainable=unfreeze)(l_add)
#position-wise
l_c1 = layers.Conv1D(N_HIDDEN,
1,
activation='relu',
trainable=unfreeze)(l_att)
l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1, trainable=unfreeze)(l_c1)
if unfreeze == True: l_c2 = layers.Dropout(rate=0.1)(l_c2)
l_ff = layers.Add()([l_att, l_c2])
l_embed = LayerNormalization(trainable=unfreeze)(l_ff)
#end of Transformer's part
l_encoder = l_embed
#text-cnn part
#https://github.com/deepchem/deepchem/blob/b7a6d3d759145d238eb8abaf76183e9dbd7b683c/deepchem/models/tensorgraph/models/text_cnn.py
l_in2 = layers.Input(shape=(None, EMBEDDING_SIZE))
kernel_sizes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20]
num_filters = [100, 200, 200, 200, 200, 100, 100, 100, 100, 100, 160, 160]
l_pool = []
for i in range(len(kernel_sizes)):
l_conv = layers.Conv1D(num_filters[i],
kernel_size=kernel_sizes[i],
padding='valid',
kernel_initializer='normal',
activation='relu')(l_in2)
l_maxpool = layers.Lambda(lambda x: tf.reduce_max(x, axis=1))(l_conv)
l_pool.append(l_maxpool)
l_cnn = layers.Concatenate(axis=1)(l_pool)
l_cnn_drop = layers.Dropout(rate=0.25)(l_cnn)
#dense part
l_dense = layers.Dense(N_HIDDEN_CNN, activation='relu')(l_cnn_drop)
#https://github.com/ParikhKadam/Highway-Layer-Keras
transform_gate = layers.Dense(
units=N_HIDDEN_CNN,
activation="sigmoid",
bias_initializer=tf.keras.initializers.Constant(-1))(l_dense)
carry_gate = layers.Lambda(lambda x: 1.0 - x,
output_shape=(N_HIDDEN_CNN, ))(transform_gate)
transformed_data = layers.Dense(units=N_HIDDEN_CNN,
activation="relu")(l_dense)
transformed_gated = layers.Multiply()([transform_gate, transformed_data])
identity_gated = layers.Multiply()([carry_gate, l_dense])
l_highway = layers.Add()([transformed_gated, identity_gated])
#Because of multitask we have here a few different outputs and a custom loss.
def mse_loss(prop):
def loss(y_true, y_pred):
y2 = y_true * l_ymask[prop] + y_pred * (1 - l_ymask[prop])
return tf.keras.losses.mse(y2, y_pred)
return loss
def binary_loss(prop):
def loss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred, K.epsilon(), 1.0 - K.epsilon())
r = y_true * K.log(y_pred) + (1.0 - y_true) * K.log(1.0 - y_pred)
r = -tf.reduce_mean(r * l_ymask[prop])
return r
return loss
l_out = []
losses = []
for prop in props:
if props[prop][2] == "regression":
l_out.append(
layers.Dense(1,
activation='linear',
name="Regression-" + props[prop][1])(l_highway))
losses.append(mse_loss(prop))
else:
l_out.append(
layers.Dense(1,
activation='sigmoid',
name="Classification-" +
props[prop][1])(l_highway))
losses.append(binary_loss(prop))
l_input = [l_in2]
l_input.extend(l_ymask)
mdl = tf.keras.Model(l_input, l_out)
mdl.compile(optimizer='adam', loss=losses)
#mdl.summary();
K.set_value(mdl.optimizer.lr, 1.0e-4)
#so far we do not train the encoder part of the model.
encoder = tf.keras.Model([l_in, l_mask], l_encoder)
encoder.compile(optimizer='adam', loss='mse')
encoder.set_weights(np.load("embeddings.npy", allow_pickle=True))
#encoder.summary();
return mdl, encoder
def fpn(encoder, input_shape, num_heads=1):
image_input = layers.Input(shape=input_shape)
# mask = layers.Input(shape=input_shape)
image_input, levels = encoder(image_input)
[f0, f1, f2, f3, f4] = levels
# paper says: f0 should be disregarded due to high memory footprint.
# ***** FPN *****
# 7x7 to 14x14
P4 = f4
f4_prime = layers.Conv2D(256, (1, 1))(P4)
x = layers.UpSampling2D((2, 2))(f4_prime)
f3_prime = layers.Conv2D(256, (1, 1))(f3)
x = layers.Add()([x, f3_prime])
x = layers.Conv2D(256, (3, 3), padding='same')(x)
x = layers.BatchNormalization(axis=-1)(x)
P3 = layers.Activation("relu")(x)
# 14x14 to 28x28
x = layers.UpSampling2D((2, 2))(P3)
f2_prime = layers.Conv2D(256, (1, 1))(f2)
x = layers.Add()([x, f2_prime])
x = layers.Conv2D(256, (3, 3), padding='same')(x)
x = layers.BatchNormalization(axis=-1)(x)
P2 = layers.Activation("relu")(x)
# 28x28 to 56x56
x = layers.UpSampling2D((2, 2))(P2)
f1_prime = layers.Conv2D(256, (1, 1))(f1)
x = layers.Add()([x, f1_prime])
x = layers.Conv2D(256, (3, 3), padding='same')(x)
x = layers.BatchNormalization(axis=-1)(x)
P1 = layers.Activation("relu")(x)
# ***** UPSAMPLE *****
# upsample to 1/4 org. image
# P4 - 1
up_p4 = layers.Conv2D(128, (3, 3), padding='same')(P4)
up_p4 = layers.BatchNormalization(axis=-1)(
up_p4) # tfa.layers.GroupNormalization()
up_p4 = layers.Activation("relu")(up_p4)
up_p4 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p4)
# P4 - 2
up_p4 = layers.Conv2D(128, (3, 3), padding='same')(up_p4)
up_p4 = layers.BatchNormalization(axis=-1)(
up_p4) # tfa.layers.GroupNormalization()
up_p4 = layers.Activation("relu")(up_p4)
up_p4 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p4)
# P4 - 3
up_p4 = layers.Conv2D(128, (3, 3), padding='same')(up_p4)
up_p4 = layers.BatchNormalization(axis=-1)(
up_p4) # tfa.layers.GroupNormalization()
up_p4 = layers.Activation("relu")(up_p4)
up_p4 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p4)
# P3 - 1
up_p3 = layers.Conv2D(128, (3, 3), padding='same')(P3)
up_p3 = layers.BatchNormalization(axis=-1)(
up_p3) # tfa.layers.GroupNormalization()
up_p3 = layers.Activation("relu")(up_p3)
up_p3 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p3)
# P3 - 2
up_p3 = layers.Conv2D(128, (3, 3), padding='same')(up_p3)
up_p3 = layers.BatchNormalization(axis=-1)(
up_p3) # tfa.layers.GroupNormalization()
up_p3 = layers.Activation("relu")(up_p3)
up_p3 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p3)
# P2 - 1
up_p2 = layers.Conv2D(128, (3, 3), padding='same')(P2)
up_p2 = layers.BatchNormalization(axis=-1)(
up_p2) # tfa.layers.GroupNormalization()
up_p2 = layers.Activation("relu")(up_p2)
up_p2 = layers.UpSampling2D((2, 2), interpolation="bilinear")(up_p2)
# P2
up_p1 = layers.Conv2D(128, (3, 3), padding='same')(P1)
up_p1 = layers.BatchNormalization(axis=-1)(
up_p1) # tfa.layers.GroupNormalization()
up_p1 = layers.Activation("relu")(up_p1)
# SUM ELEMENT WISE
y = layers.Add()([up_p4, up_p3, up_p2, up_p1])
y = layers.BatchNormalization(axis=-1)(
y) # tfa.layers.GroupNormalization()
y = layers.UpSampling2D((4, 4), interpolation="bilinear")(y)
y = layers.Conv2D(64, (3, 3), padding='same')(y)
y = layers.BatchNormalization(axis=-1)(
y) # tfa.layers.GroupNormalization()
y = layers.Activation("relu")(y)
# REGRESSION
stacked_many_heads_output = None
for i in range(num_heads): # get multiple heads working
output_reg = layers.Conv2D(3, (3, 3), padding='same')(y)
# output_masked_reg = layers.Multiply()([output_reg, mask])
if stacked_many_heads_output == None:
stacked_many_heads_output = output_reg
else:
stacked_many_heads_output = layers.Concatenate(axis=-1)(
[stacked_many_heads_output, output_reg])
# model = Model(inputs=[image_input, mask], outputs=stacked_many_heads_output)
model = Model(inputs=image_input, outputs=stacked_many_heads_output)
# model.summary()
return model
def block1(x,
filters,
kernel_size=3,
stride=1,
conv_shortcut=True,
name=None,
norm_use="bn",
use_deformable=False):
"""A residual block.
# Arguments
x: input tensor.
filters: integer, filters of the bottleneck layer.
kernel_size: default 3, kernel size of the bottleneck layer.
stride: default 1, stride of the first layer.
conv_shortcut: default True, use convolution shortcut if True,
otherwise identity shortcut.
name: string, block label.
# Returns
Output tensor for the residual block.
"""
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
if conv_shortcut is True:
shortcut = layers.Conv2D(4 * filters,
1,
strides=stride,
kernel_initializer='he_normal',
name=name + '_0_conv')(x)
shortcut = normalize_layer(shortcut,
norm_use=norm_use,
name=name + '_0_')
#shortcut = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_0_bn')(shortcut)
else:
shortcut = x
x = layers.Conv2D(filters,
1,
strides=stride,
name=name + '_1_conv',
kernel_initializer='he_normal')(x)
#x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_1_bn')(x)
x = normalize_layer(x, norm_use=norm_use, name=name + '_1_')
x = layers.Activation('relu', name=name + '_1_relu')(x)
if use_deformable:
x = DeformableConvLayer(filters=filters,
kernel_size=kernel_size,
strides=(1, 1),
padding="same",
num_deformable_group=filters // 8)(x)
else:
x = layers.Conv2D(filters,
kernel_size,
padding='SAME',
kernel_initializer='he_normal',
name=name + '_2_conv')(x)
x = normalize_layer(x, norm_use=norm_use, name=name + '_2_')
#x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_2_bn')(x)
x = layers.Activation('relu', name=name + '_2_relu')(x)
x = layers.Conv2D(4 * filters,
1,
name=name + '_3_conv',
kernel_initializer='he_normal')(x)
x = normalize_layer(x, norm_use=norm_use, name=name + '_3_')
#x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_3_bn')(x)
x = layers.Add(name=name + '_add')([shortcut, x])
x = layers.Activation('relu', name=name + '_out')(x)
return x
def transfer_model_7x7_14x14(backbone_model,
input_shape,
dims_list,
num_aspect_ratios,
wt_decay,
model_name='transfer-objdet-model-7x7-14x14'):
inputs = keras.Input(shape=input_shape)
intermediate_layer_model = keras.Model(
inputs=backbone_model.input,
outputs=backbone_model.get_layer('conv4_block6_out').output)
intermediate_output = intermediate_layer_model(inputs) #14
backbone_output = backbone_model(inputs) #7
x = layers.Conv2D(512, 1, padding='same',
kernel_regularizer=l2(wt_decay))(backbone_output) #7
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(1024, 3, padding='same',
kernel_regularizer=l2(wt_decay))(x) #7
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(512, 1, padding='same',
kernel_regularizer=l2(wt_decay))(x) #7
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(1024, 3, padding='same',
kernel_regularizer=l2(wt_decay))(x) #7
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(512, 1, padding='same',
kernel_regularizer=l2(wt_decay))(x) #7
x = layers.BatchNormalization()(x)
upsample = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(2048, 3, padding='same',
kernel_regularizer=l2(wt_decay))(upsample) #7
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
tens_7x7 = layers.Add()([x, backbone_output])
x = layers.Conv2D(256, 1, padding='same',
kernel_regularizer=l2(wt_decay))(upsample) #7
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2DTranspose(512, 5, strides=(2, 2), padding='same')(x) #14
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Concatenate()([x, intermediate_output])
x = layers.Conv2D(256, 1, padding='same',
kernel_regularizer=l2(wt_decay))(x) #14
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(512, 3, padding='same',
kernel_regularizer=l2(wt_decay))(x) #14
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(256, 1, padding='same',
kernel_regularizer=l2(wt_decay))(x) #14
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(512, 3, padding='same',
kernel_regularizer=l2(wt_decay))(x) #14
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(256, 1, padding='same',
kernel_regularizer=l2(wt_decay))(x) #14
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU(0.01)(x)
x = layers.Conv2D(512, 3, padding='same',
kernel_regularizer=l2(wt_decay))(x) #14
x = layers.BatchNormalization()(x)
tens_14x14 = layers.LeakyReLU(0.01)(x)
dim_tensor_map = {'7x7': tens_7x7, '14x14': tens_14x14}
# For each dimension, construct a predictions tensor. Accumulate them into a dictionary for keras to understand multiple labels.
preds_dict = {}
for dims in dims_list:
dimkey = '{}x{}'.format(*dims)
tens = dim_tensor_map[dimkey]
ar_preds = []
for _ in range(num_aspect_ratios):
objectness_preds = layers.Conv2D(
1, 1, kernel_regularizer=l2(wt_decay))(tens)
class_preds = layers.Conv2D(len(cat_list),
1,
kernel_regularizer=l2(wt_decay))(tens)
bbox_preds = layers.Conv2D(4, 1,
kernel_regularizer=l2(wt_decay))(tens)
ar_preds.append(layers.Concatenate()(
[objectness_preds, class_preds, bbox_preds]))
if num_aspect_ratios > 1:
predictions = layers.Concatenate()(ar_preds)
elif num_aspect_ratios == 1:
predictions = ar_preds[0]
predictions = layers.Reshape(
(*dims, num_aspect_ratios, 5 + len(cat_list)),
name=dimkey)(predictions)
preds_dict[dimkey] = predictions
model = keras.Model(inputs, preds_dict, name=model_name)
model.compile(optimizer=tf.keras.optimizers.Adam(1e-5), loss=custom_loss)
return model
Cname = [None] * 6
C = [None] * 6
for i in range(2, 6):
substr = 'conv' + str(i)
res = [x for x in layernames if substr in x]
Cname[i] = res[-1]
C[i] = basemodel.get_layer(name=res[-1])
P5 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c5p5')(C[5].output)
T1 = KL.UpSampling2D(size=(2, 2), name='fpn_p5upsampled')(P5)
T2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c4p4')(C[4].output)
P4 = KL.Add(name='fpn_p4add')([T1, T2])
T1 = KL.UpSampling2D(size=(2, 2), name='fpn_p4upsampled')(P4)
T2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c3p3')(C[3].output)
P3 = KL.Add(name='fpn_p3add')([T1, T2])
T1 = KL.UpSampling2D(size=(2, 2), name='fpn_p4upsampled')(P4)
T2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c3p3')(C[3].output)
P3 = KL.Add(name='fpn_p3add')([T1, T2])
T1 = KL.UpSampling2D(size=(2, 2), name='fpn_p3upsampled')(P3)
T2 = KL.Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1),
name='fpn_c2p2')(C[2].output)
P2 = KL.Add(name='fpn_p2add')([T1, T2])
oof = np.zeros(len(data))
train_preds = np.zeros(len(data))
models = []
cv = KFold(n_splits=5, random_state=6, shuffle=True)
for fold_, (train_idx, val_idx) in enumerate(cv.split(data, targets), 1):
print(f'Training with fold {fold_} started.')
input_layer = layers.Input(len(features))
layer_0 = layers.Dense(1000)(input_layer)
layer_1 = layers.Dense(1000, activation='sigmoid')(layer_0)
layer_2 = layers.Dense(1000, activation='relu')(layer_0)
layer_3 = layers.Dense(500, activation='sigmoid')(layer_1)
layer_4 = layers.Dense(500, activation='relu')(layer_2)
layer_5 = layers.Add()([layer_3, layer_4])
layer_6 = layers.Dense(300, activation='relu')(layer_5)
out_layer = layers.Dense(2, activation='softmax')(layer_6)
model = keras.Model(inputs=input_layer, outputs=out_layer)
model.compile(optimizer=keras.optimizers.Adam(0.0001),
loss=keras.losses.categorical_crossentropy,
metrics=[keras.metrics.AUC(name='auc'), 'acc'])
model.summary()
checkpoint = keras.callbacks.ModelCheckpoint(f"{fold_}model.hdf5",
monitor='val_auc',
verbose=0,
save_best_only=True,
mode='max')
def build(self, input_shape):
super().build(input_shape)
dense_units = self.units * self.num_heads # N*H
self.query_layer = layers.Dense(self.num_heads, name='query')
self.value_layer = layers.Dense(dense_units, name='value')
self.add = layers.Add()
def __init__(self, in_channels, out_channels, stride, dropout, name, trainable):
super(BasicBlock, self).__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.stride = stride
self.dropout = dropout
# name = name
self.trainable = trainable
self.bn1 = layers.BatchNormalization(
momentum=0.999,
trainable=self.trainable,
name=name+'_bn1'
)
self.relu1 = layers.LeakyReLU(alpha=0.1)
# self.relu1 = keras.activations.relu
self.conv1 = layers.Conv2D(
filters=self.out_channels,
kernel_size=3,
strides=self.stride,
padding='same',
use_bias=False,
kernel_initializer='he_normal',
# kernel_regularizer=regularizers.l2(config.WEIGHT_DECAY),
trainable=self.trainable,
name=name+'_conv1',
)
self.bn2 = layers.BatchNormalization(
momentum=0.999,
trainable=self.trainable,
name=name+'_bn2'
)
self.relu2 = layers.LeakyReLU(alpha=0.1)
# self.relu2 = keras.activations.relu
self.drop_layer = layers.Dropout(
rate=self.dropout,
trainable=self.trainable,
name=name+'_dropout',
)
self.conv2 = layers.Conv2D(
filters=self.out_channels,
kernel_size=3,
strides=1,
padding='same',
use_bias=False,
kernel_initializer='he_normal',
# kernel_regularizer=regularizers.l2(config.WEIGHT_DECAY),
trainable=self.trainable,
name=name+'_conv2',
)
if self.stride != 1 or self.in_channels != self.out_channels:
self.short_cut = layers.Conv2D(
filters=self.out_channels,
kernel_size=1,
strides=self.stride,
padding='same',
use_bias=False,
kernel_initializer='he_normal',
# kernel_regularizer=regularizers.l2(config.WEIGHT_DECAY),
trainable=self.trainable,
name=name+'_shortcut'
)
self.add = layers.Add(name=name+'_add')
def boundary_generator(input_img):
# per image standardization
img = layers.Lambda(
lambda x: tf.image.per_image_standardization(x))(input_img)
# downsample a bit
net = layers.Conv2D(
filters=16,
kernel_size=3,
strides=2,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(img)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
net = layers.Dropout(rate=self.dropout_rate)(net)
# (48 48 16)
net = layers.Conv2D(
filters=32,
kernel_size=3,
strides=2,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
net = layers.Dropout(rate=self.dropout_rate)(net)
# (24 24 32)
shortcut = net
net = layers.DepthwiseConv2D(
kernel_size=3,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Conv2D(
64,
kernel_size=1,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
# net = layers.Add()([shortcut, net])
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
net = layers.Dropout(rate=self.dropout_rate)(net)
shortcut = net
net = layers.DepthwiseConv2D(
kernel_size=3,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Conv2D(
128,
kernel_size=1,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
# net = layers.Add()([shortcut, net])
# net = layers.ReLU()(net)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
net = layers.Dropout(rate=self.dropout_rate)(net)
shortcut = net
net = layers.DepthwiseConv2D(
kernel_size=3,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Conv2D(
128,
kernel_size=1,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Add()([shortcut, net])
net = layers.ReLU()(net)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
net = layers.Dropout(rate=self.dropout_rate)(net)
shortcut = net
net = layers.DepthwiseConv2D(
kernel_size=3,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Conv2D(
128,
kernel_size=1,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Add()([shortcut, net])
net = layers.ReLU()(net)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
net = layers.Dropout(rate=self.dropout_rate)(net)
shortcut = net
net = layers.DepthwiseConv2D(
kernel_size=3,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Conv2D(
128,
kernel_size=1,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Add()([shortcut, net])
net = layers.ReLU()(net)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
net = layers.Dropout(rate=self.dropout_rate)(net)
shortcut = net
net = layers.DepthwiseConv2D(
kernel_size=3,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Conv2D(
128,
kernel_size=1,
strides=1,
padding='same',
kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
net = layers.Add()([shortcut, net])
# net = layers.ReLU()(net)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
net = layers.Dropout(rate=self.dropout_rate)(net)
# shortcut = net
# net = layers.DepthwiseConv2D(kernel_size=3, strides=1, padding='same',
# kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
# net = layers.Conv2D(128, kernel_size=1, strides=1, padding='same',
# kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
# net = layers.Add()([shortcut, net])
# net = layers.ReLU()(net)
# net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1, 2])(net)
# net = layers.Dropout(rate=self.dropout_rate)(net)
#
# shortcut = net
# net = layers.DepthwiseConv2D(kernel_size=3, strides=1, padding='same',
# kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
# net = layers.Conv2D(128, kernel_size=1, strides=1, padding='same',
# kernel_regularizer=tf.keras.regularizers.l1(0.01))(net)
# net = layers.Add()([shortcut, net])
# # net = layers.ReLU()(net)
# net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1, 2])(net)
# net = layers.Dropout(rate=self.dropout_rate)(net)
# transposes
net = layers.Conv2DTranspose(filters=64,
kernel_size=3,
strides=2,
padding='same',
output_padding=1,
use_bias=False)(net)
net = layers.BatchNormalization(momentum=0.99)(net)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
# net = layers.ReLU()(net)
net = layers.Conv2DTranspose(filters=32,
kernel_size=3,
strides=2,
padding='same',
output_padding=1,
use_bias=False)(net)
net = layers.BatchNormalization(momentum=0.99)(net)
net = layers.PReLU(alpha_initializer='zeros', shared_axes=[1,
2])(net)
out = layers.Conv2D(self.num_boundaries,
kernel_size=1,
strides=1,
padding='same')(net)
# out = kernel_convolution(out)
return out
def ResNetModel(Img, ImageSize, MiniBatchSize):
"""
Inputs:
Img is a MiniBatch of the current image
ImageSize - Size of the Image
Outputs:
prLogits - logits output of the network
prSoftMax - softmax output of the network
"""
#############################
# Fill your network here!
#############################
training = tf.compat.v1.placeholder_with_default(False, shape=(), name='training')
#Batch_norm 1
batch_norm1 = tf.layers.batch_normalization(Img, training=training, momentum=0.9)
# RelU layer 1
layer_relu1 = new_relu_layer(batch_norm1, name="relu1")
# Convolutional Layer 1
layer_conv1, weights_conv1 = new_conv_layer(layer_relu1, num_input_channels=3, filter_size=5, num_filters=20, name ="conv1")
#Batch_norm 2
batch_norm2 = tf.layers.batch_normalization(layer_conv1, training=training, momentum=0.9)
# RelU layer 2
layer_relu2 = new_relu_layer(batch_norm2, name="relu2")
# Convolutional Layer 2
layer_conv2, weights_conv2 = new_conv_layer(input=layer_relu2, num_input_channels=20, filter_size=3, num_filters=40, name= "conv2")
#Skip-Connection 1
#paddings = tf.constant([[0, 0], [8, 8], [8, 8], [0, 0]])
#batch_norm1pad = tf.pad(batch_norm1, paddings, "CONSTANT")
unit_conv1, weights_unit_conv1 = new_conv_layer(input=Img, num_input_channels=3, filter_size=1, num_filters=40, name ="unitconv1")
skip_conn1 = layers.Add()([unit_conv1, layer_conv2])
#Batch_norm 3
batch_norm3 = tf.layers.batch_normalization(skip_conn1, training=training, momentum=0.9)
# RelU layer 3
layer_relu3 = new_relu_layer(batch_norm3, name="relu3")
# Convolutional Layer 3
layer_conv3, weights_conv3 = new_conv_layer(input=layer_relu3, num_input_channels=40, filter_size=3, num_filters=80, name= "conv3")
#Batch_norm 4
batch_norm4 = tf.layers.batch_normalization(layer_conv3, training=training, momentum=0.9)
# RelU layer 4
layer_relu4 = new_relu_layer(batch_norm4, name="relu3")
# Convolutional Layer 4
layer_conv4, weights_conv4 = new_conv_layer(input=layer_relu4, num_input_channels=80, filter_size=3, num_filters=100, name= "conv4")
#Skip-Connection 2
unit_conv2, weights_unit_conv2 = new_conv_layer(input=skip_conn1, num_input_channels=40, filter_size=1, num_filters=100, name ="unitconv2")
skip_conn2 = layers.Add()([unit_conv2, layer_conv4])
#Batch_norm 5
batch_norm5 = tf.layers.batch_normalization(skip_conn2, training=training, momentum=0.9)
# RelU layer 5
layer_relu5 = new_relu_layer(batch_norm5, name="relu5")
# Pooling Layer 1
layer_pool1 = new_pool_layer(layer_relu5, name="pool1")
# Flatten Layer
num_features = layer_pool1.get_shape()[1:4].num_elements()
layer_flat = tf.reshape(layer_pool1, [-1, num_features])
# Fully-Connected Layer 1
layer_fc1 = new_fc_layer(layer_flat, num_inputs=num_features, num_outputs=10, name="fc1")
# Use Softmax function to normalize the output
with tf.compat.v1.variable_scope("Softmax"):
y_pred = tf.nn.softmax(layer_fc1)
y_pred_cls = tf.argmax(y_pred, axis=1)
prLogits = layer_fc1
prSoftMax = y_pred
return prLogits, prSoftMax
def dumbelXL():
inL = layers.Input(shape=(256, 256, 2))
conv01 = cl.convBlock(32, 3)(inL)
conv02 = cl.convBlock(32, 3)(conv01)
sConv01 = cl.convBlock(64, 3, 2)(conv02)
conv03 = cl.convBlock(64, 3)(sConv01)
conv04 = cl.convBlock(64, 3)(conv03)
sConv02 = cl.convBlock(128, 3, 2)(conv04)
conv05 = cl.convBlock(128, 3)(sConv02)
conv06 = cl.convBlock(128, 3)(conv05)
sConv03 = cl.convBlock(256, 3, 2)(conv06)
conv07 = cl.convBlock(256, 3)(sConv03)
conv08 = cl.convBlock(256, 3)(conv07)
sConv04 = cl.convBlock(512, 3, 2)(conv08)
res01 = cl.resBlock(512, 3)(sConv04)
res02 = cl.resBlock(512, 3)(res01)
res03 = cl.resBlock(512, 3)(res02)
res04 = cl.resBlock(512, 3)(res03)
res05 = cl.resBlock(512, 3)(res04)
res06 = cl.resBlock(512, 3)(res05)
tConv01 = cl.convTransBlock(256, 3, 2)(res06)
add01 = layers.Add()([tConv01, conv08])
conv09 = cl.convBlock(256, 3)(add01)
conv10 = cl.convBlock(256, 3)(conv09)
tConv02 = cl.convTransBlock(128, 3, 2)(conv10)
add02 = layers.Add()([tConv02, conv06])
conv11 = cl.convBlock(128, 3)(add02)
conv12 = cl.convBlock(128, 3)(conv11)
tConv03 = cl.convTransBlock(64, 3, 2)(conv12)
add03 = layers.Add()([tConv03, conv04])
conv13 = cl.convBlock(64, 3)(add03)
conv14 = cl.convBlock(64, 3)(conv13)
tConv04 = cl.convTransBlock(32, 3, 2)(conv14)
add04 = layers.Add()([tConv04, conv02])
conv15 = cl.convBlock(32, 3)(add04)
conv16 = cl.convBlock(32, 3)(conv15)
synt = layers.Conv2D(2, 3, padding="same", use_bias=False)(conv16)
model = models.Model(inputs=[inL], outputs=[synt])
return(model)
def SVBRDF(num_classes):
#=============== first layer ==================
inputs = keras.Input(shape=(256, 256) + (3, ))
x = layers.LeakyReLU()(inputs)
GF = layers.AveragePooling2D((x.shape[1], x.shape[1]))(x)
GF = layers.Dense(128)(GF)
GF = layers.Activation('selu')(GF)
x = layers.SeparableConv2D(128, 4, 2, padding="same")(x)
#previous_block_activation = x # Set aside residual
#========== define filters for unet ===================
downfilters = np.array([128, 256, 512, 512, 512, 512, 512, 512])
Upfilters = np.flip(np.copy(downfilters))
downfilters = np.delete(downfilters, 0)
#print(downfilters)
prefilter = 128
#===================== upsampling =======================
for filters in downfilters:
#print(x.shape)
#print(filters)
GFdown = layers.AveragePooling2D((x.shape[1], x.shape[1]))(x)
GFup = layers.Dense(prefilter)(x)
GF = layers.Concatenate()([GF, GFdown])
GF = layers.Dense(filters)(GF)
GF = layers.Activation('selu')(GF)
x = layers.Add()([x, GFup])
x = layers.LeakyReLU()(x)
x = layers.SeparableConv2D(filters, 4, 2, padding="same")(x)
prefilter = filters
#x = layers.BatchNormalization()(x)
# Project residual
#residual = layers.Conv2D(filters, 4,2, padding="same")(previous_block_activation)
#x = layers.add([x, residual]) # Add back residual
#previous_block_activation = x # Set aside next residual
#====================== downsampling ============================
for filters in Upfilters:
GFdown = layers.AveragePooling2D((x.shape[1], x.shape[1]))(x)
GFup = layers.Dense(prefilter)(x)
GF = layers.Concatenate()([GF, GFdown])
GF = layers.Dense(filters)(GF)
GF = layers.Activation('selu')(GF)
x = layers.Add()([x, GFup])
x = layers.LeakyReLU()(x)
x = layers.Conv2DTranspose(filters, 4, 2, padding="same")(x)
prefilter = filters
# Project residual
#residual = layers.UpSampling2D(2)(previous_block_activation)
#residual = layers.Conv2D(filters, 4, padding="same")(residual)
#x = layers.add([x, residual]) # Add back residual
#previous_block_activation = x # Set aside next residual
#====================== last connection =====================
GFup = layers.Dense(prefilter)(x)
x = layers.Add()([x, GFup])
outputs = layers.Conv2D(num_classes,
3,
activation="softmax",
padding="same")(x)
model = keras.Model(inputs, outputs)
return model
def get_generator_model(name="generator",
num_blocks=20,
num_filters=32,
input_dtypes=None,
output_dtype=None):
"""
Create generator model.
Inputs:
- input_image (N x H x W x 3) - input frame
- pre_warp (N x H * 2 x W * 2 x 3) - warped previously generated frame
Outputs:
- (N x H * 2 x W * 2 x 3) - upscaled frame
Parameters
----------
name: str
Model name
num_blocks: int
Number of residual blocks
num_filters: int
Number of filters
input_dtypes : array of str or None
Data types for input images and warped previous images
output_dtype : str or None
Output dtype
Returns
-------
keras.Model
Model
"""
images = keras.Input(shape=[None, None, 3],
name="input_image",
dtype=input_dtypes[0] if input_dtypes else None)
pre_warp = keras.Input(shape=[None, None, 3],
name="pre_warp",
dtype=input_dtypes[1] if input_dtypes else None)
inputs = layers.Concatenate()([
images,
layers.Lambda(lambda x: tf.nn.space_to_depth(x, 2))(pre_warp)
])
net = layers.Conv2D(filters=num_filters,
kernel_size=3,
strides=1,
padding="SAME",
use_bias=False)(inputs)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU()(net)
def res_block(x): # pylint: disable=invalid-name
shortcut = x
x = layers.Conv2D(filters=num_filters,
kernel_size=3,
strides=1,
padding="SAME",
use_bias=False)(x)
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU()(x)
x = layers.Conv2D(filters=num_filters,
kernel_size=3,
strides=1,
padding="SAME",
use_bias=False)(x)
x = layers.BatchNormalization()(x)
x = layers.Add()([x, shortcut])
x = layers.LeakyReLU()(x)
return x
for _ in range(num_blocks):
net = res_block(net)
net = layers.Conv2DTranspose(filters=num_filters,
kernel_size=3,
strides=2,
padding="SAME")(net)
net = layers.LeakyReLU()(net)
net = layers.Conv2D(filters=3, kernel_size=3, strides=1,
padding="SAME")(net)
net = layers.Activation(K.tanh)(net)
upscaled = UpscaleLayer(dtype="float32")(images)
output = layers.Add(dtype=output_dtype)([upscaled, net])
model = keras.Model(inputs=[images, pre_warp], outputs=output, name=name)
return model
def create_tabtransformer_classifier(
num_transformer_blocks,
num_heads,
embedding_dims,
mlp_hidden_units_factors,
dropout_rate,
use_column_embedding=False,
):
# Create model inputs.
inputs = create_model_inputs()
# encode features.
encoded_categorical_feature_list, numerical_feature_list = encode_inputs(
inputs, embedding_dims)
# Stack categorical feature embeddings for the Tansformer.
encoded_categorical_features = tf.stack(encoded_categorical_feature_list,
axis=1)
# Concatenate numerical features.
numerical_features = layers.concatenate(numerical_feature_list)
# Add column embedding to categorical feature embeddings.
if use_column_embedding:
num_columns = encoded_categorical_features.shape[1]
column_embedding = layers.Embedding(input_dim=num_columns,
output_dim=embedding_dims)
column_indices = tf.range(start=0, limit=num_columns, delta=1)
encoded_categorical_features = encoded_categorical_features + column_embedding(
column_indices)
# Create multiple layers of the Transformer block.
for block_idx in range(num_transformer_blocks):
# Create a multi-head attention layer.
attention_output = layers.MultiHeadAttention(
num_heads=num_heads,
key_dim=embedding_dims,
dropout=dropout_rate,
name=f"multihead_attention_{block_idx}",
)(encoded_categorical_features, encoded_categorical_features)
# Skip connection 1.
x = layers.Add(name=f"skip_connection1_{block_idx}")(
[attention_output, encoded_categorical_features])
# Layer normalization 1.
x = layers.LayerNormalization(name=f"layer_norm1_{block_idx}",
epsilon=1e-6)(x)
# Feedforward.
feedforward_output = create_mlp(
hidden_units=[embedding_dims],
dropout_rate=dropout_rate,
activation=keras.activations.gelu,
normalization_layer=layers.LayerNormalization(epsilon=1e-6),
name=f"feedforward_{block_idx}",
)(x)
# Skip connection 2.
x = layers.Add(name=f"skip_connection2_{block_idx}")(
[feedforward_output, x])
# Layer normalization 2.
encoded_categorical_features = layers.LayerNormalization(
name=f"layer_norm2_{block_idx}", epsilon=1e-6)(x)
# Flatten the "contextualized" embeddings of the categorical features.
categorical_features = layers.Flatten()(encoded_categorical_features)
# Apply layer normalization to the numerical features.
numerical_features = layers.LayerNormalization(
epsilon=1e-6)(numerical_features)
# Prepare the input for the final MLP block.
features = layers.concatenate([categorical_features, numerical_features])
# Compute MLP hidden_units.
mlp_hidden_units = [
factor * features.shape[-1] for factor in mlp_hidden_units_factors
]
# Create final MLP.
features = create_mlp(
hidden_units=mlp_hidden_units,
dropout_rate=dropout_rate,
activation=keras.activations.selu,
normalization_layer=layers.BatchNormalization(),
name="MLP",
)(features)
# Add a sigmoid as a binary classifer.
outputs = layers.Dense(units=1, activation="sigmoid",
name="sigmoid")(features)
model = keras.Model(inputs=inputs, outputs=outputs)
return model
name='conv_layer2',
padding='same')(conv_layer1)
# add1=layers.Add()([ concat_layer,conv_layer2 ])
conv_layer3 = layers.Conv2D(filters=64,
kernel_size=(3, 3),
activation='relu',
name='conv_layer3',
padding='same')(conv_layer2)
conv_layer4 = layers.Conv2D(filters=64,
kernel_size=(3, 3),
activation='relu',
name='conv_layer4',
padding='same')(conv_layer3)
add2 = layers.Add()([conv_layer2, conv_layer4])
conv_layer5 = layers.Conv2D(filters=64,
kernel_size=(3, 3),
activation='relu',
name='conv_layer5',
padding='same')(add2)
conv_layer6 = layers.Conv2D(filters=64,
kernel_size=(3, 3),
activation='relu',
name='conv_layer6',
padding='same')(conv_layer5)
add3 = layers.Add()([conv_layer4, conv_layer6])
conv_layer7 = layers.Conv2D(filters=64,
kernel_size=(3, 3),
activation='relu',
def residual_dense_unit(x, units, dropout=0.45):
_x = dense_unit(x, units, dropout=dropout)
x = KL.Add()([x, _x])
# if dropout > 0:
# x = KL.Dropout(dropout)(x)
return x
def __init__(self,
inshape,
pheno_input_shape,
nb_unet_features=None,
src_feats=1,
conv_image_shape=None,
conv_size=3,
conv_nb_levels=0,
conv_nb_features=32,
extra_conv_layers=3,
use_mean_stream=True,
mean_cap=100,
templcondsi=False,
templcondsi_init=None,
**kwargs):
"""
Parameters:
inshape: Input shape. e.g. (192, 192, 192)
pheno_input_shape: Pheno data input shape. e.g. (2)
nb_unet_features: Unet convolutional features. See VxmDense documentation for more information.
src_feats: Number of source (atlas) features. Default is 1.
conv_image_shape: Intermediate phenotype image shape. Default is inshape with conv_nb_features.
conv_size: Atlas generator convolutional kernel size. Default is 3.
conv_nb_levels: Number of levels in atlas generator unet. Default is 0.
conv_nb_features: Number of features in atlas generator convolutions. Default is 32.
extra_conv_layers: Number of extra convolutions after unet in atlas generator. Default is 3.
use_mean_stream: Return mean stream layer for training. Default is True.
mean_cap: Cap for mean stream. Default is 100.
templcondsi: Default is False.
templcondsi_init: Default is None.
kwargs: Forwarded to the internal VxmDense model.
"""
if conv_image_shape is None:
conv_image_shape = (*inshape, conv_nb_features)
# build initial dense pheno to image shape model
pheno_input = KL.Input(pheno_input_shape, name='pheno_input')
pheno_dense = KL.Dense(np.prod(conv_image_shape),
activation='elu')(pheno_input)
pheno_reshaped = KL.Reshape(conv_image_shape,
name='pheno_reshape')(pheno_dense)
pheno_init_model = tf.keras.models.Model(pheno_input, pheno_reshaped)
# build model to decode reshaped pheno
pheno_decoder_model = ne.models.conv_dec(
conv_nb_features,
conv_image_shape,
conv_nb_levels,
conv_size,
nb_labels=conv_nb_features,
final_pred_activation='linear',
input_model=pheno_init_model,
name='atlas_decoder')
# add extra convolutions
Conv = getattr(KL, 'Conv%dD' % len(inshape))
last = pheno_decoder_model.output
for n in range(extra_conv_layers):
last = Conv(conv_nb_features,
kernel_size=conv_size,
padding='same',
name='atlas_extra_conv_%d' % n)(last)
# final convolution to get atlas features
atlas_gen = Conv(src_feats,
kernel_size=3,
padding='same',
name='atlas_gen',
kernel_initializer=KI.RandomNormal(mean=0.0,
stddev=1e-7),
bias_initializer=KI.RandomNormal(mean=0.0,
stddev=1e-7))(last)
# image input layers
atlas_input = tf.keras.Input((*inshape, src_feats), name='atlas_input')
source_input = tf.keras.Input((*inshape, src_feats),
name='source_input')
if templcondsi:
atlas_tensor = KL.Add(name='atlas_tmp')([atlas_input, pout])
# change first channel to be result from seg with another add layer
tmp_layer = KL.Lambda(lambda x: K.softmax(x[..., 1:]))(
atlas_tensor)
conv_layer = Conv(1,
kernel_size=1,
padding='same',
use_bias=False,
name='atlas_gen',
kernel_initializer=KI.RandomNormal(mean=0,
stddev=1e-5))
x_img = conv_layer(tmp_layer)
if templcondsi_init is not None:
weights = conv_layer.get_weights()
weights[0] = templcondsi_init.reshape(weights[0].shape)
conv_layer.set_weights(weights)
atlas_tensor = KL.Lambda(
lambda x: K.concatenate([x[0], x[1][..., 1:]]),
name='atlas')([x_img, atlas_tensor])
else:
atlas_tensor = KL.Add(name='atlas')([atlas_input, atlas_gen])
# build complete pheno to atlas model
pheno_model = tf.keras.models.Model(
[pheno_decoder_model.input, atlas_input], atlas_tensor)
inputs = [pheno_decoder_model.input, atlas_input, source_input]
warp_input_model = tf.keras.Model(inputs=inputs,
outputs=[atlas_tensor, source_input])
# warp model
vxm_model = VxmDense(inshape,
nb_unet_features=nb_unet_features,
bidir=True,
input_model=warp_input_model,
**kwargs)
# extract tensors from stacked model
y_source = vxm_model.references.y_source
pos_flow = vxm_model.references.pos_flow
neg_flow = vxm_model.references.neg_flow
if use_mean_stream:
# get mean stream from negative flow
mean_stream = ne.layers.MeanStream(name='mean_stream',
cap=mean_cap)(neg_flow)
outputs = [y_source, mean_stream, pos_flow, pos_flow]
else:
outputs = [y_source, pos_flow, pos_flow]
# initialize the keras model
super().__init__(inputs=inputs, outputs=outputs)