def model(hparams, X, model_name='autoencoder1') :
tensor = layers.Conv2D(4, (3, 3), activation='relu', padding='same')(X)
# (64, 64, 4)
tensor = layers.MaxPooling2D((2, 2), padding='same')(tensor)
# (32, 32, 4)
tensor = layers.Conv2D(1, (3, 3), activation='relu', padding='same')(tensor)
# (32, 32, 1)
tensor = layers.MaxPooling2D((2, 2), padding='same')(tensor)
# (16, 16, 1)
for_reconstruct = layers.Flatten(name='embeddings')(tensor)
pold = layers.Reshape((16, 16, 1))(for_reconstruct)
tensor = layers.UpSampling2D((2, 2))(pold)
# (32, 32, 1)
tensor = layers.Deconvolution2D(4, 3, 3, activation='relu', border_mode='same')(tensor)
tensor = layers.BatchNormalization(momentum=0.9)(tensor)
# (32, 32, 4)
tensor = layers.UpSampling2D((2, 2))(tensor)
# (64, 64, 4)
tensor = layers.Deconvolution2D(1, 3, 3, activation='sigmoid', border_mode='same')(tensor)
# (64, 64, 1)
model = Model(X, tensor)
return model
def BaseCapsNet(input_shape, n_class, routings):
# assemble encoder
x = Input(shape=input_shape)
l = x
l = Conv2D(256, (9, 9), strides=(2, 2), activation='relu',
padding="same")(l)
l = BatchNormalization()(l)
l = Conv2D(256, (9, 9), strides=(2, 2), activation='relu',
padding="same")(l)
l = BatchNormalization()(l)
l = ConvertToCaps()(l)
l = Conv2DCaps(16,
6,
kernel_size=(3, 3),
strides=(2, 2),
r_num=1,
b_alphas=[1, 1, 1])(l)
l = FlattenCaps()(l)
digits_caps = CapsuleLayer(num_capsule=10,
dim_capsule=8,
routings=routings,
channels=0,
name='digit_caps')(l)
l = CapsToScalars(name='capsnet')(digits_caps)
m_capsnet = models.Model(inputs=x, outputs=l, name='capsnet_model')
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digits_caps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digits_caps)
# Decoder Network
decoder = models.Sequential(name='decoder')
decoder.add(Dense(input_dim=80, activation="relu", output_dim=8 * 8 * 16))
decoder.add(Reshape((8, 8, 16)))
decoder.add(BatchNormalization(momentum=0.8))
decoder.add(
layers.Deconvolution2D(64, 3, 3, subsample=(1, 1), border_mode='same'))
decoder.add(
layers.Deconvolution2D(32, 3, 3, subsample=(2, 2), border_mode='same'))
decoder.add(
layers.Deconvolution2D(16, 3, 3, subsample=(2, 2), border_mode='same'))
decoder.add(
layers.Deconvolution2D(3, 3, 3, subsample=(1, 1), border_mode='same'))
decoder.add(Activation("relu"))
decoder.add(layers.Reshape(target_shape=(32, 32, 3), name='out_recon'))
train_model = models.Model(
[x, y], [m_capsnet.output, decoder(masked_by_y)])
eval_model = models.Model(x, [m_capsnet.output, decoder(masked)])
train_model.summary()
return train_model, eval_model
def unet():
"unet for image reconstruction"
model_input = KL.Input(shape=(None,None,4),name="u_net_input")
conv1 = KL.Conv2D(48, (3, 3), strides=(1, 1), name='conv1', use_bias=True,padding="same")(model_input)
conv1 = KL.LeakyReLU(alpha=0.2)(conv1)
conv1a = KL.Conv2D(48, (3, 3), strides=(1, 1), name='conv1a', use_bias=True,padding="same")(conv1)
conv1a = KL.LeakyReLU(alpha=0.2)(conv1a)
P1 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_1")(conv1a)
conv2 = KL.Conv2D(96, (3, 3), strides=(1, 1), name='conv2', use_bias=True,padding="same")(P1)
conv2 = KL.LeakyReLU(alpha=0.2)(conv2)
conv2a = KL.Conv2D(96, (3, 3), strides=(1, 1), name='conv2a', use_bias=True,padding="same")(conv2)
conv2a = KL.LeakyReLU(alpha=0.2)(conv2a)
P2 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_2")(conv2a)
conv3 = KL.Conv2D(192, (3, 3), strides=(1, 1), name='conv3', use_bias=True,padding="same")(P2)
conv3 = KL.LeakyReLU(alpha=0.2)(conv3)
conv3a = KL.Conv2D(192, (3, 3), strides=(1, 1), name='conv3a', use_bias=True,padding="same")(conv3)
conv3a = KL.LeakyReLU(alpha=0.2)(conv3a)
P3 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_3")(conv3a)
conv4 = KL.Conv2D(384, (3, 3), strides=(1, 1), name='conv4', use_bias=True,padding="same")(P3)
conv4 = KL.LeakyReLU(alpha=0.2)(conv4)
conv4a = KL.Conv2D(384, (3, 3), strides=(1, 1), name='conv4a', use_bias=True,padding="same")(conv4)
conv4a = KL.LeakyReLU(alpha=0.2)(conv4a)
P4 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_4")(conv4a)
conv5 = KL.Conv2D(768, (3, 3), strides=(1, 1), name='conv5', use_bias=True,padding="same")(P4)
conv5 = KL.LeakyReLU(alpha=0.2)(conv5)
conv5a = KL.Conv2D(768, (3, 3), strides=(1, 1), name='conv5a', use_bias=True,padding="same")(conv5)
conv5a = KL.LeakyReLU(alpha=0.2)(conv5a)
P5 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_5")(conv5a)
up4 = KL.Deconvolution2D(nb_filter=384, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv4',border_mode='same')(P5)
up4 = KL.LeakyReLU(alpha=0.2)(up4)
C4 = KL.Concatenate()([P4,up4])
conv4_u = KL.Conv2D(384, (3, 3), strides=(1, 1), name='conv4_u', use_bias=True,padding="same")(C4)
conv4_u = KL.LeakyReLU(alpha=0.2)(conv4_u)
conv4a_u = KL.Conv2D(384, (3, 3), strides=(1, 1), name='conv4a_u', use_bias=True,padding="same")(conv4_u)
conv4a_u = KL.LeakyReLU(alpha=0.2)(conv4a_u)
up3 = KL.Deconvolution2D(nb_filter=192, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv3',border_mode='same')(conv4a_u)
up3 = KL.LeakyReLU(alpha=0.2)(up3)
C3 = KL.Concatenate()([P3,up3])
conv3_u = KL.Conv2D(192, (3, 3), strides=(1, 1), name='conv3_u', use_bias=True,padding="same")(C3)
conv3_u = KL.LeakyReLU(alpha=0.2)(conv3_u)
conv3a_u = KL.Conv2D(192, (3, 3), strides=(1, 1), name='conv3a_u', use_bias=True,padding="same")(conv3_u)
conv3a_u = KL.LeakyReLU(alpha=0.2)(conv3a_u)
up2 = KL.Deconvolution2D(nb_filter=96, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv2',border_mode='same')(conv3a_u)
up2 = KL.LeakyReLU(alpha=0.2)(up2)
C2 = KL.Concatenate()([P2,up2])
conv2_u = KL.Conv2D(96, (3, 3), strides=(1, 1), name='conv2_u', use_bias=True,padding="same")(C2)
conv2_u = KL.LeakyReLU(alpha=0.2)(conv2_u)
conv2a_u = KL.Conv2D(96, (3, 3), strides=(1, 1), name='conv2a_u', use_bias=True,padding="same")(conv2_u)
conv2a_u = KL.LeakyReLU(alpha=0.2)(conv2a_u)
up1 = KL.Deconvolution2D(nb_filter=48, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv1',border_mode='same')(conv2a_u)
up1 = KL.LeakyReLU(alpha=0.2)(up1)
C1 = KL.Concatenate()([P1,up1])
conv1_u = KL.Conv2D(48, (3, 3), strides=(1, 1), name='conv1_u', use_bias=True,padding="same")(C1)
conv1_u = KL.LeakyReLU(alpha=0.2)(conv1_u)
conv1a_u = KL.Conv2D(48, (3, 3), strides=(1, 1), name='conv1a_u', use_bias=True,padding="same")(conv1_u)
conv1a_u = KL.LeakyReLU(alpha=0.2)(conv1a_u)
up0 = KL.Deconvolution2D(nb_filter=24, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv0',border_mode='same')(conv1a_u)
up0 = KL.LeakyReLU(alpha=0.2)(up0)
C0 = KL.Concatenate()([model_input,up0])
conv0_u = KL.Conv2D(24, (3, 3), strides=(1, 1), name='conv0_u', use_bias=True,padding="same")(C0)
conv0_u = KL.LeakyReLU(alpha=0.2)(conv0_u)
conv0a_u = KL.Conv2D(24, (3, 3), strides=(1, 1), name='conv0a_u', use_bias=True,padding="same")(conv0_u)
conv0a_u = KL.LeakyReLU(alpha=0.2)(conv0a_u)
x = KL.Conv2D(12,(1,1),strides=(1,1),name='convr',use_bias=True,padding="same")(conv0a_u)
x = KL.LeakyReLU(alpha=0.2)(x)
model_output = KL.Lambda(lambda t:tf.depth_to_space(t,2))(x)
model = KM.Model(inputs=model_input,outputs=model_output)
model.summary()
return model
def fcn8_graph(feature_map, config, mode=None):
'''Builds the computation graph of Region Proposal Network.
feature_map: Contextual Tensor [batch, num_classes, width, depth]
Returns:
'''
print()
print('---------------')
print('>>> FCN8 Layer - mode:', mode)
print('---------------')
batch_size = config.BATCH_SIZE
height, width = config.FCN_INPUT_SHAPE[0:2]
num_classes = config.NUM_CLASSES
rois_per_class = config.TRAIN_ROIS_PER_IMAGE
weight_decay = config.WEIGHT_DECAY
# In the original implementatoin , batch_momentum was used for batch normalization layers for the ResNet
# backbone. We are not using this backbone in FCN, therefore it is unused.
# batch_momentum = config.BATCH_MOMENTUM
verbose = config.VERBOSE
feature_map_shape = (width, height, num_classes)
print(' feature map :', feature_map.shape)
print(' height :', height, 'width :', width, 'classes :', num_classes)
print(' image_data_format: ', KB.image_data_format())
print(' rois_per_class : ', KB.image_data_format())
if mode == 'training':
KB.set_learning_phase(1)
else:
KB.set_learning_phase(0)
print(' Set learning phase to :', KB.learning_phase())
# feature_map = KL.Input(shape= feature_map_shape, name="input_fcn_feature_map")
# TODO: Assert proper shape of input [batch_size, width, height, num_classes]
# TODO: check if stride of 2 causes alignment issues if the featuremap is not even.
# if batch_shape:
# img_input = Input(batch_shape=batch_shape)
# image_size = batch_shape[1:3]
# else:
# img_input = Input(shape=input_shape)
# image_size = input_shape[0:2]
##-------------------------------------------------------------------------------------------------------
## Block 1 data_format='channels_last',
##-------------------------------------------------------------------------------------------------------
x = KL.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(feature_map)
print(' Input feature map : ', feature_map.shape)
logt('Input feature map ', feature_map, verbose=1)
logt('FCN Block 11 ', x, verbose=verbose)
x = KL.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 12 ', x, verbose=verbose)
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
logt('FCN Block 13 (Max pooling) ', x, verbose=verbose)
##-------------------------------------------------------------------------------------------------------
## Block 2
##-------------------------------------------------------------------------------------------------------
x = KL.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 21 ', x, verbose=verbose)
x = KL.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 22 ', x, verbose=verbose)
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
logt('FCN Block 23 (Max pooling) ', x, verbose=verbose)
##-------------------------------------------------------------------------------------------------------
## Block 3
##-------------------------------------------------------------------------------------------------------
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 31 ', x, verbose=verbose)
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 32 ', x, verbose=verbose)
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 33 ', x, verbose=verbose)
Pool3 = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
logt('FCN Block 34 (Max pooling) ', Pool3, verbose=verbose)
##-------------------------------------------------------------------------------------------------------
## Block 4
##-------------------------------------------------------------------------------------------------------
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(Pool3)
logt('FCN Block 41 ', x, verbose=verbose)
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 42 ', x, verbose=verbose)
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 43 ', x, verbose=verbose)
Pool4 = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
logt('FCN Block 44 (Max pooling) ', Pool4, verbose=verbose)
##-------------------------------------------------------------------------------------------------------
## Block 5
##-------------------------------------------------------------------------------------------------------
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(Pool4)
logt('FCN Block 51 ', x, verbose=verbose)
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 52 ', x, verbose=verbose)
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN Block 53 ', x, verbose=verbose)
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
logt('FCN Block 54 (Max pooling) ', x, verbose=verbose)
##-------------------------------------------------------------------------------------------------------
## FCN32 Specific Structure
##-------------------------------------------------------------------------------------------------------
# Convolutional layers transfered from fully-connected layers
# changed from 4096 to 2048 - reduction of weights from 42,752,644 to
# changed ftom 2048 to 1024 - 11-05-2018
# FC_SIZE = 2048
FC_SIZE = 4096
x = KL.Conv2D(FC_SIZE, (7, 7),
activation='relu',
padding='same',
name='fcn32_fc1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print()
print(' --- FCN32 ----------------------------')
logt(' FCN fully connected 1 (fc1) ', x, verbose=verbose)
x = KL.Dropout(0.5)(x)
x = KL.Conv2D(FC_SIZE, (1, 1),
activation='relu',
padding='same',
name='fcn32_fc2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
logt('FCN fully connected 2 (fc2) ', x, verbose=verbose)
x = KL.Dropout(0.5)(x)
# Classifying layer
x = KL.Conv2D(num_classes, (1, 1),
activation='linear',
padding='valid',
strides=(1, 1),
name='fcn32_deconv2D',
kernel_initializer='he_normal',
bias_initializer='zeros')(x)
logt('FCN conv2d (fcn32_deconv2D) ', x, verbose=verbose)
##-------------------------------------------------------------------------------------------------------
## FCN16 Specific Structure
##-------------------------------------------------------------------------------------------------------
# Score Pool4 - Reduce Pool4 filters from 512 to num_classes (81)
scorePool4 = KL.Conv2D(num_classes, (1, 1),
activation='relu',
padding='valid',
name='fcn16_score_pool4',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(Pool4)
print()
print(' --- FCN16 ----------------------------')
logt('FCN scorePool4 (Conv2D(Pool4)) ', scorePool4, verbose=verbose)
# 2x Upsampling of fcn_deconv2D to generate Score2 (padding was originally "valid")
x = KL.Deconvolution2D(num_classes,
kernel_size=(4, 4),
activation=None,
padding='valid',
name='fcn16_score2',
strides=(2, 2))(x)
logt('FCN 2x Upsampling (Deconvolution2D(fcn32_classify)) ',
x,
verbose=verbose)
# Crop to appropriate shape if required
score2_c = KL.Cropping2D(cropping=((1, 1), (1, 1)),
name='fcn16_crop_score2')(x)
logt('FCN 2x Upsampling/Cropped (Cropped2D(score2)) ',
score2_c,
verbose=verbose)
# Sum Score2, scorePool4
x = KL.Add(name='fcn16_fuse_pool4')([score2_c, scorePool4])
logt('FCN Add Score2,scorePool4 Add(score2_c, scorePool4) ',
x,
verbose=verbose)
# 2x Upsampling (padding was originally "valid", I changed it to "same" )
x = KL.Deconvolution2D(num_classes,
kernel_size=(4, 4),
activation=None,
padding='same',
name='fcn16_upscore_pool4',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
strides=(2, 2))(x)
logt('FCN upscore_pool4 (Deconv(fuse_Pool4)) ', x, verbose=verbose)
##-------------------------------------------------------------------------------------------------------
## FCN8 Specific Structure
##-------------------------------------------------------------------------------------------------------
# Score Pool3 - Reduce Pool3 filters from 256 to num_classes (81)
scorePool3 = KL.Conv2D(num_classes, (1, 1),
activation='relu',
padding='valid',
name='fcn8_score_pool3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(Pool3)
print()
print(' --- FCN8 ----------------------------')
logt('FCN scorePool3 (Conv2D(Pool3)) ', scorePool3, verbose=verbose)
upscore_pool4_c = KL.Cropping2D(cropping=((0, 0), (0, 0)),
name='fcn8_crop_pool4')(x)
logt('FCN 2x Upsampling/Cropped (Cropped2D(score2)) ',
upscore_pool4_c,
verbose=verbose)
# Sum upscore_pool4_c, scorePool3
x = KL.Add(name='fcn8_fuse_pool3')([upscore_pool4_c, scorePool3])
logt('FCN Add Score2,scorePool4', x, verbose=verbose)
print()
##-------------------------------------------------------------------------------------------------------
## fcn_heatmap
##-------------------------------------------------------------------------------------------------------
# 8x Upsampling (padding was originally "valid", I changed it to "same" )
fcn_hm = KL.Deconvolution2D(num_classes,
kernel_size=(16, 16),
activation=None,
padding='same',
name='fcn8_heatmap',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
strides=(8, 8))(x)
# fcn_hm = tf.identity(fcn_hm)
fcn_hm.set_shape(feature_map.shape)
logt('FCN fcn8_classify/heatmap (Deconv(fuse_Pool4)) ',
fcn_hm,
verbose=verbose)
fcn_hm = KL.Lambda(lambda z: tf.identity(z, name='fcn_hm'),
name='fcn_heatmap_lambda')(fcn_hm)
logt('fcn_hm (final)', fcn_hm, verbose=verbose)
print()
# fcn_classify_shape = KB.int_shape(fcn_hm)
# h_factor = height / fcn_classify_shape[1]
# w_factor = width / fcn_classify_shape[2]
# print(' fcn_classify_shape:',fcn_classify_shape,' h_factor : ', h_factor, ' w_factor : ', w_factor)
# x = BilinearUpSampling2D(size=(h_factor, w_factor), name='fcn_bilinear')(x)
# print(' FCN Bilinear upsmapling layer shape is : ' , KB.int_shape(x), ' Keras tensor ', KB.is_keras_tensor(x) )
##-------------------------------------------------------------------------------------------------------
## fcn_heatmap
##-------------------------------------------------------------------------------------------------------
fcn_sm = KL.Activation("softmax", name="fcn8_softmax")(fcn_hm)
logt('fcn8_softmax ', fcn_sm, verbose=verbose)
fcn_sm = KL.Lambda(lambda z: tf.identity(z, name='fcn_sm'),
name='fcn_softmax_lambda')(fcn_hm)
logt('fcn_sm (final)', fcn_sm, verbose=verbose)
print()
#---------------------------------------------------------------------------------------------
# heatmap L2 normalization
# Normalization using the `gauss_sum` (batchsize , num_classes, height, width)
# 17-05-2018 (New method, replace dthe previous method that usedthe transposed gauss sum
# 17-05-2018 Replaced with normalization across the CLASS axis
# normalize along the CLASS axis
#---------------------------------------------------------------------------------------------
# print('\n L2 normalization ------------------------------------------------------')
# fcn_hm_L2norm = KL.Lambda(lambda z: tf.nn.l2_normalize(z, axis = 3, name = 'fcn_heatmap_L2norm'),\
# name = 'fcn_heatmap_L2norm')(x)
# print('\n normalization ------------------------------------------------------')
# fcn_hm_norm = KL.Lambda(normalize, name="fcn_heatmap_norm") (x)
return fcn_hm, fcn_sm
def CapsNet(input_shape, n_class, routings):
"""
Defining the CapsNet
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
"""
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=64,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
conv2 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv2')(conv1)
conv3 = layers.Conv2D(filters=256,
kernel_size=3,
strides=2,
padding='valid',
activation='relu',
name='conv3')(conv2)
primarycaps = PrimaryCap(conv3,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
channels=32,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
"""
Decoder Network
"""
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y])
masked = Mask()(digitcaps)
decoder = models.Sequential(name='decoder')
decoder.add(
Dense(input_dim=16 * n_class, activation="relu",
output_dim=7 * 7 * 32))
decoder.add(Reshape((7, 7, 32)))
decoder.add(BatchNormalization(momentum=0.8))
decoder.add(
layers.Deconvolution2D(32,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(16,
3,
3,
subsample=(2, 2),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(8,
3,
3,
subsample=(2, 2),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(4,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(1,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="sigmoid"))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
"""
Models for training and evaluation (prediction)
"""
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def fcn16_graph(feature_map, config):
'''Builds the computation graph of Region Proposal Network.
feature_map: Contextual Tensor [batch, num_classes, width, depth]
Returns:
'''
print()
print('---------------')
print('>>> FCN16 Layer ')
print('---------------')
height, width = config.FCN_INPUT_SHAPE[0:2]
num_classes = config.NUM_CLASSES
rois_per_class = config.TRAIN_ROIS_PER_IMAGE
weight_decay = config.WEIGHT_DECAY
batch_momentum = config.BATCH_MOMENTUM
feature_map_shape = (width, height, num_classes)
print(' feature map :', feature_map.shape)
print(' height :', height, 'width :', width, 'classes :', num_classes)
print(' image_data_format: ', KB.image_data_format())
print(' rois_per_class : ', KB.image_data_format())
# feature_map = KL.Input(shape= feature_map_shape, name="input_fcn_feature_map")
# TODO: Assert proper shape of input [batch_size, width, height, num_classes]
# TODO: check if stride of 2 causes alignment issues if the featuremap is not even.
# if batch_shape:
# img_input = Input(batch_shape=batch_shape)
# image_size = batch_shape[1:3]
# else:
# img_input = Input(shape=input_shape)
# image_size = input_shape[0:2]
## , kernel_regularizer=l2(weight_decay)
## Block 1 data_format='channels_last',
x = KL.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(feature_map)
print(' FCN Block 11 shape is : ', x.get_shape())
x = KL.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 12 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
print(' FCN Block 13 (Max pooling - Pool1) shape is : ', x.get_shape())
x0 = x
## Block 2
x = KL.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 21 shape is : ', x.get_shape())
x = KL.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 22 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
print(' FCN Block 23 (Max pooling - Pool2) shape is : ', x.get_shape())
x1 = x
## Block 3
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 31 shape is : ', x.get_shape())
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 32 shape is : ', x.get_shape())
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 33 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
print(' FCN Block 34 (Max pooling - Pool3) shape is : ', x.get_shape())
## Block 4
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 41 shape is : ', x.get_shape())
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 42 shape is : ', x.get_shape())
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 43 shape is : ', x.get_shape())
Pool4 = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
print(' FCN Block 44 (Max pooling - Pool4) shape is : ',
Pool4.get_shape())
## Block 5
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(Pool4)
print(' FCN Block 51 shape is : ', x.get_shape())
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 52 shape is : ', x.get_shape())
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay))(x)
print(' FCN Block 53 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
print(' FCN Block 54 (Max pooling - Pool5) shape is : ', x.get_shape())
##-------------------------------------------------------------------------------------------------------
## FCN32 Specific Structure
##-------------------------------------------------------------------------------------------------------
# Convolutional layers transfered from fully-connected layers
# changed from 4096 to 2048 - reduction of weights from 42,752,644 to
# changed ftom 2048 to 1024 - 11-05-2018
FC_SIZE = 4096
x = KL.Conv2D(FC_SIZE, (7, 7),
activation='relu',
padding='same',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay),
name='fcn32_fc1')(x)
print(' FCN fully connected 1 (fcn_fc1) shape is : ', x.get_shape())
x = KL.Dropout(0.5)(x)
x = KL.Conv2D(FC_SIZE, (1, 1),
activation='relu',
padding='same',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay),
name='fcn32_fc2')(x)
print(' FCN fully connected 2 (fcn_fc2) shape is : ', x.get_shape())
x = KL.Dropout(0.5)(x)
#classifying layer
x = KL.Conv2D(num_classes, (1, 1),
kernel_initializer='he_normal',
bias_initializer='zeros',
activation='linear',
padding='valid',
strides=(1, 1),
kernel_regularizer=l2(weight_decay),
name='fcn32_deconv2D')(x)
print(' FCN conv2d (fcn32_deconv2D) shape is : ', x.get_shape(),
' keras_tensor ', KB.is_keras_tensor(x))
##-------------------------------------------------------------------------------------------------------
## FCN16 Specific Structure
##-------------------------------------------------------------------------------------------------------
# Score Pool4
scorePool4 = KL.Conv2D(num_classes, (1, 1),
activation="relu",
padding='valid',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=l2(weight_decay),
name="fcn16_score_pool4")(Pool4)
print()
print(' --- FCN16 ----------------------------')
print(' FCN scorePool4 (Conv2D(Pool4)) shape is : ',
KB.int_shape(scorePool4), ' keras_tensor ',
KB.is_keras_tensor(scorePool4))
# 2x Upsampling to generate Score2 (padding was originally "valid")
score2 = KL.Deconvolution2D(num_classes,
kernel_size=(4, 4),
strides=(2, 2),
padding="valid",
activation=None,
name="fcn16_score2")(x)
score2_c = KL.Cropping2D(cropping=((1, 1), (1, 1)),
name="fcn16_crop_score2")(score2)
print(' FCN 2x Upsampling (Deconvolution2D(fcn32_classify)) shape : ',
KB.int_shape(score2), ' keras_tensor ', KB.is_keras_tensor(score2))
print(' FCN 2x Upsampling/Cropped (Cropped2D(score2)) shape : ',
KB.int_shape(score2_c), ' keras_tensor ',
KB.is_keras_tensor(score2_c))
# Sum Score2, Pool4
x = KL.Add(name="fcn16_fuse_pool4")([score2_c, scorePool4])
print(' FCN Add Score2,scorePool4 Add(score2_c, scorePool4) shape : ',
KB.int_shape(x), ' keras_tensor ', KB.is_keras_tensor(x))
##-------------------------------------------------------------------------------------------------------
## fcn output heatmap
##-------------------------------------------------------------------------------------------------------
# Upsampling (padding was originally "valid", I changed it to "same" )
x = KL.Deconvolution2D(num_classes,
kernel_size=(32, 32),
strides=(16, 16),
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
padding="same",
activation=None,
name="fcn16_classify")(x)
print()
print(' FCN fcn16_classify (Deconv(fuse_Pool4)) shape is : ',
KB.int_shape(x), ' keras_tensor ', KB.is_keras_tensor(x))
fcn_classify_shape = KB.int_shape(x)
h_factor = height / fcn_classify_shape[1]
w_factor = width / fcn_classify_shape[2]
print(' fcn_classify_shape:', fcn_classify_shape, ' h_factor : ',
h_factor, ' w_factor : ', w_factor)
# x = BilinearUpSampling2D(size=(h_factor, w_factor), name='fcn_bilinear')(x)
# print(' FCN Bilinear upsmapling layer shape is : ' , KB.int_shape(x), ' Keras tensor ', KB.is_keras_tensor(x) )
fcn_hm = KL.Lambda(lambda z: tf.identity(z, name="fcn_heatmap"),
name="fcn_heatmap")(x)
print(' FCN fcn_heatmap Lambda(fcn32_classify) : ',
KB.int_shape(fcn_hm), ' Keras tensor ', KB.is_keras_tensor(fcn_hm))
print()
#---------------------------------------------------------------------------------------------
# heatmap L2 normalization
# Normalization using the `gauss_sum` (batchsize , num_classes, height, width)
# 17-05-2018 (New method, replace dthe previous method that usedthe transposed gauss sum
# 17-05-2018 Replaced with normalization across the CLASS axis
# normalize along the CLASS axis
#---------------------------------------------------------------------------------------------
# print('\n L2 normalization ------------------------------------------------------')
# fcn_hm_L2norm = KL.Lambda(lambda z: tf.nn.l2_normalize(z, axis = 3, name = 'fcn_heatmap_L2norm'),\
# name = 'fcn_heatmap_L2norm')(x)
# print('\n normalization ------------------------------------------------------')
# fcn_hm_norm = KL.Lambda(normalize, name="fcn_heatmap_norm") (x)
return fcn_hm # fcn_hm_norm, fcn_hm_L2norm
def fcn32_graph(feature_map, config, mode=None):
'''Builds the computation graph of Region Proposal Network.
feature_map: Contextual Tensor [batch, num_classes, width, depth]
Returns:
'''
print()
print('---------------')
print('>>> FCN32 Layer - mode:', mode)
print('---------------')
batch_size = config.BATCH_SIZE
height, width = config.FCN_INPUT_SHAPE[0:2]
num_classes = config.NUM_CLASSES
rois_per_class = config.TRAIN_ROIS_PER_IMAGE
weight_decay = config.WEIGHT_DECAY
batch_momentum = config.BATCH_MOMENTUM
verbose = config.VERBOSE
feature_map_shape = (width, height, num_classes)
print(' feature map :', feature_map.shape)
print(' height :', height, 'width :', width, 'classes :', num_classes)
print(' image_data_format: ', KB.image_data_format())
print(' rois_per_class : ', KB.image_data_format())
# feature_map = KL.Input(shape= feature_map_shape, name="input_fcn_feature_map")
# TODO: Assert proper shape of input [batch_size, width, height, num_classes]
# TODO: check if stride of 2 causes alignment issues if the featuremap is not even.
# if batch_shape:
# img_input = Input(batch_shape=batch_shape)
# image_size = batch_shape[1:3]
# else:
# img_input = Input(shape=input_shape)
# image_size = input_shape[0:2]
## , kernel_regularizer=l2(weight_decay)
# Block 1 data_format='channels_last',
x = KL.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(feature_map)
print(' FCN Block 11 shape is : ', x.get_shape())
x = KL.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 12 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
print(' FCN Block 13 shape is : ', x.get_shape())
x0 = x
# Block 2
x = KL.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 21 shape is : ', x.get_shape())
x = KL.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 22 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
print(' FCN Block 23 (Max pooling) shape is : ', x.get_shape())
x1 = x
# Block 3
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 31 shape is : ', x.get_shape())
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 32 shape is : ', x.get_shape())
x = KL.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 33 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
print(' FCN Block 34 (Max pooling) shape is : ', x.get_shape())
# Block 4
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 41 shape is : ', x.get_shape())
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 42 shape is : ', x.get_shape())
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 43 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
print(' FCN Block 44 (Max pooling) shape is : ', x.get_shape())
# Block 5
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 51 shape is : ', x.get_shape())
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 52 shape is : ', x.get_shape())
x = KL.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3',
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN Block 53 shape is : ', x.get_shape())
x = KL.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
print(' FCN Block 54 (Max pooling) shape is : ', x.get_shape())
##-------------------------------------------------------------------------------------------------------
## FCN32 Specific Structure
##-------------------------------------------------------------------------------------------------------
# Convolutional layers transfered from fully-connected layers
# changed from 4096 to 2048 - reduction of weights from 42,752,644 to
# changed ftom 2048 to 1024 - 11-05-2018
# FC_SIZE = 2048
FC_SIZE = 4096
x = KL.Conv2D(FC_SIZE, (7, 7),
activation='relu',
padding='same',
name="fc1",
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print()
print(' --- FCN32 ----------------------------')
print(' FCN fully connected 1 (fcn_fc1) shape is : ', KB.int_shape(x))
x = KL.Dropout(0.5)(x)
#fc2
x = KL.Conv2D(FC_SIZE, (1, 1),
activation='relu',
padding='same',
name="fc2",
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(x)
print(' FCN fully connected 2 (fcn_fc2) shape is : ', x.get_shape())
x = KL.Dropout(0.5)(x)
#classifying layer
x = KL.Conv2D(num_classes, (1, 1),
kernel_initializer='he_normal',
bias_initializer='zeros',
activation='linear',
padding='valid',
strides=(1, 1),
name="fcn_classify")(x)
print(' FCN final conv2d (fcn_classify) shape is : ', x.get_shape(),
' keras_tensor ', KB.is_keras_tensor(x))
fcn_classify_shape = KB.int_shape(x)
h_factor = height / fcn_classify_shape[1]
w_factor = height / fcn_classify_shape[2]
print(' h_factor : ', h_factor, 'w_factor : ', w_factor)
# x = BilinearUpSampling2D(size=(h_factor, w_factor), name='fcn_bilinear')(x)
# print(' FCN Bilinear upsmapling layer shape is : ' , x.get_shape(), ' Keras tensor ', KB.is_keras_tensor(x) )
##-------------------------------------------------------------------------------------------------------
## fcn_heatmap
##-------------------------------------------------------------------------------------------------------
# 8x Upsampling (padding was originally "valid", I changed it to "same" )
fcn_hm = KL.Deconvolution2D(num_classes,
kernel_size=(16, 16),
strides=(32, 32),
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
padding='same',
activation=None,
name="fcn8_heatmap")(x)
# fcn_hm = tf.identity(fcn_hm)
fcn_hm.set_shape(feature_map.shape)
logt('FCN fcn8_classify/heatmap (Deconv(fuse_Pool4)) ',
fcn_hm,
verbose=verbose)
fcn_hm = KL.Lambda(lambda z: tf.identity(z, name='fcn_hm'),
name='fcn_heatmap_lambda')(fcn_hm)
logt('fcn_hm (final)', fcn_hm, verbose=verbose)
print()
##-------------------------------------------------------------------------------------------------------
## fcn_SOFTMAX
##-------------------------------------------------------------------------------------------------------
fcn_sm = KL.Activation("softmax", name="fcn8_softmax")(fcn_hm)
logt('fcn8_softmax ', fcn_sm, verbose=verbose)
fcn_sm = KL.Lambda(lambda z: tf.identity(z, name='fcn_sm'),
name='fcn_softmax_lambda')(fcn_hm)
logt('fcn_sm (final)', fcn_sm, verbose=verbose)
print()
#---------------------------------------------------------------------------------------------
# heatmap L2 normalization
# Normalization using the `gauss_sum` (batchsize , num_classes, height, width)
# 17-05-2018 (New method, replace dthe previous method that usedthe transposed gauss sum
# 17-05-2018 Replaced with normalization across the CLASS axis
# normalize along the CLASS axis
#---------------------------------------------------------------------------------------------
# print('\n L2 normalization ------------------------------------------------------')
# fcn_hm_L2norm = KL.Lambda(lambda z: tf.nn.l2_normalize(z, axis = 3, name = 'fcn_heatmap_L2norm'),\
# name = 'fcn_heatmap_L2norm')(x)
# print('\n normalization ------------------------------------------------------')
# fcn_hm_norm = KL.Lambda(normalize, name="fcn_heatmap_norm") (x)
print(' fcn_heatmap : ', fcn_hm.shape, ' Keras tensor ',
KB.is_keras_tensor(fcn_hm))
# print(' fcn_heatmap_norm : ', fcn_hm_norm.shape ,' Keras tensor ', KB.is_keras_tensor(fcn_hm_norm) )
# print(' fcn_heatmap_L2norm: ', fcn_hm_L2norm.shape ,' Keras tensor ', KB.is_keras_tensor(fcn_hm_L2norm) )
return fcn_hm, fcn_sm
for layer in conv_base.layers:
layer.trainable = False
#Outputs for skip connections
for layer in conv_base.layers:
if (layer.name == 'block4_pool'):
pool4_output = layer.output
if (layer.name == 'block3_pool'):
pool3_output = layer.output
if (layer.name == 'block1_pool'):
pool1_output = layer.output
#Create upsampling section of FCN
decode = layers.Deconvolution2D(512,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='1x1_conv')(conv_base.output)
decode = layers.Dropout(0.5)(decode)
decode = layers.Deconvolution2D(512,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='decode_block1_conv')(decode)
decode = layers.UpSampling2D((2, 2), name='decode_block1_upsample')(decode)
decode = layers.add([decode, pool4_output]) #Add skip connection
decode = layers.Dropout(0.5)(decode)
decode = layers.Deconvolution2D(256,
kernel_size=(3, 3),
activation='relu',
padding='same',
debug = False
digit = 5
save_dir = '.'
t = False
w = None
ep_num = 0
decoder = models.Sequential(name='decoder')
decoder.add(Dense(input_dim=32, activation="relu", output_dim=8 * 8 * 64))
decoder.add(Reshape((8, 8, 64)))
decoder.add(BatchNormalization(momentum=0.8))
decoder.add(
layers.Deconvolution2D(256,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(256,
3,
3,
subsample=(2, 2),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(256,
3,
3,
subsample=(2, 2),
border_mode='same',
def unet(pretrained_weights=None,
input_size=input_shape,
depth=3,
init_filter=8,
filter_size=3,
padding='same',
pool_size=[2, 2],
strides=[2, 2]):
inputs = klayers.Input(input_size)
current_layer = inputs
encoding_layers = []
# Encoder path
for d in range(depth + 1):
num_filters = init_filter * 2**d
conv = klayers.Conv2D(num_filters,
filter_size,
padding=padding,
kernel_initializer='he_normal')(current_layer)
conv = klayers.BatchNormalization()(conv)
conv = klayers.Activation('relu')(conv)
conv = klayers.Conv2D(num_filters,
filter_size,
padding=padding,
kernel_initializer='he_normal')(conv)
conv = klayers.BatchNormalization()(conv)
conv = klayers.Activation('relu')(conv)
encoding_layers.append(conv)
pool = klayers.MaxPooling2D(pool_size=pool_size)(conv)
if d == depth:
# Bridge
current_layer = conv
else:
current_layer = pool
# Decoder path
for d in range(depth, 0, -1):
num_filters = init_filter * 2**d
up = klayers.Deconvolution2D(num_filters * 2,
pool_size,
strides=strides)(current_layer)
crop_layer = encoding_layers[d - 1]
# Calculate two layers shape
up_shape = np.array(up._keras_shape[1:-1])
conv_shape = np.array(crop_layer._keras_shape[1:-1])
# Calculate crop size of left and right
crop_left = (conv_shape - up_shape) // 2
crop_right = (conv_shape - up_shape) // 2 + (conv_shape - up_shape) % 2
crop_sizes = tuple(zip(crop_left, crop_right))
crop = klayers.Cropping2D(cropping=crop_sizes)(crop_layer)
# Concatenate
up = klayers.Concatenate(axis=-1)([crop, up])
conv = klayers.Conv2D(num_filters,
filter_size,
padding=padding,
kernel_initializer='he_normal')(up)
conv = klayers.BatchNormalization()(conv)
conv = klayers.Activation('relu')(conv)
conv = klayers.Conv2D(num_filters,
filter_size,
padding=padding,
kernel_initializer='he_normal')(conv)
conv = klayers.BatchNormalization()(conv)
conv = klayers.Activation('relu')(conv)
current_layer = conv
outputs = klayers.Conv2D(n_classes,
1,
padding=padding,
kernel_initializer='he_normal')(current_layer)
outputs = klayers.Activation('softmax')(outputs)
model = Model(inputs=inputs, outputs=outputs)
if (pretrained_weights):
model.load_weights(pretrained_weights)
return model
def unet():
"unet for image reconstruction"
P0 = x = KL.Input(shape=(None, None, 3), name="u_net_input")
x = KL.Conv2D(64, (3, 3),
strides=(1, 1),
name='conv1',
use_bias=True,
padding="same")(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(64, (3, 3),
strides=(2, 2),
name='conv1a',
use_bias=True,
padding="same")(x)
P1 = x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(64, (3, 3),
strides=(1, 1),
name='conv2',
use_bias=True,
padding="same")(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(64, (3, 3),
strides=(2, 2),
name='conv2a',
use_bias=True,
padding="same")(x)
P2 = x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(128, (3, 3),
strides=(1, 1),
name='conv3',
use_bias=True,
padding="same")(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(128, (3, 3),
strides=(2, 2),
name='conv3a',
use_bias=True,
padding="same")(x)
P3 = x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(128, (3, 3),
strides=(1, 1),
name='conv4',
use_bias=True,
padding="same")(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(128, (3, 3),
strides=(2, 2),
name='conv4a',
use_bias=True,
padding="same")(x)
P4 = x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(256, (3, 3),
strides=(1, 1),
name='conv5',
use_bias=True,
padding="same")(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Conv2D(256, (3, 3),
strides=(2, 2),
name='conv5a',
use_bias=True,
padding="same")(x)
P5 = x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Deconvolution2D(nb_filter=128,
nb_row=3,
nb_col=3,
subsample=(1, 1),
name='deconv4',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Deconvolution2D(nb_filter=128,
nb_row=3,
nb_col=3,
subsample=(2, 2),
name='deconv4a',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
C4 = x = KL.Add()([P4, x])
x = KL.Deconvolution2D(nb_filter=128,
nb_row=3,
nb_col=3,
subsample=(1, 1),
name='deconv3',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Deconvolution2D(nb_filter=128,
nb_row=3,
nb_col=3,
subsample=(2, 2),
name='deconv3a',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
C3 = x = KL.Add()([P3, x])
x = KL.Deconvolution2D(nb_filter=64,
nb_row=3,
nb_col=3,
subsample=(1, 1),
name='deconv2',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Deconvolution2D(nb_filter=64,
nb_row=3,
nb_col=3,
subsample=(2, 2),
name='deconv2a',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
C2 = x = KL.Add()([P2, x])
x = KL.Deconvolution2D(nb_filter=64,
nb_row=3,
nb_col=3,
subsample=(1, 1),
name='deconv1',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Deconvolution2D(nb_filter=64,
nb_row=3,
nb_col=3,
subsample=(2, 2),
name='deconv1a',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
C1 = x = KL.Add()([P1, x])
x = KL.Deconvolution2D(nb_filter=64,
nb_row=3,
nb_col=3,
subsample=(1, 1),
name='deconv0',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
x = KL.Deconvolution2D(nb_filter=3,
nb_row=3,
nb_col=3,
subsample=(2, 2),
name='deconv0a',
border_mode='same')(x)
x = KL.Lambda(lambda t: keras.activations.relu(t, alpha=0.1))(x)
C0 = x = KL.Add()([P0, x])
x = KL.Conv2D(3, (3, 3),
strides=(1, 1),
name='convr',
use_bias=True,
padding="same")(x)
denoised_image = KL.Activation('linear')(x)
model = KM.Model(inputs=P0, outputs=denoised_image)
model.summary()
return model
def unet():
model_input = KL.Input(shape=(None,None,4),name="u_net_input")
conv1 = KL.Conv2D(32, (3, 3), name='conv1',padding="same")(model_input)
conv1 = KL.LeakyReLU(alpha=0.2)(conv1)
conv1 = KL.Conv2D(32, (3, 3), name='conv1a',padding="same")(conv1)
conv1 = KL.LeakyReLU(alpha=0.2)(conv1)
pool1 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_1")(conv1)
conv2 = KL.Conv2D(64, (3, 3), name='conv2',padding="same")(pool1)
conv2 = KL.LeakyReLU(alpha=0.2)(conv2)
conv2 = KL.Conv2D(64, (3, 3), name='conv2a',padding="same")(conv2)
conv2 = KL.LeakyReLU(alpha=0.2)(conv2)
pool2 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_2")(conv2)
conv3 = KL.Conv2D(128, (3, 3), name='conv3',padding="same")(pool2)
conv3 = KL.LeakyReLU(alpha=0.2)(conv3)
conv3 = KL.Conv2D(128, (3, 3), name='conv3a',padding="same")(conv3)
conv3 = KL.LeakyReLU(alpha=0.2)(conv3)
pool3 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_3")(conv3)
conv4 = KL.Conv2D(256, (3, 3), name='conv4',padding="same")(pool3)
conv4 = KL.LeakyReLU(alpha=0.2)(conv4)
conv4 = KL.Conv2D(256, (3, 3), name='conv4a',padding="same")(conv4)
conv4 = KL.LeakyReLU(alpha=0.2)(conv4)
pool4 = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding="same",name="max_pooling_4")(conv4)
conv5 = KL.Conv2D(512, (3, 3), name='conv5',padding="same")(pool4)
conv5 = KL.LeakyReLU(alpha=0.2)(conv5)
conv5 = KL.Conv2D(512, (3, 3), name='conv5a',padding="same")(conv5)
conv5 = KL.LeakyReLU(alpha=0.2)(conv5)
up6 = KL.Deconvolution2D(nb_filter=256, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv0',border_mode='same')(conv5)
up6 = KL.Concatenate()([conv4,up6])
conv6 = KL.Conv2D(256, (3, 3), name='conv6',padding="same")(up6)
conv6 = KL.LeakyReLU(alpha=0.2)(conv6)
conv6 = KL.Conv2D(256, (3, 3), name='conv6a',padding="same")(conv6)
conv6 = KL.LeakyReLU(alpha=0.2)(conv6)
up7 = KL.Deconvolution2D(nb_filter=128, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv1',border_mode='same')(conv6)
up7 = KL.Concatenate()([conv3,up7])
conv7 = KL.Conv2D(128, (3, 3), name='conv7',padding="same")(up7)
conv7 = KL.LeakyReLU(alpha=0.2)(conv7)
conv7 = KL.Conv2D(128, (3, 3), name='conv7a',padding="same")(conv7)
conv7 = KL.LeakyReLU(alpha=0.2)(conv7)
up8 = KL.Deconvolution2D(nb_filter=64, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv2',border_mode='same')(conv7)
up8 = KL.Concatenate()([conv2,up8])
conv8 = KL.Conv2D(64, (3, 3), name='conv8',padding="same")(up8)
conv8 = KL.LeakyReLU(alpha=0.2)(conv8)
conv8 = KL.Conv2D(64, (3, 3), name='conv8a',padding="same")(conv8)
conv8 = KL.LeakyReLU(alpha=0.2)(conv8)
up9 = KL.Deconvolution2D(nb_filter=32, nb_row=3, nb_col=3,subsample=(2, 2),name='deconv3',border_mode='same')(conv8)
up9 = KL.Concatenate()([conv1,up9])
conv9 = KL.Conv2D(32, (3, 3), name='conv9',padding="same")(up9)
conv9 = KL.LeakyReLU(alpha=0.2)(conv9)
conv9 = KL.Conv2D(32, (3, 3), name='conv9a',padding="same")(conv9)
conv9 = KL.LeakyReLU(alpha=0.2)(conv9)
conv10 = KL.Conv2D(12, (3, 3), name='conv10',padding="same")(conv9)
conv10 = KL.LeakyReLU(alpha=0.2)(conv10)
model_output = KL.Lambda(lambda t:tf.depth_to_space(t,2))(conv10)
model = KM.Model(inputs=model_input,outputs=model_output)
return model
