def build_model():
with tf.device('/device:GPU:2'):
base_model = applications.VGG19(weights='imagenet',
include_top=False,
input_shape=train_tensors.shape[1:])
add_model = Sequential()
add_model.add(Flatten(input_shape=base_model.output_shape[1:]))
added0_model = Model(inputs=base_model.input,
outputs=add_model(base_model.output))
stn_model = Sequential()
stn_model.add(
Lambda(lambda x: 2 * x - 1.,
input_shape=train_tensors.shape[1:],
output_shape=train_tensors.shape[1:]))
stn_model.add(BatchNormalization())
stn_model.add(
SpatialTransformer(localization_net=locnet(),
output_size=train_tensors.shape[1:3]))
added_model = Model(inputs=stn_model.input,
outputs=added0_model(stn_model.output))
inp = Input(batch_shape=(None, train_data.shape[1]))
extra_model = Model(input=inp, output=inp)
x = concatenate([added_model.output, extra_model.output])
x = Dropout(0.5)(x)
x = Dense(256, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(1, activation='sigmoid')(x)
model = Model(input=[added_model.input, extra_model.input], output=x)
model.summary()
return model
def create_model():
weights = get_init_weight()
img_input = Input(shape=( None, None, 3))
input_shape = (None, None, 3)
conv1 = Conv2D(32, (3, 3))(img_input)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(32, (3, 3))(pool1)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
out1 = GlobalAveragePooling2D(pool2)
out1 = Dense(50, activation = 'relu')(out1)
out = Dense(6, weights=weights)(out1)
locnet = Model(inputs=img_input, outputs= out)
in1 = SpatialTransformer(localization_net=locnet,
output_size=(32,32), input_shape=input_shape)
conv1=make_conv_block(32,in1,1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
# drop1 = Dropout(0.5)(pool1)
conv2=make_conv_block(64,pool1,2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
flat1 = Flatten()(pool2)
dense1 = Dense(hidden_size, activation = 'relu')(flat1)
out = Dense(num_class, activation = 'softmax')(dense1)
model = Model(inputs=img_input, outputs= out)
return model
def conv_model(input_shape=(32, 32, 3)):
model = Sequential()
model.add(
Lambda(lambda x: x / 127.5 - 1.,
input_shape=(32, 32, 3),
output_shape=(32, 32, 3)))
model.add(BatchNormalization())
model.add(Conv2D(10, (1, 1), padding='same', kernel_regularizer=l2(0.05)))
model.add(LeakyReLU(alpha=0.5))
model.add(BatchNormalization())
model.add(Conv2D(3, (1, 1), padding='same', kernel_regularizer=l2(0.05)))
model.add(LeakyReLU(alpha=0.5))
model.add(BatchNormalization())
model.add(
SpatialTransformer(localization_net=locnet(), output_size=(32, 32)))
# model.add(Conv2D(64,( 3, 3), strides=1, padding='same', data_format='channels_first',input_shape=(3, 32, 32)))
# model.add(BatchNormalization())
# model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3), kernel_regularizer=l2(0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='valid', kernel_regularizer=l2(0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3), kernel_regularizer=l2(0.05)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(5, activation="softmax"))
model.summary()
return model
def conv_model(input_shape=(32, 32, 3)):
model = Sequential()
model.add(InputLayer(input_shape=(32, 32, 3)))
model.add(
SpatialTransformer(localization_net=locnet(), output_size=(32, 32)))
model.add(
Conv2D(16, (3, 3),
padding='same',
activation='relu',
kernel_regularizer=l2(0.05)))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(32, (3, 3),
padding='same',
activation='relu',
kernel_regularizer=l2(0.05)))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(64, (3, 3),
padding='same',
activation='relu',
kernel_regularizer=l2(0.05)))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dropout(0.6))
model.add(Dense(43, activation="softmax"))
return model
def simpConvNNSTN(input_shape=(64, 64, 3)):
"""
Build the classifier simpConvNNSTN
"""
model = Sequential()
model.add(
SpatialTransformer(localization_net=locnet,
downsample_factor=3,
input_shape=input_shape))
model.add(
MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
input_shape=input_shape))
#model went from 64x64x3 to 32x32x3
model.add(Conv2D(64, (3, 3), strides=(2, 2), activation='softplus'))
#model is now 16x16x64
model.add(Conv2D(32, (3, 3), activation='softplus'))
#model is now 16x16x32
model.add(Conv2D(16, (3, 3), activation='softplus'))
#model is now 16x16x16
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
#model is now 8x8x16
model.add(Flatten())
#model is now 1024 (flattened from 8x8x16)
model.add(Dense(128, activation='softplus'))
model.add(Dense(32, activation='softplus'))
model.add(Dense(1, activation='sigmoid'))
return model
def create_stn_model(input_shape=(1600,), output_dim=10):
b = np.zeros((2, 3), dtype='float32')
b[0, 0] = 1
b[1, 1] = 1
W = np.zeros((50, 6), dtype='float32')
weights = [W, b.flatten()]
locnet = Sequential()
locnet.add(MaxPooling2D(pool_size=(2,2), input_shape=input_shape))
locnet.add(Conv2D(20, (5, 5)))
locnet.add(MaxPooling2D(pool_size=(2,2)))
locnet.add(Conv2D(20, (5, 5)))
locnet.add(Flatten())
locnet.add(Dense(50))
locnet.add(Activation('relu'))
locnet.add(Dense(6, weights=weights))
model = Sequential()
model.add(SpatialTransformer(localization_net=locnet,
output_size=(40,40), input_shape=input_shape))
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dense(output_dim, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
def conv_model(input_shape=(32, 32, 3)):
l2_reg = 0.05
model = Sequential()
model.add(Lambda(
lambda x: x * 2 - 1.,
input_shape=(32, 32, 3),
output_shape=(32, 32, 3)))
model.add(BatchNormalization())
model.add(Conv2D(10, (1, 1), padding='same',
kernel_regularizer=l2(l2_reg)))
model.add(LeakyReLU(alpha=0.5))
model.add(BatchNormalization())
model.add(Conv2D(3, (1, 1), padding='same', kernel_regularizer=l2(l2_reg)))
model.add(LeakyReLU(alpha=0.5))
model.add(BatchNormalization())
model.add(SpatialTransformer(localization_net=locnet(),
output_size=(32, 32)))
model.add(Conv2D(16, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(BatchNormalization())
model.add(Conv2D(32, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(BatchNormalization())
model.add(Conv2D(64, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(BatchNormalization())
model.add(Conv2D(96, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(BatchNormalization())
model.add(Conv2D(128, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(BatchNormalization())
model.add(Conv2D(192, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(256, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(BatchNormalization())
model.add(Conv2D(64, (5, 5), padding='same',
activation='relu', kernel_regularizer=l2(l2_reg)))
model.add(MaxPooling2D(pool_size=(8, 8)))
model.add(Flatten())
model.add(Dropout(0.6))
model.add(Dense(43, activation='softmax'))
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.01)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=adam, metrics=['accuracy'])
return model
def template_match_nn():
l2_reg = 0.01
model = Sequential()
model.add(Lambda(
lambda x: x * 2 - 1.,
input_shape=(32, 32, 3),
output_shape=(32, 32, 3)))
model.add(SpatialTransformer(localization_net=locnet(),
output_size=(32, 32)))
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.01)
model.compile(loss='mean_absolute_error', optimizer=adam)
return model
def stn(self):
# initial weights
b = np.zeros((2, 3), dtype='float32')
b[0, 0] = 1
b[1, 1] = 1
W = np.zeros((50, 6), dtype='float32')
weights = [W, b.flatten()]
locnet = Sequential()
locnet.add(MaxPooling2D(pool_size=(2, 2), input_shape=self.input_shape))
locnet.add(Conv2D(20, (5, 5)))
locnet.add(MaxPooling2D(pool_size=(2, 2)))
locnet.add(Conv2D(20, (5, 5)))
locnet.add(Flatten())
locnet.add(Dense(50))
locnet.add(Activation('relu'))
locnet.add(Dense(6, weights=weights))
# stn
model = Sequential()
# conv1
model.add(SpatialTransformer(localization_net=locnet,
output_size=(32, 32), input_shape=self.input_shape))
# model.add(SpatialTransformer(localization_net=locnet,
# output_size=self.input_shape, input_shape=self.input_shape))
model.add(Conv2D(20, kernel_size=(5, 5),
padding="same",
activation='relu'))
# pool1
# model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# conv2
model.add(Conv2D(50, kernel_size=(5, 5),
padding="same",
activation='relu'))
# pool2
# model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# model.add(Dropout(0.25))
model.add(Flatten())
# ip1
model.add(Dense(500, activation='relu', name='fc1'))
# model.add(Dropout(0.5))
# ip2
model.add(Dense(self.num_classes, activation='softmax', name='predictions'))
return model
def crnn_stn(self):
from spatial_transformer import SpatialTransformer
import numpy as np
# initial weights
b = np.zeros((2, 3), dtype='float32')
b[0, 0] = 1
b[1, 1] = 1
W = np.zeros((20, 6), dtype='float32')
weights = [W, b.flatten()]
locnet = Sequential()
locnet.add(MaxPooling2D(pool_size=(2, 2),
input_shape=self.image_shape))
locnet.add(Conv2D(20, (5, 5)))
locnet.add(MaxPooling2D(pool_size=(2, 2)))
locnet.add(Conv2D(20, (5, 5)))
locnet.add(Dense(6, weights=weights))
#vgg 16
model = Sequential()
model.add(
TimeDistributed(SpatialTransformer(localization_net=locnet,
output_size=(30, 30)),
input_shape=self.input_shape))
model.add(
TimeDistributed(Conv2D(8, (3, 3),
activation='relu',
padding='same'),
input_shape=self.input_shape))
model.add(
TimeDistributed(
Conv2D(32, (3, 3), activation='relu', padding='same')))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(16, return_sequences=True))
model.add(Flatten())
model.add(Dense(self.nb_classes, activation='softmax'))
return model
def conv_model(input_shape=(32, 32, 3)): model = Sequential() model.add(Lambda( lambda x: (x - 128) / 128, input_shape=(32, 32, 3), output_shape=(32, 32, 3))) model.add(SpatialTransformer(localization_net=locnet(), output_size=(32, 32))) # model.add(BatchNormalization()) # model.add(Conv2D(10, (1, 1), padding='same', kernel_regularizer=l2(0.05))) # model.add(LeakyReLU(alpha=0.5)) # model.add(BatchNormalization()) # model.add(Conv2D(3, (1, 1), padding='same', kernel_regularizer=l2(0.05))) # model.add(LeakyReLU(alpha=0.5)) # model.add(BatchNormalization()) # model.add(SpatialTransformer(localization_net=locnet(), # output_size=(32, 32))) model.add(Conv2D(16, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) model.add(BatchNormalization()) model.add(Conv2D(32, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) model.add(BatchNormalization()) model.add(Conv2D(64, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) model.add(BatchNormalization()) # model.add(Conv2D(96, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) # model.add(BatchNormalization()) # model.add(Conv2D(128, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) # model.add(BatchNormalization()) # model.add(Conv2D(192, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) # model.add(BatchNormalization()) # model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Conv2D(256, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) # model.add(BatchNormalization()) # model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Conv2D(128, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) # model.add(BatchNormalization()) model.add(Conv2D(64, (5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.05))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dropout(0.6)) model.add(Dense(43, activation="softmax")) return model
locnet = Sequential()
locnet.add(MaxPooling2D(pool_size=(2,2), input_shape=input_shape))
locnet.add(Conv2D(20, (5, 5)))
locnet.add(MaxPooling2D(pool_size=(2,2)))
locnet.add(Conv2D(20, (5, 5)))
locnet.add(Flatten())
locnet.add(Dense(50))
locnet.add(Activation('relu'))
locnet.add(Dense(6, weights=weights))
#locnet.add(Activation('sigmoid'))
model = Sequential()
model.add(SpatialTransformer(localization_net=locnet,
output_size=(30,30), input_shape=input_shape))
model.add(Conv2D(32, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
base_model = applications.VGG19(weights='imagenet',
include_top=False,
input_shape=(64, 64, 3))
add_model = Sequential()
add_model.add(Flatten(input_shape=base_model.output_shape[1:]))
added0_model = Model(inputs=base_model.input, outputs=add_model(base_model.output))
stn_model = Sequential()
stn_model.add(Lambda(
lambda x: 2*x - 1.,
input_shape=(64, 64, 3),
output_shape=(64, 64, 3)))
stn_model.add(BatchNormalization())
stn_model.add(SpatialTransformer(localization_net=locnet(),
output_size=(64, 64)))
added_model = Model(inputs=stn_model.input, outputs=added0_model(stn_model.output))
inp = Input(batch_shape=(None, 5))
# out = Dense(8)(inp)
extra_model = Model(input=inp, output=inp)
x = concatenate([added_model.output,
extra_model.output])
# x = Dropout(0.5)(x)
# x = Dense(1024, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(256, activation='relu')(x)
x = Dropout(0.5)(x)
def __init__(self):
super().__init__()
self.transform = SpatialTransformer()
self.classify = ClassNet()
def build_model_FCN_model_api(batch_size,
optimizer,
patch_size=(128, 128),
base_weight_decay=0.0005,
output_ROI_mask=True):
print('Using build_model_FCN_model_api')
# define the shared model:
net_name = 'Multi-view_FCN'
scale_number = 3
##################### input ###############################################
input_shape0 = (batch_size, patch_size[0], patch_size[1], 1)
input_shape1 = (batch_size, patch_size[0] / 2, patch_size[1] / 2, 1)
input_shape2 = (batch_size, patch_size[0] / 4, patch_size[1] / 4, 1)
input_shape3 = (1, patch_size[0] / 4, patch_size[1] / 4, 1)
input_patches1_s0 = Input(batch_shape=input_shape0, name='patches1_s0')
input_patches1_s1 = Input(batch_shape=input_shape1, name='patches1_s1')
input_patches1_s2 = Input(batch_shape=input_shape2, name='patches1_s2')
input_patches2_s0 = Input(batch_shape=input_shape0, name='patches2_s0')
input_patches2_s1 = Input(batch_shape=input_shape1, name='patches2_s1')
input_patches2_s2 = Input(batch_shape=input_shape2, name='patches2_s2')
input_patches3_s0 = Input(batch_shape=input_shape0, name='patches3_s0')
input_patches3_s1 = Input(batch_shape=input_shape1, name='patches3_s1')
input_patches3_s2 = Input(batch_shape=input_shape2, name='patches3_s2')
input_patches4_s0 = Input(batch_shape=input_shape0, name='patches4_s0')
input_patches4_s1 = Input(batch_shape=input_shape1, name='patches4_s1')
input_patches4_s2 = Input(batch_shape=input_shape2, name='patches4_s2')
input_depth_maps_v1 = Input(batch_shape=input_shape3,
name='depth_ratio_v1')
input_depth_maps_v2 = Input(batch_shape=input_shape3,
name='depth_ratio_v2')
input_depth_maps_v3 = Input(batch_shape=input_shape3,
name='depth_ratio_v3')
input_depth_maps_v4 = Input(batch_shape=input_shape3,
name='depth_ratio_v4')
if output_ROI_mask:
# the output density patch/map is down-sampled by a factor of 4
output_masks = Input(batch_shape=(batch_size, patch_size[0],
patch_size[1], 1),
name='output_masks')
train_flag = False
####################### view 1 #############################################
# image pyramids:
x1_s0_output = feature_extraction_view1(base_weight_decay,
input_patches1_s0, train_flag)
x1_s1_output = feature_extraction_view1(base_weight_decay,
input_patches1_s1, train_flag)
x1_s2_output = feature_extraction_view1(base_weight_decay,
input_patches1_s2, train_flag)
# view 1 decoder
# x1_7 = view1_decoder(base_weight_decay, x1_s0_output)
# conv block 5
x1_5 = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_5')(x1_s0_output)
x1_5 = Activation('relu', name='conv_block_5_act')(x1_5)
# conv block 6
x1_6 = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_6')(x1_5)
x1_6 = Activation('relu', name='conv_block_6_act')(x1_6)
# conv block 7
x1_7 = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_7')(x1_6)
x1_7 = Activation('relu', name='conv_block_7_act')(x1_7)
####################### view 2 #############################################
# image pyramids:
x2_s0_output = feature_extraction_view2(base_weight_decay,
input_patches2_s0, train_flag)
x2_s1_output = feature_extraction_view2(base_weight_decay,
input_patches2_s1, train_flag)
x2_s2_output = feature_extraction_view2(base_weight_decay,
input_patches2_s2, train_flag)
# view 2 decoder
# x2_7 = view2_decoder(base_weight_decay, x2_s0_output)
# dmap
# conv block 5
x2_5 = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_5_2')(x2_s0_output)
x2_5 = Activation('relu', name='conv_block_5_2_act')(x2_5)
# conv block 6
x2_6 = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_6_2')(x2_5)
x2_6 = Activation('relu', name='conv_block_6_2_act')(x2_6)
# conv block 7
x2_7 = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_7_2')(x2_6)
x2_7 = Activation('relu', name='conv_block_7_2_act')(x2_7)
####################### view 3 #############################################
# image pyramids:
x3_s0_output = feature_extraction_view3(base_weight_decay,
input_patches3_s0, train_flag)
x3_s1_output = feature_extraction_view3(base_weight_decay,
input_patches3_s1, train_flag)
x3_s2_output = feature_extraction_view3(base_weight_decay,
input_patches3_s2, train_flag)
# view 3 decoder
# x3_7 = view3_decoder(base_weight_decay, x3_s0_output)
# conv block 5
x3_5 = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_5_3')(x3_s0_output)
x3_5 = Activation('relu', name='conv_block_5_3_act')(x3_5)
# conv block 6
x3_6 = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_6_3')(x3_5)
x3_6 = Activation('relu', name='conv_block_6_3_act')(x3_6)
# conv block 7
x3_7 = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_7_3')(x3_6)
x3_7 = Activation('relu', name='conv_block_7_3_act')(x3_7)
####################### view 4 #############################################
# image pyramids:
x4_s0_output = feature_extraction_view4(base_weight_decay,
input_patches4_s0, train_flag)
x4_s1_output = feature_extraction_view4(base_weight_decay,
input_patches4_s1, train_flag)
x4_s2_output = feature_extraction_view4(base_weight_decay,
input_patches4_s2, train_flag)
# view 4 decoder
# x4_7 = view4_decoder(base_weight_decay, x4_s0_output)
# conv block 5
x4_5 = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_5_4')(x4_s0_output)
x4_5 = Activation('relu', name='conv_block_5_4_act')(x4_5)
# conv block 6
x4_6 = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_6_4')(x4_5)
x4_6 = Activation('relu', name='conv_block_6_4_act')(x4_6)
# conv block 7
x4_7 = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_7_4')(x4_6)
x4_7 = Activation('relu', name='conv_block_7_4_act')(x4_7)
#################################### fusion #############################################
################# get the scale-selection mask #####################
# view depth of image
batch_size = x1_s0_output.shape[0].value
height = x1_s0_output.shape[1].value
width = x1_s0_output.shape[2].value
num_channels = x1_s0_output.shape[3].value
output_shape = [1, height, width, 1]
# view1_depth = feature_scale_fusion_layer_mask(scale_number=scale_number,
# view = 1, output_shape=output_shape)
# view2_depth = feature_scale_fusion_layer_mask(scale_number=scale_number,
# view = 2, output_shape=output_shape)
# view3_depth = feature_scale_fusion_layer_mask(scale_number=scale_number,
# view = 3, output_shape=output_shape)
# view1_scale = scale_selection_mask(base_weight_decay, input_depth_maps_v1)
# view2_scale = scale_selection_mask(base_weight_decay, input_depth_maps_v2)
# view3_scale = scale_selection_mask(base_weight_decay, input_depth_maps_v3)
view1_scale = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer=Constant(value=-1),
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
bias_initializer='ones',
activation=None,
name='scale_fusion1')(input_depth_maps_v1)
view2_scale = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer=Constant(value=-1),
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
bias_initializer='ones',
activation=None,
name='scale_fusion2')(input_depth_maps_v2)
view3_scale = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer=Constant(value=-1),
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
bias_initializer='ones',
activation=None,
name='scale_fusion3')(input_depth_maps_v3)
view4_scale = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer=Constant(value=-1),
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
bias_initializer='ones',
activation=None,
name='scale_fusion4')(input_depth_maps_v4)
view1_scale_mask = feature_scale_fusion_layer_rbm(
scale_number=scale_number)(view1_scale)
view2_scale_mask = feature_scale_fusion_layer_rbm(
scale_number=scale_number)(view2_scale)
view3_scale_mask = feature_scale_fusion_layer_rbm(
scale_number=scale_number)(view3_scale)
view4_scale_mask = feature_scale_fusion_layer_rbm(
scale_number=scale_number)(view4_scale)
#################### fusion with mask ##################
# view 1
## conv
x1_s0_output_fusion = fusion_conv_v1(base_weight_decay, x1_s0_output)
x1_s1_output_fusion = fusion_conv_v1(base_weight_decay, x1_s1_output)
x1_s2_output_fusion = fusion_conv_v1(base_weight_decay, x1_s2_output)
## up sampling
x1_s1_output_fusion = UpSampling_layer(size=[height, width])(
[x1_s1_output_fusion])
x1_s2_output_fusion = UpSampling_layer(size=[height, width])(
[x1_s2_output_fusion])
concatenated_map_v1 = Concatenate(name='cat_map_v1')(
[x1_s0_output_fusion, x1_s1_output_fusion, x1_s2_output_fusion])
fusion_v1 = feature_scale_fusion_layer(scale_number=scale_number)(
[concatenated_map_v1, view1_scale_mask])
## proj
fusion_v1_proj = SpatialTransformer(
1, [int(480 / 4), int(640 / 4)])(fusion_v1)
# view 2
## conv
x2_s0_output_fusion = fusion_conv_v2(base_weight_decay, x2_s0_output)
x2_s1_output_fusion = fusion_conv_v2(base_weight_decay, x2_s1_output)
x2_s2_output_fusion = fusion_conv_v2(base_weight_decay, x2_s2_output)
## up sampling
x2_s1_output_fusion = UpSampling_layer(size=[height, width])(
[x2_s1_output_fusion])
x2_s2_output_fusion = UpSampling_layer(size=[height, width])(
[x2_s2_output_fusion])
concatenated_map_v2 = Concatenate(name='cat_map_v2')(
[x2_s0_output_fusion, x2_s1_output_fusion, x2_s2_output_fusion])
fusion_v2 = feature_scale_fusion_layer(scale_number=scale_number)(
[concatenated_map_v2, view2_scale_mask])
## proj
fusion_v2_proj = SpatialTransformer(
2, [int(480 / 4), int(640 / 4)])(fusion_v2)
# view 3
## conv
x3_s0_output_fusion = fusion_conv_v3(base_weight_decay, x3_s0_output)
x3_s1_output_fusion = fusion_conv_v3(base_weight_decay, x3_s1_output)
x3_s2_output_fusion = fusion_conv_v3(base_weight_decay, x3_s2_output)
## up sampling
x3_s1_output_fusion = UpSampling_layer(size=[height, width])(
[x3_s1_output_fusion])
x3_s2_output_fusion = UpSampling_layer(size=[height, width])(
[x3_s2_output_fusion])
concatenated_map_v3 = Concatenate(name='cat_map_v3')(
[x3_s0_output_fusion, x3_s1_output_fusion, x3_s2_output_fusion])
fusion_v3 = feature_scale_fusion_layer(scale_number=scale_number)(
[concatenated_map_v3, view3_scale_mask])
## proj
fusion_v3_proj = SpatialTransformer(
3, [int(480 / 4), int(640 / 4)])(fusion_v3)
# view 4
## conv
x4_s0_output_fusion = fusion_conv_v4(base_weight_decay, x4_s0_output)
x4_s1_output_fusion = fusion_conv_v4(base_weight_decay, x4_s1_output)
x4_s2_output_fusion = fusion_conv_v4(base_weight_decay, x4_s2_output)
## up sampling
x4_s1_output_fusion = UpSampling_layer(size=[height, width])(
[x4_s1_output_fusion])
x4_s2_output_fusion = UpSampling_layer(size=[height, width])(
[x4_s2_output_fusion])
concatenated_map_v4 = Concatenate(name='cat_map_v4')(
[x4_s0_output_fusion, x4_s1_output_fusion, x4_s2_output_fusion])
fusion_v4 = feature_scale_fusion_layer(scale_number=scale_number)(
[concatenated_map_v4, view4_scale_mask])
## proj
fusion_v4_proj = SpatialTransformer(
4, [int(480 / 4), int(640 / 4)])(fusion_v4)
################# concatenate ################
concatenated_map = Concatenate(name='cat_map_fusion')(
[fusion_v1_proj, fusion_v2_proj, fusion_v3_proj, fusion_v4_proj])
fusion_v123 = Conv2D(data_format='channels_last',
trainable=True,
filters=96,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='scale_fusion')(concatenated_map)
fusion_v123 = Activation('relu', name='scale_fusion_act')(fusion_v123)
#################### fusion and decode #####################################
# conv block 9
x = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_fusion1')(fusion_v123)
x = Activation('relu', name='conv_block_fusion1_act')(x)
x = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_fusion2')(x)
x = Activation('relu', name='conv_block_fusion2_act')(x)
x = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_fusion3')(x)
x_output = Activation('relu', name='conv_block_fusion3_act')(x)
if output_ROI_mask:
rgr_output = 'den_map_roi'
output = Multiply(name=rgr_output)([x_output, output_masks])
print('Layer name of regression output: {}'.format(rgr_output))
model = Model(inputs=[
input_patches1_s0, input_patches1_s1, input_patches1_s2,
input_patches2_s0, input_patches2_s1, input_patches2_s2,
input_patches3_s0, input_patches3_s1, input_patches3_s2,
input_patches4_s0, input_patches4_s1, input_patches4_s2,
input_depth_maps_v1, input_depth_maps_v2, input_depth_maps_v3,
input_depth_maps_v4, output_masks
],
outputs=[x_output],
name=net_name)
else:
model = Model(
inputs=[
input_patches1_s0,
input_patches1_s1,
input_patches1_s2,
input_patches2_s0,
input_patches2_s1,
input_patches2_s2,
input_patches3_s0,
input_patches3_s1,
input_patches3_s2,
input_patches4_s0,
input_patches4_s1,
input_patches4_s2,
input_depth_maps_v1,
input_depth_maps_v2,
input_depth_maps_v3,
input_depth_maps_v4,
],
outputs=[x_output], #x1_7, x2_7, x3_7, x4_7,
name=net_name + 'overall')
print('Compiling ...')
model.compile(optimizer=optimizer, loss='mse')
return model
locnet.add(MaxPooling2D(pool_size=(2, 2)))
locnet.add(Convolution2D(20, (3, 3), padding='same'))
locnet.add(Activation('linear'))
locnet.add(MaxPooling2D(pool_size=(2, 2)))
locnet.add(Flatten())
#locnet.add(Dense(100))
locnet.add(Activation('linear'))
locnet.add(Dense(9, weights=weights))
locnet.add(Activation('linear'))
#-----Now we add a STN layer which will output the transformed image-----#
transf1 = Sequential()
transf1.add(
SpatialTransformer(localization_net=locnet,
output_size=(photo_height, photo_width, 1),
input_shape=input_shape))
from keras import optimizers
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
transf1.compile(loss='mse', optimizer=sgd, metrics=['mse'])
locnet.summary()
transf1.summary()
#-----LOAD DATASET-----#
X, y, X_val, y_val = read_train_set("dataset/train_set")
print "Train tensor shape: ", X.shape
print "Validation tensor shape: ", X_val.shape
#-----FITTING-----#
#-----Fit on batch. This is useful for GPU training to prevent Out Of Memory (OOM) error-----#
def train_net(config_file):
net = SpatialTransformer(config_file)
X = T.tensor4(name='input_tensor')
y = T.ivector(name='label_vector')
X_shared = theano.shared(np.zeros(net.input_tensor_shape, 'float32'),
'input')
y_shared = theano.shared(np.zeros(net.output_shape, 'int32'), 'output')
lr = theano.shared(net.learning_rate, 'learning rate')
net.set_training_status(True)
pred = net(X)
cost = net.build_cost(pred, y)
updates = net.build_updates(cost, net.trainable, **{'learning_rate': lr})
error_rate = metrics.mean_classification_error(pred, y) * 100.
train = net.compile([], [cost, error_rate],
updates=updates,
givens={
X: X_shared,
y: y_shared
})
net.set_training_status(False)
pred = net(X)
trans = net.inference_trans_net(X)
error_rate = metrics.mean_classification_error(pred, y) * 100.
eval = net.compile([], [error_rate, trans],
givens={
X: X_shared,
y: y_shared
})
data = read_data.load_data(net.config['data']['path'])
dm = utils.DataManager(config_file, (X_shared, y_shared))
dm.training_set = (data['X_train'], data['y_train'])
dm.num_train_data = data['num_examples_train']
dm.testing_set = (data['X_valid'], data['y_valid'])
dm.num_test_data = data['num_examples_valid']
mon = monitor.Monitor(config_file)
epoch = 0
print('Start training...')
while epoch < net.n_epochs:
if (epoch + 1) % 20 == 0:
new_lr = lr.get_value() * 0.7
print('New LR: %f' % new_lr)
lr.set_value(new_lr)
batches = dm.get_batches(epoch, net.n_epochs, True, False)
for X_, y_ in batches:
dm.update_input((X_, y_))
_cost, _error = train()
mon.plot('training cost', _cost)
mon.plot('training classification error', _error)
mon.tick()
if epoch % net.validation_frequency == 0:
valid_batches = dm.get_batches(training=False)
for X_, y_ in valid_batches:
dm.update_input((X_, y_))
_error, _trans = eval()
mon.plot('validation classification error', _error)
mon.save_image('transformed image', _trans[:10])
mon.flush()
epoch += 1
print('Training ended.')