def SDN_incept3(inputs):
z1 = MyBlockDown(inputs, 16)
z2 = MyBlockDown(z1)
z3 = MyBlockDown(z2)
z4 = MyBlockDown(z3)
z3t = MyBlockUp(z4)
# z3t = ZeroPadding3D(((0,1),(0,0),(0,1)))(z3t)
z3t = add([z3t, incept4(z3, 8, 'LeakyReLU')])
z2t = MyBlockUp(z3t)
z2t = ZeroPadding3D(((0, 0), (0, 1), (0, 0)))(z2t)
z2t = add([z2t, incept4(z2, 8, 'LeakyReLU')])
z1t = MyBlockUp(z2t)
# z1t = ZeroPadding3d(((0,1),(0,1),(0,1)))(z1t)
z1t = add([z1t, incept4(z1, 8, 'LeakyReLU')])
z0t = MyBlockUp(z1t)
# z0t = ZeroPadding3d(((1,1),(1,1),(1,1)))(z0t)
disp = Conv3D(16, par['kernel_size'], padding='same')(z0t)
disp = LeakyReLU(0.2)(disp)
disp = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(disp)
disp = add([disp, incept4(inputs, 4, 'LeakyReLU')])
disp = Conv3D(3,
par['kernel_size'],
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-2),
activation='linear')(disp)
return disp
def SDN_incept4(inputs):
z1 = Blockdown(inputs)
z2 = Blockdown(z1)
z3 = Blockdown(z2)
z4 = Blockdown(z3)
z5 = Conv3D(32, (2, 2, 2), padding='same')(z4)
z5 = add([z5, z4])
z3t = Blockup(z5)
z3t = add([z3t, z3])
z2t = Blockup(z3t)
z2t = ZeroPadding3D(((0, 0), (0, 1), (0, 0)))(z2t)
z2t = add([z2t, z2])
z1t = Blockup(z2t)
z1t = add([z1t, z1])
z0t = Blockup(z1t)
z0t = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(z0t)
disp = Conv3D(32, par['kernel_size'], padding='same')(inputs)
z0t = add([z0t, disp])
zzzz = Conv3D(3,
par['kernel_size'],
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-2),
activation='linear')(z0t)
return zzzz
def SDN_ver5(inputs):
#z1_1 = Conv3D(32, (2,2,2), padding = 'same')(inputs)
z1_2 = Conv3D(32, (2,2,2), strides = 2, padding = 'valid')(inputs)
z1_2 = PReLU(shared_axes = [4])(z1_2)
#z2_1 = Conv3D(64, (2,2,2), padding = 'same')(z1_2)
z2_2 = Conv3D(64, (2,2,2), strides = 2, padding = 'valid')(z1_2)
z2_2 = PReLU(shared_axes = [4])(z2_2)
z4 = Conv3DTranspose(32, (2,2,2), strides=2, padding = 'valid')(z2_2)
z4 = ZeroPadding3D(((1,0), (0,0), (1,0)))(z4)
z4 = add([z4, z1_2])
z4 = PReLU(shared_axes = [4])(z4)
z4 = Conv3D(32, (2,2,2), padding = 'same')(z4)
#z4 = PReLU()(z4)
z5 = Conv3DTranspose(64, (2,2,2), strides=2, padding = 'valid')(z4)
z5 = Conv3D(64, (2,2,2), padding = 'same', activation = 'linear')(z5)
#z5 = PReLU()(z5)
z5 = ZeroPadding3D(((1,0),(1,0),(1,0)))(z5) #Extra padding to make size match
zzzz = Conv3D(3, (2,2,2), padding = 'same',
# kernel_initializer= 'he_uniform',
# bias_initializer = 'he_uniform',
# activity_regularizer = l1(0.001),
activation = 'tanh')(z5)
locnet = Model(inputs, zzzz)
x1 = SpatialDeformer3D(localization_net=locnet,
output_size=(input_shape[0],input_shape[1], input_shape[2]),
input_shape=input_shape)(inputs)
return x1, locnet(inputs)
def SDN_ver5(inputs):
#z1_1 = Conv3D(32, (2,2,2), padding = 'same')(inputs)
z1_2 = Conv3D(32, (2, 2, 2), strides=2, padding='valid')(inputs)
z1_2 = PReLU(shared_axes=[4])(z1_2)
#z2_1 = Conv3D(64, (2,2,2), padding = 'same')(z1_2)
z2_2 = Conv3D(64, (2, 2, 2), strides=2, padding='valid')(z1_2)
z2_2 = PReLU(shared_axes=[4])(z2_2)
z4 = Conv3DTranspose(32, (2, 2, 2), strides=2, padding='valid')(z2_2)
z4 = ZeroPadding3D(((1, 0), (0, 0), (1, 0)))(z4)
z4 = add([z4, z1_2])
z4 = PReLU(shared_axes=[4])(z4)
z4 = Conv3D(32, (2, 2, 2), padding='same')(z4)
#z4 = PReLU()(z4)
z5 = Conv3DTranspose(64, (2, 2, 2), strides=2, padding='valid')(z4)
z5 = Conv3D(64, (2, 2, 2), padding='same', activation='linear')(z5)
#z5 = PReLU()(z5)
z5 = ZeroPadding3D(
((1, 0), (1, 0), (1, 0)))(z5) #Extra padding to make size match
zzzz = Conv3D(
3,
(2, 2, 2),
padding='same',
# kernel_initializer= 'he_uniform',
# bias_initializer = 'he_uniform',
# activity_regularizer = l1(0.001),
activation='tanh')(z5)
return zzzz
def manipulate_input_stack(self, inputlayer):
pad_crop = ((0, 0), (0, 0), hp.calculate_pad_crop_value(self.e_v))
if self.manipulation == MANIPULATION.SPATIAL_UP:
output = ZeroPadding3D(padding=pad_crop,
data_format="channels_last")(inputlayer)
elif self.manipulation == MANIPULATION.SPATIAL_DOWN:
output = Cropping3D(cropping=pad_crop,
data_format="channels_last")(inputlayer)
elif self.manipulation == MANIPULATION.SPATIAL_MIN:
x = hp.calculate_stack_resize(self.vol_depth, 'min')[0]
if 2**x < self.vol_depth:
output = Cropping3D(cropping=pad_crop,
data_format="channels_last")(inputlayer)
else:
output = ZeroPadding3D(padding=pad_crop,
data_format="channels_last")(inputlayer)
elif self.manipulation == MANIPULATION.FREQUENCY_UP:
print('MANIPULATION.FREQUENCY_UP not yet implemented')
self.fourier_transform(inputlayer, pad_crop)
elif self.manipulation == MANIPULATION.FREQUENCY_DOWN:
print('MANIPULATION.FREQUENCY_DOWN not yet implemented')
elif self.manipulation == MANIPULATION.FREQUENCY_MIN:
print('MANIPULATION.FREQUENCY_MIN not yet implemented')
return output
def _pad_or_crop_to_shape_3D(x, in_shape, tgt_shape):
'''
in_shape, tgt_shape are both 2x1 numpy arrays
'''
im_diff = np.asarray(in_shape[:3]) - np.asarray(tgt_shape[:3])
if im_diff[0] < 0:
pad_amt = (int(np.ceil(abs(im_diff[0]) / 2.0)), int(np.floor(abs(im_diff[0]) / 2.0)))
x = ZeroPadding3D((
pad_amt,
(0, 0),
(0, 0)
))(x)
if im_diff[1] < 0:
pad_amt = (int(np.ceil(abs(im_diff[1]) / 2.0)), int(np.floor(abs(im_diff[1]) / 2.0)))
x = ZeroPadding3D(((0, 0), pad_amt, (0, 0)))(x)
if im_diff[2] < 0:
pad_amt = (int(np.ceil(abs(im_diff[2]) / 2.0)), int(np.floor(abs(im_diff[2]) / 2.0)))
x = ZeroPadding3D(((0, 0), (0, 0), pad_amt))(x)
if im_diff[0] > 0:
crop_amt = (int(np.ceil(im_diff[0] / 2.0)), int(np.floor(im_diff[0] / 2.0)))
x = Cropping3D((crop_amt, (0, 0), (0, 0)))(x)
if im_diff[1] > 0:
crop_amt = (int(np.ceil(im_diff[1] / 2.0)), int(np.floor(im_diff[1] / 2.0)))
x = Cropping3D(((0, 0), crop_amt, (0, 0)))(x)
if im_diff[2] > 0:
crop_amt = (int(np.ceil(im_diff[2] / 2.0)), int(np.floor(im_diff[2] / 2.0)))
x = Cropping3D(((0, 0), (0, 0), crop_amt))(x)
return x
def build_standard_model(input_shape, class_count, activation, dropout_fraction):
"""
Creates a 3D convolutional neural network with three filter layers and two prediction layers.
:param input_shape: tuple(int), four-dimensional tuple of input batch size, usually (batch, window, window, 1)
:param class_count: int, the number of classes
:param activation: str, the used activation function for the network
:param dropout_fraction: float, the fraction level of dropout applied dropout regularization
:return: model: keras.Sequential, the constructed model
"""
model = Sequential()
model.add(Conv3D(48, kernel_size=(3, 3, 5), activation=activation, input_shape=input_shape))
model.add(Dropout(dropout_fraction))
model.add(MaxPooling3D(pool_size=(1, 1, 3)))
model.add(ZeroPadding3D((0, 0, 2), data_format=None))
model.add(Conv3D(32, kernel_size=(3, 3, 5), activation=activation))
model.add(Dropout(dropout_fraction))
model.add(MaxPooling3D(pool_size=(1, 1, 3)))
model.add(ZeroPadding3D((0, 0, 2), data_format=None))
model.add(Conv3D(32, kernel_size=(3, 3, 5), activation=activation))
model.add(Dropout(dropout_fraction))
model.add(MaxPooling3D(pool_size=(1, 1, 2)))
model.add(Flatten())
model.add(Dense(128, activation=activation))
model.add(Dropout(dropout_fraction))
model.add(Dense(128, activation=activation))
model.add(Dense(class_count, activation='softmax'))
return model
def mobilenet_3d(input_shape: tuple, model_2d: Model):
input_image = Input(input_shape)
x = ZeroPadding3D()(input_image)
x = conv2d3d(model_2d.get_layer('conv1'))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = ZeroPadding3D()(x)
x = DepthwiseConv2D()(x)
def SDN_ver3(inputs):
'''
Uses three different CONV blocks at last in the decoder part for each dimension
'''
z1_2 = Conv3D(32,
par['kernel_size'],
strides=par['kernel_strides'],
padding='valid',
activation='linear')(inputs)
z1_2 = PReLU(shared_axes=[4])(z1_2)
z2_2 = Conv3D(64,
par['kernel_size'],
strides=par['kernel_strides'],
padding='valid',
activation='linear')(z1_2)
z2_2 = PReLU(shared_axes=[4])(z2_2)
z4 = Conv3DTranspose(64,
par['kernel_size'],
strides=par['kernel_strides'],
padding='valid',
activation='linear')(z2_2)
z4 = Conv3D(64, (2, 2, 2), padding='same')(z4)
z5x = Conv3DTranspose(32,
par['kernel_size'],
strides=par['kernel_strides'],
padding='valid',
activation='linear')(z4)
z5x = Conv3D(32, (2, 2, 2), padding='same')(z5x)
z5x = ZeroPadding3D((2, 1, 2))(z5x)
zzzzx = Conv3D(1, par['kernel_size'], padding='valid',
activation='tanh')(z5x)
z5y = Conv3DTranspose(32,
par['kernel_size'],
strides=par['kernel_strides'],
padding='valid',
activation='linear')(z4)
z5y = Conv3D(32, (2, 2, 2), padding='same')(z5y)
z5y = ZeroPadding3D((2, 1, 2))(z5y)
zzzzy = Conv3D(1, par['kernel_size'], padding='valid',
activation='tanh')(z5y)
z5z = Conv3DTranspose(32,
par['kernel_size'],
strides=par['kernel_strides'],
padding='valid',
activation='linear')(z4)
z5z = Conv3D(32, (2, 2, 2), padding='same')(z5z)
z5z = ZeroPadding3D((2, 1, 2))(z5z)
zzzzz = Conv3D(1, par['kernel_size'], padding='valid',
activation='tanh')(z5z)
zzzz = concatenate([zzzzx, zzzzy, zzzzz], axis=-1)
return zzzz
def of_network():
"""The neural network as proposed in 'Dual Temporal Scale Convolutional
Neural Network for Micro-Expression Recognition' by Peng et al.
This CNN was originally proposed for micro expression classification on
CASME I and CASME II but we try to adapt it to Cognitive Load Estimation
from facial movement.
Input layer has the shape batch_sizex64x128x128x3
Returns:
keras model: the model object storing the information about the network
"""
model = Sequential()
model.add(ZeroPadding3D(padding=(2, 1, 1)))
model.add(
Conv3D(16,
kernel_size=(8, 3, 3),
activation='relu',
strides=(4, 2, 2),
input_shape=(64, 128, 128, 3)))
model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2)))
model.add(ZeroPadding3D(padding=(1, 1, 1)))
model.add(
Conv3D(32, kernel_size=(3, 3, 3), activation='relu',
strides=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2)))
model.add(ZeroPadding3D(padding=(1, 1, 1)))
model.add(
Conv3D(64, kernel_size=(3, 3, 3), activation='relu',
strides=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2)))
model.add(ZeroPadding3D(padding=(0, 1, 1)))
model.add(
Conv3D(128,
kernel_size=(4, 3, 3),
activation='relu',
strides=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2)))
model.add(GlobalMaxPool3D())
model.add(Dense(1024, activation='relu'))
model.add(Dense(512, activation='relu'))
model.add(Dense(5, activation='softmax'))
return model
def ACBlock(input_tensor, name=None, filters=None, ksize=None, strides=(1, 1), use_same_padding=False,
momentum=0.99, epsilon=0.001, moving_mean_initializer='zeros', moving_variance_initializer='ones',
kernel_initializer='glorot_uniform', kernel_regularizer=None, beta_initializer='zeros', gamma_initializer='ones', deploy=False):
if name == None:
raise Exception("Please set the name of this convolution layer")
if filters == None:
raise Exception('Please set the number of filters this convolution layer')
if ksize == None:
raise Exception('Please set the kernel size of this convolution layer')
if not isinstance(ksize, int):
raise Exception('kernel size must be an integer')
pad_size = ksize // 2
if deploy:
outs_fusion = ZeroPadding3D(padding=(pad_size, pad_size, pad_size), name=name+'.fused_pad')(input_tensor)
outs_fusion = Conv3D(filters=filters, kernel_size=(ksize, ksize, ksize), strides=strides, padding='valid', kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer, use_bias=True, name=name+'.fused_conv')(outs_fusion)
return outs_fusion
else:
if use_same_padding:
outs_nxnxn = Conv3D(filters=filters, kernel_size=(ksize, ksize, ksize), strides=strides, padding='same', kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer, use_bias=True, name=name+'.conv_nxnxn')(input_tensor)
else:
outs_nxnxn = ZeroPadding3D(padding=(pad_size, pad_size, pad_size), name=name+'.pad_nxnxn')(input_tensor)
outs_nxnxn = Conv3D(filters=filters, kernel_size=(ksize, ksize, ksize), strides=strides, padding='valid', kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer, use_bias=True, name=name+'.conv_nxnxn')(outs_nxnxn)
outs_1xnxn = ZeroPadding3D(padding=(0, pad_size, pad_size), name=name+'.pad_1xnxn')(input_tensor)
outs_nx1xn = ZeroPadding3D(padding=(pad_size, 0, pad_size), name=name+'.pad_nx1xn')(input_tensor)
outs_nxnx1 = ZeroPadding3D(padding=(pad_size, pad_size, 0), name=name+'_pad_nxnx1')(input_tensor)
outs_1xnxn = Conv3D(filters=filters, kernel_size=(1, ksize, ksize), strides=strides, padding='valid', kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer, use_bias=False, name=name+'.conv_1xnxn')(outs_1xnxn)
outs_nx1xn = Conv3D(filters=filters, kernel_size=(ksize, 1, ksize), strides=strides, padding='valid', kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer, use_bias=False, name=name+'.conv_nx1xn')(outs_nx1xn)
outs_nxnx1 = Conv3D(filters=filters, kernel_size=(ksize, ksize, 1), strides=strides, padding='valid', kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer, use_bias=False, name=name+'.conv_nxnx1')(outs_nxnx1)
outs_nxnxn = BatchNormalization(momentum=momentum, epsilon=epsilon, beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer, moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer, name=name+'.bn_nxnxn')(outs_nxnxn)
outs_1xnxn = BatchNormalization(momentum=momentum, epsilon=epsilon, beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer, moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer, name=name+'.bn_1xnxn')(outs_1xnxn)
outs_nx1xn = BatchNormalization(momentum=momentum, epsilon=epsilon, beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer, moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer, name=name+'.bn_nx1xn')(outs_nx1xn)
outs_nxnx1 = BatchNormalization(momentum=momentum, epsilon=epsilon, beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer, moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer, name=name+'.bn_nxnx1')(outs_nxnx1)
added = Add(name = name+'.add')([outs_nxnxn, outs_1xnxn, outs_nx1xn, outs_nxnx1])
return added
def contract3d(prev_layer, n_kernel, kernel_size, pool_size, padding, act,
pooling):
conv = ZeroPadding3D(padding=(0, kernel_size // 2,
kernel_size // 2))(prev_layer)
conv = Conv3D(n_kernel, kernel_size, padding='valid')(conv)
conv = Activation(act)(BatchNormalization()(conv))
n_kernel = n_kernel << 1
conv = ZeroPadding3D(padding=(0, kernel_size // 2, kernel_size // 2))(conv)
conv = Conv3D(n_kernel, kernel_size, padding='valid')(conv)
conv = Activation(act)(BatchNormalization()(conv))
pool = MaxPooling3D(pool_size=pool_size, strides=pool_size)((conv))
return conv, pool
def build_critic(self):
model = Sequential()
model.add(
Conv3D(filters=32,
kernel_size=(2, 2, 10),
padding='same',
input_shape=self.img_shape))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(rate=0.25))
model.add(ZeroPadding3D(padding=(2, 2, 2)))
model.add(
Conv3D(filters=8,
kernel_size=(2, 2, 10),
padding='valid'
#, data_format='channels_last'
))
model.add(LeakyReLU())
model.add(BatchNormalization(momentum=0.8))
model.add(Dropout(rate=0.25))
model.add(ZeroPadding3D(padding=(2, 2, 2)))
model.add(Conv3D(filters=8, kernel_size=(2, 2, 2), padding='valid'))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dropout(rate=0.25))
model.add(ZeroPadding3D(padding=(1, 1, 1)))
model.add(
Conv3D(filters=8,
kernel_size=(2, 2, 2),
padding='valid'
#, data_format='channels_last'
))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dropout(rate=0.25))
model.add(AveragePooling3D(pool_size=(2, 2, 2)))
model.add(Flatten())
model.add(Dense(1))
model.summary()
img = Input(shape=self.img_shape)
validity = model(img)
return Model(img, validity)
def convolution_factory_3d(model, number_of_filters,
kernel_size=(3, 3, 3),
dropout_rate=0.0, weight_decay=1e-4):
# Bottleneck layer
model = BatchNormalization(axis=concatenation_axis)(model)
model = Scale(axis=concatenation_axis)(model)
model = Activation('relu')(model)
model = Conv3D(filters=(number_of_filters * 4),
kernel_size=(1, 1, 1),
use_bias=False)(model)
if dropout_rate > 0.0:
model = Dropout(rate=dropout_rate)(model)
# Convolution layer
model = BatchNormalization(axis=concatenation_axis,
epsilon=1.1e-5)(model)
model = Scale(axis=concatenation_axis)(model)
model = Activation(activation='relu')(model)
model = ZeroPadding3D(padding=(1, 1, 1))(model)
model = Conv3D(filters=number_of_filters,
kernel_size=kernel_size,
use_bias=False)(model)
if dropout_rate > 0.0:
model = Dropout(rate=dropout_rate)(model)
return(model)
def SDN_ver11(inputs):
z1_1 = incept(inputs, 8) #location of it?
z1_2 = Conv3D(32, par['kernel_size'], strides=2, padding='valid')(z1_1)
z1_2 = PReLU(shared_axes=[4])(z1_2)
z2_2 = Conv3D(64, par['kernel_size'], strides=2, padding='valid')(z1_2)
z2_2 = PReLU(shared_axes=[4])(z2_2)
# z2_3 = incept4(z2_2, 16, 'linear')
# z2_2 = add([z2_2, z2_3])
z4 = Conv3DTranspose(64, par['kernel_size'], strides=2,
padding='valid')(z2_2)
z4 = Conv3D(64, par['kernel_size'], padding='same',
activation='linear')(z4)
#z4 = add([z4, z1_2])
z5 = Conv3DTranspose(32, par['kernel_size'], strides=2,
padding='valid')(z4)
z5 = Conv3D(32, par['kernel_size'], padding='same',
activation='linear')(z5)
#z5 = add([z5, z1_1])
z5 = ZeroPadding3D(
((0, 1), (0, 1), (0, 1)))(z5) #Extra padding to make size match
zzzz = Conv3D(3,
par['kernel_size'],
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-2),
activation='linear')(z5)
return zzzz
def conv_block3d(x, stage, branch, nb_filter, dropout_rate=None, weight_decay=1e-4):
'''Apply BatchNorm, Relu, bottleneck 1x1 Conv3D, 3x3 Conv3D, and option dropout
# Arguments
x: input tensor
stage: index for dense block
branch: layer index within each dense block
nb_filter: number of filters
dropout_rate: dropout rate
weight_decay: weight decay factor
'''
eps = 1.1e-5
conv_name_base = '3dconv' + str(stage) + '_' + str(branch)
relu_name_base = '3drelu' + str(stage) + '_' + str(branch)
# 1x1 Convolution (Bottleneck layer)
inter_channel = nb_filter * 4
x = BatchNormalization(epsilon=eps, axis=4, name=conv_name_base+'_x1_bn', momentum=1.0, trainable=False)(x, training=False)
x = Scale(axis=4, name=conv_name_base+'_x1_scale')(x)
x = Activation('relu', name=relu_name_base+'_x1')(x)
x = Conv3D(inter_channel, (1, 1, 1), name=conv_name_base+'_x1', use_bias=False)(x)
if dropout_rate:
x = Dropout(dropout_rate)(x)
# 3x3 Convolution
x = BatchNormalization(epsilon=eps, axis=4, name=conv_name_base+'_x2_bn', momentum=1.0, trainable=False)(x, training=False)
x = Scale(axis=4, name=conv_name_base+'_x2_scale')(x)
x = Activation('relu', name=relu_name_base+'_x2')(x)
x = ZeroPadding3D((1, 1, 1), name=conv_name_base+'_x2_zeropadding')(x)
x = Conv3D(nb_filter, (3, 3, 3), name=conv_name_base+'_x2', use_bias=False)(x)
if dropout_rate:
x = Dropout(dropout_rate)(x)
return x
def deconv3d(layer_input, skip_input, filters, axis=-1, se_res_block=True, se_ratio=16):
u1 = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(layer_input)
u1 = Conv3DTranspose(filters, (2, 2, 2), strides=(2, 2, 2), use_bias=False, padding='same')(u1)
u1 = InstanceNormalization(axis=axis)(u1)
# u1 = LeakyReLU(alpha=0.3)(u1)
u1 = Activation('selu')(u1)
u1 = CropToConcat3D()([u1, skip_input])
u2 = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u1)
u2 = InstanceNormalization(axis = axis)(u2)
# u2 = LeakyReLU(alpha=0.3)(u2)
u2 = Activation('selu')(u2)
u2 = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u2)
u2 = InstanceNormalization(axis = axis)(u2)
if se_res_block == True:
se = GlobalAveragePooling3D()(u2)
se = Dense(filters // se_ratio, activation='relu')(se)
se = Dense(filters, activation='sigmoid')(se)
se = Reshape([1, 1, 1, filters])(se)
u2 = Multiply()([u2, se])
shortcut = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u1)
shortcut = InstanceNormalization(axis=axis)(shortcut)
u2 = layers.add([u2, shortcut])
# u2 = LeakyReLU(alpha=0.3)(u2)
u2 = Activation('selu')(u2)
return u2
def _vshifted_conv(x, num_filters, name):
"""
Vertically shifted 3-d convolution
"""
filter_size = [3, 3, 3]
# Assumes the height is the second dimension
k = filter_size[1] // 2
### 2d code ###
# x = ZeroPadding2D([[k,0],[0,0]])(x)
# x = Conv2D(filters=num_filters, kernel_size=filter_size, padding='same', kernel_initializer='he_normal', name=name)(x)
# x = LeakyReLU(0.1)(x)
# x = Cropping2D([[0,k],[0,0]])(x)
### 3d adaptation ###
# assumes first tuple is frame number, second is height, 3rd is width
# padding on height
x = ZeroPadding3D([[0, 0], [k, 0], [0, 0]])(x)
x = Conv3D(filters=num_filters,
kernel_size=filter_size,
padding='same',
kernel_initializer='he_normal',
name=name)(x)
x = LeakyReLU(0.1)(x)
x = Cropping3D([[0, 0], [0, k], [0, 0]])(x)
return x
def build_discriminator(self):
model = Sequential()
model.add(Conv3D(32, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv3D(64, kernel_size=3, strides=2, padding="same"))
model.add(ZeroPadding3D(padding=((0,1),(0,1),(0,1))))
model.add(BatchNormalization(momentum=0.8))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv3D(128, kernel_size=3, strides=2, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv3D(256, kernel_size=3, strides=1, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(1, activation='sigmoid'))
model.summary()
img = Input(shape=self.img_shape)
validity = model(img)
return Model(img, validity)
def Unet(x_in, vol_size, enc_nf, dec_nf, full_size=True):
"""
unet network for voxelmorph
Args:
vol_size: volume size. e.g. (256, 256, 256)
enc_nf: encoder filters. right now it needs to be to 1x4.
e.g. [16,32,32,32]
TODO: make this flexible.
dec_nf: encoder filters. right now it's forced to be 1x7.
e.g. [32,32,32,32,8,8,3]
TODO: make this flexible.
full_size
"""
x_in = ZeroPadding3D(((3, 2), (2, 1), (3, 2)))(x_in)
# down-sample path.
x0 = myConv(x_in, enc_nf[0], 2) # 80x96x112
x1 = myConv(x0, enc_nf[1], 2) # 40x48x56
x2 = myConv(x1, enc_nf[2], 2) # 20x24x28
x3 = myConv(x2, enc_nf[3], 2) # 10x12x14
# up-sample path.
x = myConv(x3, dec_nf[0])
x = UpSampling3D()(x)
#x2 = ZeroPadding3D(((1,0), (0,0), (1,0)))(x2)
x = concatenate([x, x2])
x = myConv(x, dec_nf[1])
x = UpSampling3D()(x)
#x1 = ZeroPadding3D(((1,0), (0,0), (1,0)))(x1)
x = concatenate([x, x1])
x = myConv(x, dec_nf[2])
x = UpSampling3D()(x)
#x0 = ZeroPadding3D(((1,1), (1,0), (1,1)))(x0)
x = concatenate([x, x0])
x = myConv(x, dec_nf[3])
x = myConv(x, dec_nf[4])
if full_size:
x = UpSampling3D()(x)
x = concatenate([x, x_in])
x = myConv(x, dec_nf[5])
# optional convolution
if (len(dec_nf) == 8):
x = myConv(x, dec_nf[6])
# transform the results into a flow.
flow = Conv3D(dec_nf[-1],
kernel_size=3,
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5),
name='flow')(x)
flow = Lambda(lambda x: x[:, 3:-2, 2:-1, 3:-2, :])(
flow) #truncate to match size
return flow
def modelC3D(cluster_length, height, width, channel,summary=False, load_weights=True):
## Imports
import numpy as np
import os
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Activation, Dense, Dropout, Flatten, MaxPooling2D, Input, Concatenate,MaxPooling3D, Reshape, ZeroPadding3D
from keras.layers.convolutional import Conv3D
from keras.layers.convolutional_recurrent import ConvLSTM2D
from keras.layers.normalization import BatchNormalization
from keras.models import Model
import numpy as np
import pylab as plt
import keras
import h5py
import keras.backend as K
'''
Model implementation of C3D network used for activity recognition
Ref : http://arxiv.org/pdf/1412.0767.pdf
Courtesy : https://github.com/imatge-upc/activitynet-2016-cvprw/tree/master/data
'''
input_cnn = Input(shape=(cluster_length, height, width, channel))
cnn_1=Conv3D(filters=64, kernel_size=(3,3,3),padding='same',activation='relu',name='conv1')(input_cnn)
pool_1=MaxPooling3D(pool_size=(1, 2, 2),strides=(1, 2, 2),padding='valid',name='pool1')(cnn_1)
cnn_2=Conv3D(filters=128, kernel_size=(3,3,3),padding='same',activation='relu',name='conv2')(pool_1)
pool_2=MaxPooling3D(pool_size=(2, 2, 2),strides=(2, 2, 2),padding='valid',name='pool2')(cnn_2)
cnn_3a=Conv3D(filters=256, kernel_size=(3,3,3),padding='same',activation='relu',name='conv3a')(pool_2)
cnn_3b=Conv3D(filters=256, kernel_size=(3,3,3),padding='same',activation='relu',name='conv3b')(cnn_3a)
pool_3=MaxPooling3D(pool_size=(2,2,2),strides=(2,2,2),padding='valid',name='pool3')(cnn_3b)
cnn_4a=Conv3D(filters=512, kernel_size=(3,3,3),padding='same',activation='relu',name='conv4a')(pool_3)
cnn_4b=Conv3D(filters=512, kernel_size=(3,3,3),padding='same',activation='relu',name='conv4b')(cnn_4a)
#zero_4=ZeroPadding3D(padding=((0, 0), (0, 1), (0, 1)), name='zeropad4')(cnn_4b)
pool_4=MaxPooling3D(pool_size=(2, 2, 2),strides=(2, 2, 2),padding='valid',name='pool4')(cnn_4b)
cnn_5a=Conv3D(filters=512, kernel_size=(3,3,0),padding='same',activation='relu',name='conv5a')(pool_4)
cnn_5b=Conv3D(filters=512, kernel_size=(3,3,0),padding='same',activation='relu',name='conv5b')(cnn_5a)
zero_5=ZeroPadding3D(padding=((0, 1), (0, 0), (0, 0)), name='zeropad5')(cnn_5b)
pool_5=MaxPooling3D(pool_size=(2, 2, 2),strides=(2, 2, 2),padding='valid',name='pool5')(zero_5)
flat= Flatten(name='flatten')(pool_5)
#flat=Reshape([1,-1])(pool_3)
# FC layers group
fc_1=Dense(4096, activation='relu',name='fc6')(flat)
drop_1=Dropout(0.5, name='do1')(fc_1)
fc_2=Dense(4096, activation='relu',name='fc7')(drop_1)
drop_2=Dropout(0.5, name='do2')(fc_2)
fc_2=Dense(487, activation='sigmoid', name='fc8')(drop_2)
model_cnn=Model(input_cnn,fc_2)
if load_weights:
model_cnn.load_weights('/nobackup/leopauly/c3d-sports1M_weights.h5')
if summary:
print(model_cnn.summary())
return model_cnn
def ConvNet(input_shape = (64, 64, 64, 1), classes = 2):
X_input = Input(shape = input_shape)
X = ZeroPadding3D((1, 1, 1), data_format = 'channels_last')(X_input)
X = Conv3D(16, (3, 3, 3), strides = 1, padding = 'same', data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X_input)
X = Conv3D(16, (3, 3, 3), strides = 1, padding = 'same', data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = Conv3D(32, (3, 3, 3), strides = 1, padding = 'same', data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = Conv3D(32, (3, 3, 3), strides = 1, padding = 'same', data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = Conv3D(64, (3, 3, 3), strides = 1, padding = 'same', data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = Conv3D(64, (3, 3, 3), strides = 1, padding = 'same', data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = Conv3D(128, (3, 3, 3), strides = 1, padding = 'same', data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = Conv3D(256, (3, 3, 3), strides = 1, padding = 'same', data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = Conv3D(16, (3, 3, 3), strides = 1, data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X_input)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = Conv3D(32, (3, 3, 3), strides = 1, data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = Conv3D(64, (3, 3, 3), strides = 1, data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = Conv3D(64, (3, 3, 3), strides = 1, data_format = 'channels_last', kernel_initializer = glorot_uniform(seed = 0))(X)
X = BatchNormalization()(X)
X = Activation('relu')(X)
X = MaxPooling3D((2, 2, 2), strides = 2)(X)
X = AveragePooling3D((2, 2, 2),strides = 2,padding = 'valid')(X)
X = Flatten()(X)
X = Dense(32, activation = 'relu')(X)
X = Dropout(0.2)(X)
X = Dense(16, activation = 'relu')(X)
X = Dropout(0.2)(X)
X = Dense(8, activation = 'relu')(X)
X = Dense(classes, activation = 'softmax')(X)
model = Model(inputs = X_input, outputs = X, name = 'ResNet')
return model
def Conv3dBLSTM(self) :
model = Sequential()
# 1st layer group
model = Sequential()
# 1st layer group
# (samples, conv_dim1, conv_dim2, conv_dim3, channels)
model.add(Conv3D(64, 3, 3, 3, activation='relu',
border_mode='same', name='conv1',
subsample=(1, 1, 1),
input_shape=self.shapeOfInputConv3d)) # 40 80 80 3
model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2),
border_mode='valid', name='pool1'))
# 2nd layer group
model.add(Conv3D(128, 3, 3, 3, activation='relu',
border_mode='same', name='conv2',
subsample=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool2'))
# 3rd layer group
model.add(Conv3D(256, 3, 3, 3, activation='relu',
border_mode='same', name='conv3a',
subsample=(1, 1, 1)))
model.add(Conv3D(256, 3, 3, 3, activation='relu',
border_mode='same', name='conv3b',
subsample=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool3'))
# 4th layer group
model.add(Conv3D(512, 3, 3, 3, activation='relu',
border_mode='same', name='conv4a',
subsample=(1, 1, 1)))
model.add(Conv3D(512, 3, 3, 3, activation='relu',
border_mode='same', name='conv4b',
subsample=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool4'))
# 5th layer group
model.add(Conv3D(512, 3, 3, 3, activation='relu',
border_mode='same', name='conv5a',
subsample=(1, 1, 1)))
model.add(Conv3D(512, 3, 3, 3, activation='relu',
border_mode='same', name='conv5b',
subsample=(1, 1, 1)))
model.add(ZeroPadding3D(padding=(0, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool5'))
model.add(Reshape(target_shape=(2, 512* 3* 3)))
# (batch_size, timesteps, input_dim)
model.add(Bidirectional(LSTM(512* 3* 3, input_shape=self.shapeOfInputLSTM,
return_sequences=False,
activation="softmax",
dropout=0.5)))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(self.classesNumber, activation='softmax'))
return model
def create_3D_CNN_model():
global model
model = Sequential(name="3D-CNN Model")
# 1st layer
model.add(Conv3D(64, (3, 3, 3), activation="relu", name="conv1",
input_shape=(D, W, H, C),
strides=(1, 1, 1), padding="same"))
model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2), name="pool1", padding="valid"))
# 2nd layer
model.add(Conv3D(128, (3, 3, 3), activation="relu", name="conv2",
strides=(1, 1, 1), padding="same"))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), name="pool2", padding="valid"))
# 3rd layer
model.add(Conv3D(256, (3, 3, 3), activation="relu", name="conv3a",
strides=(1, 1, 1), padding="same"))
model.add(Conv3D(256, (3, 3, 3), activation="relu", name="conv3b",
strides=(1, 1, 1), padding="same"))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), name="pool3", padding="valid"))
# 4th layer
model.add(Conv3D(512, (3, 3, 3), activation="relu", name="conv4a",
strides=(1, 1, 1), padding="same"))
model.add(Conv3D(512, (3, 3, 3), activation="relu", name="conv4b",
strides=(1, 1, 1), padding="same"))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), name="pool4", padding="valid"))
# 5th layer
model.add(Conv3D(512, (3, 3, 3), activation="relu", name="conv5a",
strides=(1, 1, 1), padding="same"))
model.add(Conv3D(512, (3, 3, 3), activation="relu", name="conv5b",
strides=(1, 1, 1), padding="same"))
model.add(ZeroPadding3D(padding=(0, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), name="pool5", padding="valid"))
model.add(Flatten())
# FC layers group
model.add(Dense(4096, activation='relu', name='fc6'))
model.add(Dropout(0.5))
model.add(Dense(4096, activation='relu', name='fc7'))
model.add(Dropout(.5))
model.add(Dense(101, activation='softmax', name='fc8'))
if weights_path:
model.load_weights(weights_path)
model.layers.pop()
pre_last_layer_output = model.layers[-1].output
last_layer_output = Dense(5, activation='softmax', name='fc9')(pre_last_layer_output)
model = Model(model.input, last_layer_output)
for layer in model.layers[:-5]:
layer.trainable = False
def SDN_ver2(inputs): #should control the size carefully, larger strides to downsample
# z1_1 = Conv3D(32, par['kernel_size'], padding = 'same')(inputs)
z1_2 = Conv3D(32, par['kernel_size'], strides = par['kernel_strides'], padding = 'valid', activation = 'linear')(inputs)
#z1_2 = BatchNormalization()(z1_2)
z1_2 = PReLU(shared_axes = [4])(z1_2)
# z2_1 = Conv3D(64, par['kernel_size'], padding = 'same')(z1_2)
z2_2 = Conv3D(64, par['kernel_size'], strides = par['kernel_strides'], padding = 'valid', activation = 'linear')(z1_2)
# z2_2 = BatchNormalization()(z2_2)
z2_2 = PReLU(shared_axes = [4])(z2_2)
z4x = Conv3DTranspose(64, par['kernel_size'], strides= par['kernel_strides'], padding = 'valid', activation = 'linear')(z2_2)
z4x = Conv3D(64, (2,2,2), padding = 'same')(z4x)
z5x = Conv3DTranspose(32, par['kernel_size'], strides= par['kernel_strides'], padding = 'valid', activation = 'linear')(z4x)
z5x = Conv3D(32, (2,2,2), padding = 'same')(z5x)
z5x = ZeroPadding3D((2,1,2))(z5x)
zzzzx = Conv3D(1, par['kernel_size'], padding = 'valid', activation = 'tanh')(z5x)
z4y = Conv3DTranspose(64, par['kernel_size'], strides= par['kernel_strides'], padding = 'valid',activation = 'linear')(z2_2)
z4y = Conv3D(64, (2,2,2), padding = 'same')(z4y)
z5y = Conv3DTranspose(32, par['kernel_size'], strides= par['kernel_strides'], padding = 'valid', activation = 'linear')(z4y)
z5y = Conv3D(32, (2,2,2), padding = 'same')(z5y)
z5y = ZeroPadding3D((2,1,2))(z5y)
zzzzy = Conv3D(1, par['kernel_size'], padding = 'valid', activation = 'tanh')(z5y)
z4z = Conv3DTranspose(64, par['kernel_size'], strides= par['kernel_strides'], padding = 'valid', activation = 'linear')(z2_2)
z4z = Conv3D(64, (2,2,2), padding = 'same')(z4z)
z5z = Conv3DTranspose(32, par['kernel_size'], strides= par['kernel_strides'], padding = 'valid', activation = 'linear')(z4z)
z5z = Conv3D(32, (2,2,2), padding = 'same')(z5z)
z5z = ZeroPadding3D((2,1,2))(z5z)
zzzzz = Conv3D(1, par['kernel_size'], padding = 'valid', activation = 'tanh')(z5z)
zzzz = concatenate([zzzzx, zzzzy, zzzzz], axis = -1)
locnet = Model(inputs, zzzz)
x1 = SpatialDeformer3D(localization_net=locnet, output_size=(input_shape[0],input_shape[1], input_shape[2]), input_shape=input_shape)(inputs)
return x1, locnet(inputs)
def get_cnn():
sae = load_model(SAE_PATH)
# sae = get_sae()
print(sae.summary())
print("Wha?")
x = sae.layers[15].output
x = Conv3D(64, kernel_size=(3, 3, 3), padding='same',
name='cnn_conv3d_4')(x)
x = Activation('relu', name='cnn_activation_4')(x)
x = ZeroPadding3D((1, 1, 1))(x)
x = Conv3D(64, kernel_size=(3, 3, 3), padding='same',
name='cnn_conv3d_5')(x)
x = Activation('relu', name='cnn_activation_5')(x)
x = MaxPooling3D(pool_size=(2, 2, 2), name='cnn_max_pooling3d_3')(x)
x = ZeroPadding3D((1, 1, 1))(x)
x = Conv3D(128, kernel_size=(3, 3, 3), padding='same',
name='cnn_conv3d_6')(x)
x = Activation('relu', name='cnn_activation_6')(x)
x = ZeroPadding3D((1, 1, 1))(x)
x = Conv3D(128, kernel_size=(3, 3, 3), padding='same',
name='cnn_conv3d_7')(x)
x = Activation('relu', name='cnn_activation_7')(x)
x = MaxPooling3D(pool_size=(2, 2, 2), name='cnn_max_pooling3d_4')(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
x = Dense(5, activation='sigmoid')(x)
model = Model(inputs=sae.input, outputs=x)
for layer in sae.layers:
layer.trainable = False
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.SGD(lr=0.05))
print(model.summary())
return model
def VGG11_3D(weights_path=None):
model = Sequential()
model.add(ZeroPadding3D((0, 1, 1), input_shape=(32, 32, 32, 1)))
model.add(Convolution3D(64, 3, 3, 3, activation='relu', dim_ordering='tf'))
model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2)))
model.add(ZeroPadding3D((0, 1, 1)))
model.add(Convolution3D(128, 3, 3, 3, activation='relu',
dim_ordering='tf'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2),
dim_ordering='tf'))
model.add(ZeroPadding3D((0, 1, 1)))
model.add(Convolution3D(256, 3, 3, 3, activation='relu',
dim_ordering='tf'))
model.add(ZeroPadding3D((0, 1, 1)))
model.add(Convolution3D(256, 3, 3, 3, activation='relu',
dim_ordering='tf'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2),
dim_ordering='tf'))
model.add(ZeroPadding3D((0, 1, 1)))
model.add(Convolution3D(512, 3, 3, 3, activation='relu',
dim_ordering='tf'))
model.add(ZeroPadding3D((0, 1, 1)))
model.add(Convolution3D(512, 3, 3, 3, activation='relu',
dim_ordering='tf'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
dim_ordering='tf'))
model.add(ZeroPadding3D((0, 1, 1)))
model.add(Convolution3D(512, 3, 3, 3, activation='relu',
dim_ordering='tf'))
model.add(ZeroPadding3D((0, 1, 1)))
model.add(Convolution3D(512, 3, 3, 3, activation='relu',
dim_ordering='tf'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
dim_ordering='tf'))
model.add(Flatten())
xout = model.output_shape
xin = model.input_shape
model.add(Dense(4096, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(4096, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2, activation='softmax'))
return model
def get_model(summary=False):
""" Return the Keras model of the network
"""
model = Sequential()
# 1st layer group
model.add(Convolution3D(64, 3, 3, 3, trainable=False, activation='relu',
border_mode='same', name='conv1',
subsample=(1, 1, 1),
input_shape=(3, 16, 112, 112)))
model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2),
border_mode='valid', name='pool1'))
# 2nd layer group
model.add(Convolution3D(128, 3, 3, 3, trainable=False, activation='relu',
border_mode='same', name='conv2',
subsample=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool2'))
# 3rd layer group
model.add(Convolution3D(256, 3, 3, 3, trainable=False, activation='relu',
border_mode='same', name='conv3a',
subsample=(1, 1, 1)))
model.add(Convolution3D(256, 3, 3, 3, trainable=False, activation='relu',
border_mode='same', name='conv3b',
subsample=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool3'))
# 4th layer group
model.add(Convolution3D(512, 3, 3, 3, trainable=False, activation='relu',
border_mode='same', name='conv4a',
subsample=(1, 1, 1)))
model.add(Convolution3D(512, 3, 3, 3, activation='relu',
border_mode='same', name='conv4b',
subsample=(1, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool4'))
# 5th layer group
model.add(Convolution3D(512, 3, 3, 3, trainable=False, activation='relu',
border_mode='same', name='conv5a',
subsample=(1, 1, 1)))
model.add(Convolution3D(512, 3, 3, 3, activation='relu',
border_mode='same', name='conv5b',
subsample=(1, 1, 1)))
model.add(ZeroPadding3D(padding=(0, 1, 1)))
model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool5'))
model.add(Flatten())
# FC layers group
model.add(Dense(4096, activation='relu', name='fc6'))
model.add(Dropout(0))
model.add(Dense(4096, activation='relu', name='fc7'))
model.add(Dropout(0))
model.add(Dense(487, activation='softmax', name='fc8'))
if summary:
print(model.summary())
return model
def conv_block(x,
stage,
branch,
nb_filter,
dropout_rate=None,
weight_decay=1e-4):
'''Apply BatchNorm, Relu, bottleneck 1x1 Conv2D, 3x3 Conv2D, and option dropout
# Arguments
x: input tensor
stage: index for dense block
branch: layer index within each dense block
nb_filter: number of filters
dropout_rate: dropout rate
weight_decay: weight decay factor
'''
eps = 1.1e-5
conv_name_base = 'conv' + str(stage) + '_' + str(branch)
relu_name_base = 'relu' + str(stage) + '_' + str(branch)
## 1x1x1 Convolution (Bottleneck layer)
## Note: I'm not really sure what this 4 is. This isn't the number of layers (see nb_layers above)
inter_channel = nb_filter * 4
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name=conv_name_base + '_x1_bn')(x)
x = Scale(axis=concat_axis, name=conv_name_base + '_x1_scale')(x)
x = Activation('relu', name=relu_name_base + '_x1')(x)
x = Convolution3D(inter_channel,
1,
1,
1,
name=conv_name_base + '_x1',
bias=False)(x)
if dropout_rate:
x = Dropout(dropout_rate)(x)
## 3x3x3 Convolution
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name=conv_name_base + '_x2_bn')(x)
x = Scale(axis=concat_axis, name=conv_name_base + '_x2_scale')(x)
x = Activation('relu', name=relu_name_base + '_x2')(x)
x = ZeroPadding3D((1, 1, 1), name=conv_name_base + '_x2_zeropadding')(x)
x = Convolution3D(nb_filter,
3,
3,
3,
name=conv_name_base + '_x2',
bias=False)(x)
if dropout_rate:
x = Dropout(dropout_rate)(x)
return x
def model_main(rgb):
import keras.backend as K
from keras.models import Sequential
from keras.layers.core import Dense,Activation
from keras.layers import Convolution3D,MaxPooling3D,TimeDistributed,Flatten,ZeroPadding3D,Dropout,Lambda
nb_classes = 249
sum_dim1 = Lambda(lambda xin: K.sum(xin, axis = 1), output_shape=(1024,))
model = Sequential()
# 1st layer group
if rgb:
model.add(TimeDistributed(Convolution3D(16, 3, 3, 3, activation='relu',
border_mode='same', name='conv1',
subsample=(1, 1, 1)),
input_shape=(None,3,8, 112, 112)))
else:
model.add(TimeDistributed(Convolution3D(16, 3, 3, 3, activation='relu',
border_mode='same', name='conv1',
subsample=(1, 1, 1)),
input_shape=(None,1,8, 112, 112)))
model.add(TimeDistributed(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2),
border_mode='valid', name='pool1')))
# 2nd layer group
model.add(TimeDistributed(Convolution3D(16, 3, 3, 3, activation='relu',
border_mode='same', name='conv2',
subsample=(1, 1, 1))))
model.add(TimeDistributed(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool2')))
# 3rd layer group
model.add(TimeDistributed(Convolution3D(32, 3, 3, 3, activation='relu',
border_mode='same', name='conv3b',
subsample=(1, 1, 1))))
model.add(TimeDistributed(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool3')))
# 4th layer group
model.add(TimeDistributed(Convolution3D(64, 3, 3, 3, activation='relu',
border_mode='same', name='conv4b',
subsample=(1, 1, 1))))
model.add(TimeDistributed(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool4')))
model.add(TimeDistributed(ZeroPadding3D(padding=(0, 1, 1), name='zeropadding')))
model.add(TimeDistributed(MaxPooling3D(pool_size=(1, 2, 2), strides=(2, 2, 2),
border_mode='valid', name='pool5')))
model.add(TimeDistributed(Flatten(name='flatten')))
# FC layers group
model.add(sum_dim1)
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
return model