def MT_CAN_3D(n_frame,
nb_filters1,
nb_filters2,
input_shape,
kernel_size=(3, 3, 3),
dropout_rate1=0.25,
dropout_rate2=0.5,
pool_size=(2, 2, 2),
nb_dense=128):
diff_input = Input(shape=input_shape)
rawf_input = Input(shape=input_shape)
d1 = Conv3D(nb_filters1, kernel_size, padding='same',
activation='tanh')(diff_input)
d2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(d1)
# Appearance Branch
r1 = Conv3D(nb_filters1, kernel_size, padding='same',
activation='tanh')(rawf_input)
r2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(r1)
g1 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r2)
g1 = Attention_mask()(g1)
gated1 = multiply([d2, g1])
d3 = AveragePooling3D(pool_size)(gated1)
d4 = Dropout(dropout_rate1)(d3)
d5 = Conv3D(nb_filters2, kernel_size, padding='same',
activation='tanh')(d4)
d6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(d5)
r3 = AveragePooling3D(pool_size)(r2)
r4 = Dropout(dropout_rate1)(r3)
r5 = Conv3D(nb_filters2, kernel_size, padding='same',
activation='tanh')(r4)
r6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(r5)
g2 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r6)
g2 = Attention_mask()(g2)
gated2 = multiply([d6, g2])
d7 = AveragePooling3D(pool_size)(gated2)
d8 = Dropout(dropout_rate1)(d7)
d9 = Flatten()(d8)
d10_y = Dense(nb_dense, activation='tanh')(d9)
d11_y = Dropout(dropout_rate2)(d10_y)
out_y = Dense(n_frame, name='output_1')(d11_y)
d10_r = Dense(nb_dense, activation='tanh')(d9)
d11_r = Dropout(dropout_rate2)(d10_r)
out_r = Dense(n_frame, name='output_2')(d11_r)
model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r])
return model
def residual_layer(self, input_x, out_dim, layer_num, res_block=None):
# split + transform(bottleneck) + transition + merge
# input_dim = input_x.get_shape().as_list()[-1]
if res_block is None:
res_block = self.blocks
for i in range(res_block):
input_dim = input_x.get_shape().as_list()[-1]
if input_dim * 2 == out_dim:
flag = True
strides = (2, 2, 2)
channel = input_dim // 2
else:
flag = False
strides = (1, 1, 1)
x = self.split_layer(input_x, strides=strides, layer_name='split_layer_' + layer_num + '_' + str(i))
x = self.transition_layer(x, out_dim=out_dim, scope='trans_layer_' + layer_num + '_' + str(i))
x = self.squeeze_excitation_layer(x, out_dim=out_dim, ratio=self.reduction_ratio, layer_name='squeeze_layer_' + layer_num + '_' + str(i))
if flag is True:
pad_input_x = AveragePooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), padding='same')(input_x)
pad_input_x = Lambda(lambda _x: tf.pad(_x, [[0, 0], [0, 0], [0, 0], [0, 0], [channel, channel]]), output_shape=x.get_shape().as_list())(pad_input_x) # [?, height, width, channel]
else:
pad_input_x = input_x
input_x = Add()([x, pad_input_x])
input_x = Activation('relu')(input_x)
return input_x
def define_generator(self):
"""Makes a generator that takes a CT image as input to generate a dose distribution of the same dimensions"""
# Define inputs
ct_image = Input(self.ct_shape)
roi_masks = Input(self.roi_masks_shape)
# Build Model starting with Conv3D layers
x = concatenate([ct_image, roi_masks])
x1 = self.generator_convolution(x, self.initial_number_of_filters)
x2 = self.generator_convolution(x1, 2 * self.initial_number_of_filters)
x3 = self.generator_convolution(x2, 4 * self.initial_number_of_filters)
x4 = self.generator_convolution(x3, 8 * self.initial_number_of_filters)
x5 = self.generator_convolution(x4, 8 * self.initial_number_of_filters)
x6 = self.generator_convolution(x5, 8 * self.initial_number_of_filters, use_batch_norm=False)
# Build model back up from bottleneck
x5b = self.generator_convolution_transpose(x6, 8 * self.initial_number_of_filters, use_dropout=False)
x4b = self.generator_convolution_transpose(x5b, 8 * self.initial_number_of_filters, skip_x=x5)
x3b = self.generator_convolution_transpose(x4b, 4 * self.initial_number_of_filters, use_dropout=False, skip_x=x4)
x2b = self.generator_convolution_transpose(x3b, 2 * self.initial_number_of_filters, skip_x=x3)
x1b = self.generator_convolution_transpose(x2b, self.initial_number_of_filters, use_dropout=False, skip_x=x2)
# Final layer
x0b = concatenate([x1b, x1])
x0b = Conv3DTranspose(1, self.filter_size, strides=self.stride_size, padding="same")(x0b)
x_final = AveragePooling3D((3, 3, 3), strides=(1, 1, 1), padding="same")(x0b)
final_dose = Activation("relu")(x_final)
# Compile model for use
self.generator = Model(inputs=[ct_image, roi_masks], outputs=final_dose, name="generator")
self.generator.compile(loss="mean_absolute_error", optimizer=self.gen_optimizer)
self.generator.summary()
def build(self):
input_seq = Input(shape=self.input_shape)
x = Conv3D(filters=32,
kernel_size=(self.filter_dims, 1, 1),
padding='same',
use_bias=False,
kernel_regularizer=l1(1e-3))(input_seq)
x = BatchNormalization(-1)(x)
x = AveragePooling3D(pool_size=(self.filter_dims, 1, 1),
strides=(int(self.filter_dims / 3), 1, 1),
padding='valid')(x)
x = ConvLSTM2D(filters=64,
kernel_size=(3, 3),
return_sequences=False,
padding='same')(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
activation='relu',
kernel_regularizer=l2(1e-3))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
activation='relu',
kernel_regularizer=l2(1e-3))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
activation='relu',
kernel_regularizer=l2(1e-3))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Conv2D(filters=1024,
kernel_size=(5, 5),
activation='relu',
kernel_regularizer=l2(1e-3))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Flatten()(x)
logits = Dense(self.output_classes,
activation='softmax',
kernel_regularizer=l2(1e-3))(x)
model = Model(inputs=input_seq, outputs=logits)
model.summary()
return model
def pooling_combo3D(x, window=(2, 2, 2), prefix='3d_unet_pooling'):
x = concatenate([
MaxPooling3D(pool_size=window,
data_format='channels_last',
name=prefix + "_max")(x),
AveragePooling3D(pool_size=window,
data_format='channels_last',
name=prefix + "_avg")(x)
],
axis=-1,
name=prefix + "_max_and_avg")
return x
def _transmit_block(x, is_last):
bn_scale = PARAMS['bn_scale']
activation = PARAMS['activation']
kernel_initializer = PARAMS['kernel_initializer']
weight_decay = PARAMS['weight_decay']
compression = PARAMS['compression']
x = BatchNormalization(scale=bn_scale, axis=-1)(x)
x = activation()(x)
if is_last:
x = GlobalAvgPool3D()(x)
else:
*_, f = x.get_shape().as_list()
x = Conv3D(f // compression, kernel_size=(1, 1, 1), padding='same', use_bias=True,
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_penalty(weight_decay))(x)
x = AveragePooling3D((2, 2, 2), padding='valid')(x)
return x
def build(self):
input_seq = Input(shape=self.input_shape)
x = Conv3D(filters=32,
kernel_size=(self.filter_dims, 1, 1),
padding='same',
use_bias=False)(input_seq)
x = BatchNormalization(-1)(x)
x = AveragePooling3D(pool_size=(self.filter_dims, 1, 1),
strides=(int(self.filter_dims / 3), 1, 1),
padding='valid')(x)
x = TimeDistributed(
Conv2D(filters=32, kernel_size=(3, 3), padding='same'))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = TimeDistributed(
Conv2D(filters=64, kernel_size=(3, 3), padding='same'))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = TimeDistributed(
Conv2D(filters=128, kernel_size=(3, 3), padding='same'))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Bidirectional(
ConvLSTM2D(filters=32, kernel_size=(3, 3),
return_sequences=True))(x)
x = Bidirectional(
ConvLSTM2D(filters=32, kernel_size=(3, 3),
return_sequences=False))(x)
x = Flatten()(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.5)(x)
logits = Dense(self.output_classes, activation='softmax')(x)
model = Model(inputs=input_seq, outputs=logits)
model.summary()
return model
def test_delete_channels_averagepooling3d(channel_index, data_format):
layer = AveragePooling3D([2, 3, 2], data_format=data_format)
layer_test_helper_flatten_3d(layer, channel_index, data_format=data_format)
def create(self):
"""
Create model and return it
:return: keras model
"""
input_layer = Input(shape=self.input_shape)
x = input_layer
init_size = max(self.input_shape[:-1])
size = init_size
convolutions = self.convolutions
connection = []
i = 0
if self.input_pyramid:
scaled_input = []
scaled_input.append(x)
for i, nbc in enumerate(self.convolutions[:-1]):
ds_input = AveragePooling3D(pool_size=(2, 2,
2))(scaled_input[i])
scaled_input.append(ds_input)
for i, nbc in enumerate(self.convolutions[:-1]):
if not self.input_pyramid or (i == 0):
x, x_before_ds = encoder_block(
x,
nbc,
use_bn=self.encoder_use_bn,
spatial_dropout=self.encoder_spatial_dropout)
else:
x, x_before_ds = encoder_block_pyramid(
x,
scaled_input[i],
nbc,
use_bn=self.encoder_use_bn,
spatial_dropout=self.encoder_spatial_dropout)
connection.insert(
0, x_before_ds
) # Append in reverse order for easier use in the next block
x = convolution_block(x, self.convolutions[-1], self.encoder_use_bn,
self.encoder_spatial_dropout)
connection.insert(0, x)
inverse_conv = self.convolutions[::-1]
inverse_conv = inverse_conv[1:]
decoded_layers = []
for i, nbc in enumerate(inverse_conv):
x = decoder_block(x,
connection[i + 1],
nbc,
use_bn=self.decoder_use_bn,
spatial_dropout=self.decoder_spatial_dropout)
decoded_layers.append(x)
if not self.deep_supervision:
# Final activation layer
x = Convolution3D(self.nb_classes, 1, activation='softmax')(x)
else:
recons_list = []
for i, lay in enumerate(decoded_layers):
x = Convolution3D(self.nb_classes, 1,
activation='softmax')(lay)
recons_list.append(x)
x = recons_list[::-1]
return Model(inputs=input_layer, outputs=x)
def Inception_Inflated3d(include_top=True,
weights=None,
input_tensor=None,
input_shape=None,
dropout_prob=0.0,
endpoint_logit=True,
classes=400):
"""Instantiates the Inflated 3D Inception v1 architecture.
Optionally loads weights pre-trained
on Kinetics. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input frame(image) size for this model is 224x224.
# Arguments
include_top: whether to include the the classification
layer at the top of the network.
weights: one of `None` (random initialization)
or 'kinetics_only' (pre-training on Kinetics dataset only).
or 'imagenet_and_kinetics' (pre-training on ImageNet and Kinetics datasets).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(NUM_FRAMES, 224, 224, 3)` (with `channels_last` data format)
or `(NUM_FRAMES, 3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 inputs channels.
NUM_FRAMES should be no smaller than 8. The authors used 64
frames per example for training and testing on kinetics dataset
Also, Width and height should be no smaller than 32.
E.g. `(64, 150, 150, 3)` would be one valid value.
dropout_prob: optional, dropout probability applied in dropout layer
after global average pooling layer.
0.0 means no dropout is applied, 1.0 means dropout is applied to all features.
Note: Since Dropout is applied just before the classification
layer, it is only useful when `include_top` is set to True.
endpoint_logit: (boolean) optional. If True, the model's forward pass
will end at producing logits. Otherwise, softmax is applied after producing
the logits to produce the class probabilities prediction. Setting this parameter
to True is particularly useful when you want to combine results of rgb model
and optical flow model.
- `True` end model forward pass at logit output
- `False` go further after logit to produce softmax predictions
Note: This parameter is only useful when `include_top` is set to True.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in WEIGHTS_NAME or weights is None or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or %s' %
str(WEIGHTS_NAME) + ' '
'or a valid path to a file containing `weights` values')
if weights in WEIGHTS_NAME and include_top and classes != 400:
raise ValueError('If using `weights` as one of these %s, with `include_top`'
' as true, `classes` should be 400' % str(WEIGHTS_NAME))
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_frame_size=224,
min_frame_size=32,
default_num_frames=64,
min_num_frames=8,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 4
# Downsampling via convolution (spatial and temporal)
x = conv3d_bn(img_input, 64, 7, 7, 7, strides=(2, 2, 2), padding='same', name='Conv3d_1a_7x7')
# Downsampling (spatial only)
x = MaxPooling3D((1, 3, 3), strides=(1, 2, 2), padding='same', name='MaxPool2d_2a_3x3')(x)
x = conv3d_bn(x, 64, 1, 1, 1, strides=(1, 1, 1), padding='same', name='Conv3d_2b_1x1')
x = conv3d_bn(x, 192, 3, 3, 3, strides=(1, 1, 1), padding='same', name='Conv3d_2c_3x3')
# Downsampling (spatial only)
x = MaxPooling3D((1, 3, 3), strides=(1, 2, 2), padding='same', name='MaxPool2d_3a_3x3')(x)
# Mixed 3b
branch_0 = conv3d_bn(x, 64, 1, 1, 1, padding='same', name='Conv3d_3b_0a_1x1')
branch_1 = conv3d_bn(x, 96, 1, 1, 1, padding='same', name='Conv3d_3b_1a_1x1')
branch_1 = conv3d_bn(branch_1, 128, 3, 3, 3, padding='same', name='Conv3d_3b_1b_3x3')
branch_2 = conv3d_bn(x, 16, 1, 1, 1, padding='same', name='Conv3d_3b_2a_1x1')
branch_2 = conv3d_bn(branch_2, 32, 3, 3, 3, padding='same', name='Conv3d_3b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_3b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 32, 1, 1, 1, padding='same', name='Conv3d_3b_3b_1x1')
######edit##############
q = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_3b_3a_3x3')(x)
q = conv3d_bn(q, 32, 1, 1, 1, padding='same', name='Conv3d_3b_3b_1x1')
########################
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_3b')
# Mixed 3c
branch_0 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_3c_0a_1x1')
branch_1 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_3c_1a_1x1')
branch_1 = conv3d_bn(branch_1, 192, 3, 3, 3, padding='same', name='Conv3d_3c_1b_3x3')
branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_3c_2a_1x1')
branch_2 = conv3d_bn(branch_2, 96, 3, 3, 3, padding='same', name='Conv3d_3c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_3c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_3c_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_3c')
# Downsampling (spatial and temporal)
x = MaxPooling3D((3, 3, 3), strides=(2, 2, 2), padding='same', name='MaxPool2d_4a_3x3')(x)
# Mixed 4b
branch_0 = conv3d_bn(x, 192, 1, 1, 1, padding='same', name='Conv3d_4b_0a_1x1')
branch_1 = conv3d_bn(x, 96, 1, 1, 1, padding='same', name='Conv3d_4b_1a_1x1')
branch_1 = conv3d_bn(branch_1, 208, 3, 3, 3, padding='same', name='Conv3d_4b_1b_3x3')
branch_2 = conv3d_bn(x, 16, 1, 1, 1, padding='same', name='Conv3d_4b_2a_1x1')
branch_2 = conv3d_bn(branch_2, 48, 3, 3, 3, padding='same', name='Conv3d_4b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4b_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4b')
# Mixed 4c
branch_0 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_4c_0a_1x1')
branch_1 = conv3d_bn(x, 112, 1, 1, 1, padding='same', name='Conv3d_4c_1a_1x1')
branch_1 = conv3d_bn(branch_1, 224, 3, 3, 3, padding='same', name='Conv3d_4c_1b_3x3')
branch_2 = conv3d_bn(x, 24, 1, 1, 1, padding='same', name='Conv3d_4c_2a_1x1')
branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4c_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4c')
# Mixed 4d
branch_0 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_4d_0a_1x1')
branch_1 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_4d_1a_1x1')
branch_1 = conv3d_bn(branch_1, 256, 3, 3, 3, padding='same', name='Conv3d_4d_1b_3x3')
branch_2 = conv3d_bn(x, 24, 1, 1, 1, padding='same', name='Conv3d_4d_2a_1x1')
branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4d_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4d_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4d_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4d')
# Mixed 4e
branch_0 = conv3d_bn(x, 112, 1, 1, 1, padding='same', name='Conv3d_4e_0a_1x1')
branch_1 = conv3d_bn(x, 144, 1, 1, 1, padding='same', name='Conv3d_4e_1a_1x1')
branch_1 = conv3d_bn(branch_1, 288, 3, 3, 3, padding='same', name='Conv3d_4e_1b_3x3')
branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_4e_2a_1x1')
branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4e_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4e_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4e_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4e')
# Mixed 4f
branch_0 = conv3d_bn(x, 256, 1, 1, 1, padding='same', name='Conv3d_4f_0a_1x1')
branch_1 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_4f_1a_1x1')
branch_1 = conv3d_bn(branch_1, 320, 3, 3, 3, padding='same', name='Conv3d_4f_1b_3x3')
branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_4f_2a_1x1')
branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_4f_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4f_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_4f_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4f')
# Downsampling (spatial and temporal)
x = MaxPooling3D((2, 2, 2), strides=(2, 2, 2), padding='same', name='MaxPool2d_5a_2x2')(x)
# Mixed 5b
branch_0 = conv3d_bn(x, 256, 1, 1, 1, padding='same', name='Conv3d_5b_0a_1x1')
branch_1 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_5b_1a_1x1')
branch_1 = conv3d_bn(branch_1, 320, 3, 3, 3, padding='same', name='Conv3d_5b_1b_3x3')
branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_5b_2a_1x1')
branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_5b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_5b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_5b_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_5b')
# Mixed 5c
branch_0 = conv3d_bn(x, 384, 1, 1, 1, padding='same', name='Conv3d_5c_0a_1x1')
branch_1 = conv3d_bn(x, 192, 1, 1, 1, padding='same', name='Conv3d_5c_1a_1x1')
branch_1 = conv3d_bn(branch_1, 384, 3, 3, 3, padding='same', name='Conv3d_5c_1b_3x3')
branch_2 = conv3d_bn(x, 48, 1, 1, 1, padding='same', name='Conv3d_5c_2a_1x1')
branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_5c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_5c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_5c_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_5c')
if include_top:
# Classification block
x = AveragePooling3D((2, 7, 7), strides=(1, 1, 1), padding='valid', name='global_avg_pool')(x)
x = Dropout(dropout_prob)(x)
x = conv3d_bn(x, classes, 1, 1, 1, padding='same',
use_bias=True, use_activation_fn=False, use_bn=False, name='Conv3d_6a_1x1')
num_frames_remaining = int(x.shape[1])
x = Reshape((num_frames_remaining, classes))(x)
# logits (raw scores for each class)
x = Lambda(lambda x: K.mean(x, axis=1, keepdims=False),
output_shape=lambda s: (s[0], s[2]))(x)
if not endpoint_logit:
x = Activation('softmax', name='prediction')(x)
else:
h = int(x.shape[2])
w = int(x.shape[3])
x = AveragePooling3D((2, h, w), strides=(1, 1, 1), padding='valid', name='global_avg_pool')(x)
inputs = img_input
# create model
model = Model(inputs, x, name='i3d_inception')
# load weights
if weights in WEIGHTS_NAME:
if weights == WEIGHTS_NAME[0]: # rgb_kinetics_only
if include_top:
weights_url = WEIGHTS_PATH['rgb_kinetics_only']
model_name = 'i3d_inception_rgb_kinetics_only.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['rgb_kinetics_only']
model_name = 'i3d_inception_rgb_kinetics_only_no_top.h5'
elif weights == WEIGHTS_NAME[1]: # flow_kinetics_only
if include_top:
weights_url = WEIGHTS_PATH['flow_kinetics_only']
model_name = 'i3d_inception_flow_kinetics_only.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['flow_kinetics_only']
model_name = 'i3d_inception_flow_kinetics_only_no_top.h5'
elif weights == WEIGHTS_NAME[2]: # rgb_imagenet_and_kinetics
if include_top:
weights_url = WEIGHTS_PATH['rgb_imagenet_and_kinetics']
model_name = 'i3d_inception_rgb_imagenet_and_kinetics.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['rgb_imagenet_and_kinetics']
model_name = 'i3d_inception_rgb_imagenet_and_kinetics_no_top.h5'
elif weights == WEIGHTS_NAME[3]: # flow_imagenet_and_kinetics
if include_top:
weights_url = WEIGHTS_PATH['flow_imagenet_and_kinetics']
model_name = 'i3d_inception_flow_imagenet_and_kinetics.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['flow_imagenet_and_kinetics']
model_name = 'i3d_inception_flow_imagenet_and_kinetics_no_top.h5'
downloaded_weights_path = get_file(model_name, weights_url, cache_subdir='models')
model.load_weights(downloaded_weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first' and K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your keras config '
'at ~/.keras/keras.json.')
elif weights is not None:
model.load_weights(weights)
return model
def MT_Hybrid_CAN(n_frame,
nb_filters1,
nb_filters2,
input_shape_1,
input_shape_2,
kernel_size_1=(3, 3, 3),
kernel_size_2=(3, 3),
dropout_rate1=0.25,
dropout_rate2=0.5,
pool_size_1=(2, 2, 2),
pool_size_2=(2, 2),
nb_dense=128):
diff_input = Input(shape=input_shape_1)
rawf_input = Input(shape=input_shape_2)
# Motion branch
d1 = Conv3D(nb_filters1, kernel_size_1, padding='same',
activation='tanh')(diff_input)
d2 = Conv3D(nb_filters1, kernel_size_1, activation='tanh')(d1)
# App branch
r1 = Conv2D(nb_filters1, kernel_size_2, padding='same',
activation='tanh')(rawf_input)
r2 = Conv2D(nb_filters1, kernel_size_2, activation='tanh')(r1)
# Mask from App (g1) * Motion Branch (d2)
g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2)
g1 = Attention_mask()(g1)
g1 = K.expand_dims(g1, axis=-1)
gated1 = multiply([d2, g1])
# Motion Branch
d3 = AveragePooling3D(pool_size_1)(gated1)
d4 = Dropout(dropout_rate1)(d3)
d5 = Conv3D(nb_filters2, kernel_size_1, padding='same',
activation='tanh')(d4)
d6 = Conv3D(nb_filters2, kernel_size_1, activation='tanh')(d5)
# App branch
r3 = AveragePooling2D(pool_size_2)(r2)
r4 = Dropout(dropout_rate1)(r3)
r5 = Conv2D(nb_filters2, kernel_size_2, padding='same',
activation='tanh')(r4)
r6 = Conv2D(nb_filters2, kernel_size_2, activation='tanh')(r5)
# Mask from App (g2) * Motion Branch (d6)
g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6)
g2 = Attention_mask()(g2)
g2 = K.repeat_elements(g2, d6.shape[3], axis=-1)
g2 = K.expand_dims(g2, axis=-1)
gated2 = multiply([d6, g2])
# Motion Branch
d7 = AveragePooling3D(pool_size_1)(gated2)
d8 = Dropout(dropout_rate1)(d7)
# Motion Branch
d9 = Flatten()(d8)
d10_y = Dense(nb_dense, activation='tanh')(d9)
d11_y = Dropout(dropout_rate2)(d10_y)
out_y = Dense(n_frame, name='output_1')(d11_y)
d10_r = Dense(nb_dense, activation='tanh')(d9)
d11_r = Dropout(dropout_rate2)(d10_r)
out_r = Dense(n_frame, name='output_2')(d11_r)
model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r])
return model