def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropout_rate=None, weight_decay=1e-4,
grow_nb_filters=True, return_concat_list=False):
''' Build a dense_block where the output of each conv_block is fed to subsequent ones
Args:
x: keras tensor
nb_layers: the number of layers of conv_block to append to the model.
nb_filter: number of filters
growth_rate: growth rate
bottleneck: bottleneck block
dropout_rate: dropout rate
weight_decay: weight decay factor
grow_nb_filters: flag to decide to allow number of filters to grow
return_concat_list: return the list of feature maps along with the actual output
Returns: keras tensor with nb_layers of conv_block appended
'''
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
x_list = [x]
for i in range(nb_layers):
cb = __conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay)
x_list.append(cb)
x = concatenate([x, cb], axis=concat_axis)
if grow_nb_filters:
nb_filter += growth_rate
if return_concat_list:
return x, nb_filter, x_list
else:
return x, nb_filter
def squeezenet_fire_module(input, input_channel_small=16, input_channel_large=64):
channel_axis = 3
input = Conv2D(input_channel_small, (1,1), padding="valid" )(input)
input = Activation("relu")(input)
input_branch_1 = Conv2D(input_channel_large, (1,1), padding="valid" )(input)
input_branch_1 = Activation("relu")(input_branch_1)
input_branch_2 = Conv2D(input_channel_large, (3, 3), padding="same")(input)
input_branch_2 = Activation("relu")(input_branch_2)
input = concatenate([input_branch_1, input_branch_2], axis=channel_axis)
return input
def transformer3_filter(ih, iw, nb_conv, size_conv):
"""
The cnn model for image transformation with 3 times downsampling. This model does not include fully connected
layers.
Parameters
----------
ih, iw : int
The input image dimension
nb_conv : int
Number of convolution kernels for each layer
size_conv : int
The size of convolution kernel
Returns
-------
mdl
Description.
"""
inputs = Input((ih, iw, 1))
conv1 = Conv2D(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(inputs)
conv1a = Conv2D(nb_conv, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv1)
conv2 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv1a)
conv2a = Conv2D(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv2)
conv3 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv2a)
conv3a = Conv2D(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv3)
conv4 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(conv3a)
conv4 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(conv4)
conv4 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(conv4)
#
conv5 = Conv2DTranspose(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(conv4)
conv5 = Conv2DTranspose(nb_conv * 8, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv5)
up1 = concatenate([conv5, conv3], axis=3)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up1)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv6)
up2 = concatenate([conv6, conv2], axis=3)
conv7 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up2)
conv7 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv7)
up3 = concatenate([conv7, conv1], axis=3)
conv8 = Conv2DTranspose(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(up3)
conv8 = Conv2DTranspose(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(conv8)
conv8 = Conv2DTranspose(1, (3, 3), activation='relu',
padding='same')(conv8)
mdl = Model(inputs=inputs, outputs=conv8)
mdl.compile(loss='mse', optimizer='Adam', metrics=['accuracy'])
return mdl
def _adjust_block(p, ip, filters, block_id=None):
"""Adjusts the input `previous path` to match the shape of the `input`.
Used in situations where the output number of filters needs to be changed.
Arguments:
p: Input tensor which needs to be modified
ip: Input tensor whose shape needs to be matched
filters: Number of output filters to be matched
block_id: String block_id
Returns:
Adjusted Keras tensor
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
img_dim = 2 if K.image_data_format() == 'channels_first' else -2
ip_shape = K.int_shape(ip)
if p is not None:
p_shape = K.int_shape(p)
with K.name_scope('adjust_block'):
if p is None:
p = ip
elif p_shape[img_dim] != ip_shape[img_dim]:
with K.name_scope('adjust_reduction_block_%s' % block_id):
p = Activation('relu', name='adjust_relu_1_%s' % block_id)(p)
p1 = AveragePooling2D(
(1, 1),
strides=(2, 2),
padding='valid',
name='adjust_avg_pool_1_%s' % block_id)(
p)
p1 = Conv2D(
filters // 2, (1, 1),
padding='same',
use_bias=False,
name='adjust_conv_1_%s' % block_id,
kernel_initializer='he_normal')(
p1)
p2 = ZeroPadding2D(padding=((0, 1), (0, 1)))(p)
p2 = Cropping2D(cropping=((1, 0), (1, 0)))(p2)
p2 = AveragePooling2D(
(1, 1),
strides=(2, 2),
padding='valid',
name='adjust_avg_pool_2_%s' % block_id)(
p2)
p2 = Conv2D(
filters // 2, (1, 1),
padding='same',
use_bias=False,
name='adjust_conv_2_%s' % block_id,
kernel_initializer='he_normal')(
p2)
p = concatenate([p1, p2], axis=channel_dim)
p = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='adjust_bn_%s' % block_id)(
p)
elif p_shape[channel_dim] != filters:
with K.name_scope('adjust_projection_block_%s' % block_id):
p = Activation('relu')(p)
p = Conv2D(
filters, (1, 1),
strides=(1, 1),
padding='same',
name='adjust_conv_projection_%s' % block_id,
use_bias=False,
kernel_initializer='he_normal')(
p)
p = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='adjust_bn_%s' % block_id)(
p)
return p
def transformer2(ih, iw, nb_conv, size_conv, nb_gpu=1):
"""
The simple cnn model for image transformation with 2 times of downsampling. It is a choice for fast running.
However, it will lose resolution during the transformation.
Parameters
----------
ih, iw : int
The input image dimension
nb_conv : int
Number of convolution kernels for each layer
size_conv : int
The size of convolution kernel
Returns
-------
mdl
Description.
"""
inputs = Input((ih, iw, 1))
conv1 = Conv2D(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(inputs)
conv1 = Conv2D(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(pool1)
conv2 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(pool2)
#
fc1 = Flatten()(conv3)
fc1 = Dense(iw * ih / 16)(fc1)
fc1 = Reshape((ih // 4, iw // 4, 1))(fc1)
conv4 = Conv2DTranspose(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(fc1)
up1 = concatenate([UpSampling2D(size=(2, 2))(conv4), conv2], axis=3)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up1)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv6)
up2 = concatenate([UpSampling2D(size=(2, 2))(conv6), conv1], axis=3)
conv7 = Conv2DTranspose(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(up2)
conv7 = Conv2DTranspose(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(conv7)
conv8 = Conv2DTranspose(1, (size_conv, size_conv),
activation='relu',
padding='same')(conv7)
mdl = Model(inputs=inputs, outputs=conv8)
# if nb_gpu > 1:
# mdl = multi_gpu_model(mdl, nb_gpu)
mdl.compile(loss='mse', optimizer='Adam')
return mdl
def transformer3_super(ih, iw, nb_conv, size_conv):
"""
The cnn model for image transformation with 3 times downsampling. The downsampling uses strides. The model also
merge the convolution layers from encoding and decoding parts to keep the resolution of the image. It works good
for super-resolution and image enhancement.
Parameters
----------
ih, iw : int
The input image dimension
nb_conv : int
Number of convolution kernels for each layer
size_conv : int
The size of convolution kernel
Returns
-------
mdl
Description.
"""
inputs = Input((ih, iw, 1))
conv1 = Conv2D(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(inputs)
conv1a = Conv2D(nb_conv, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv1)
conv2 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv1a)
conv2a = Conv2D(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv2)
conv3 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv2a)
conv3a = Conv2D(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv3)
conv4 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(conv3a)
conv4 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(conv4)
conv4 = Conv2D(1, (size_conv, size_conv),
activation='relu',
padding='same')(conv4)
#
fc1 = Flatten()(conv4)
fc1 = Dense(iw * ih / 128, activation='relu')(fc1)
fc1 = Dropout(0.2)(fc1)
fc1 = Dense(iw * ih / 128, activation='relu')(fc1)
fc1 = Dropout(0.25)(fc1)
fc1 = Dense(iw * ih / 64, activation='relu')(fc1)
fc1 = Dropout(0.25)(fc1)
fc1 = Reshape((ih // 8, iw // 8, 1))(fc1)
fc2 = Conv2DTranspose(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(fc1)
fc2 = Conv2DTranspose(nb_conv * 8, (size_conv, size_conv),
trides=(2, 2),
activation='relu',
padding='same')(fc2)
up1 = concatenate([fc2, conv3], axis=3)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up1)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv6)
up2 = concatenate([conv6, conv2], axis=3)
conv7 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up2)
conv7 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv7)
up3 = concatenate([conv7, conv1], axis=3)
conv8 = Conv2DTranspose(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(up3)
conv8 = Conv2DTranspose(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(conv8)
conv8 = Conv2DTranspose(1, (3, 3), activation='relu',
padding='same')(conv8)
mdl = Model(inputs=inputs, outputs=conv8)
mdl.compile(loss=psnr, optimizer='Adam', metrics=['mse'])
return mdl
def DeepLabV3Plus(input_shape, classes=66, *args, **kwargs):
print('*** Building DeepLabv3Plus Network ***')
img_height = input_shape[0]
img_width = input_shape[1]
# base_model = ResNet50(input_shape=input_shape, weights=None, include_top=False)
base_model = ResNet101V2(input_shape=input_shape, include_top=False)
base_model.summary()
image_features = base_model.output
# tf.keras.utils.plot_model(base_model, 'ResNet101V2.png')
x_a = ASPP(image_features)
x_a = Upsample(tensor=x_a, size=[img_height // 4,
img_width // 4]) # (None, 64, 64, 256)
x_b = base_model.get_layer('conv2_block2_out').output
x_b = Conv2D(filters=48,
kernel_size=1,
padding='same',
kernel_initializer='he_normal',
name='low_level_projection',
use_bias=False)(x_b)
x_b = BatchNormalization(name=f'bn_low_level_projection')(x_b)
x_b = Activation('relu', name='low_level_activation')(x_b)
x = concatenate([x_a, x_b], name='decoder_concat')
x = Conv2D(filters=256,
kernel_size=3,
padding='same',
activation='relu',
kernel_initializer='he_normal',
name='decoder_conv2d_1',
use_bias=False)(x)
x = BatchNormalization(name=f'bn_decoder_1')(x)
x = Activation('relu', name='activation_decoder_1')(x)
x = Conv2D(filters=256,
kernel_size=3,
padding='same',
activation='relu',
kernel_initializer='he_normal',
name='decoder_conv2d_2',
use_bias=False)(x)
x = BatchNormalization(name=f'bn_decoder_2')(x)
x = Activation('relu', name='activation_decoder_2')(x)
x = Upsample(x, [img_height, img_width])
x = Conv2D(classes, (1, 1), name='output_layer')(x)
# x = Activation('sigmoid')(x)
x = Activation('softmax', dtype='float32')(x)
'''
x = Activation('softmax')(x)
tf.losses.SparseCategoricalCrossentropy(from_logits=True)
Args:
from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
we assume that `y_pred` encodes a probability distribution.
'''
model = Model(inputs=base_model.input, outputs=x, name='DeepLabV3_Plus')
print(f'*** Output_Shape => {model.output_shape} ***')
return model
def construct_model(self,
tuned_params: Dict[str, Union[int, float]],
hps: HyperParameters = None) -> Model:
hpf = HyperParameterFactory(self.default_parameters_values,
tuned_params, hps)
max_pool = hpf.get_choice(MAXPOOL_NAME, [1, 2, 4, 8])
filter_0 = hpf.get_choice(FILTER0_NAME, [4, 8, 16, 32])
filter_1 = hpf.get_choice(FILTER1_NAME, [32, 48, 64])
filter_2 = hpf.get_choice(FILTER2_NAME, [64, 96, 128])
dense = hpf.get_int(DENSE_NAME, 32, 128, step=8)
hp_learning_rate = hpf.get_choice(LEARNING_RATE_NAME,
[1e-2, 1e-3, 1e-4])
input_ant_view = Input(shape=(12, 12, 7), name='input_ant_view')
avm = Conv2D(filter_0,
2,
strides=1,
activation=tf.nn.relu,
name='Conv2D_av1_32')(input_ant_view)
avm = Conv2D(filter_1,
3,
strides=1,
activation=tf.nn.relu,
name='Conv2D_av2_64')(avm)
avm = Conv2D(filter_2,
2,
strides=1,
activation=tf.nn.relu,
name='Conv2D_av3_128')(avm)
avm = Flatten(name='Flatten_av')(avm)
avm = Dense(dense, activation=tf.nn.relu, name='Dense_av')(avm)
avm = Model(inputs=input_ant_view, outputs=avm)
input_map_view = Input(shape=(43, 39, 7), name='input_map_view')
mvm = MaxPooling2D(max_pool, name='MaxPool_mv')(input_map_view)
mvm = Conv2D(filter_0,
2,
strides=1,
activation=tf.nn.relu,
name='Conv2D_mv1_32')(mvm)
mvm = Conv2D(filter_1,
3,
strides=1,
activation=tf.nn.relu,
name='Conv2D_mv2_64')(mvm)
mvm = Conv2D(filter_2,
2,
strides=1,
activation=tf.nn.relu,
name='Conv2D_mv3_128')(mvm)
mvm = Flatten(name='Flatten_mv')(mvm)
mvm = Dense(dense, activation=tf.nn.relu, name='Dense_wmv')(mvm)
mvm = Model(inputs=input_map_view, outputs=mvm)
combined = concatenate([avm.output, mvm.output])
output = Dense(5, activation=tf.nn.softmax)(combined)
model = Model(inputs=[avm.input, mvm.input], outputs=output)
loss_fn = tf.keras.losses.CategoricalCrossentropy()
opt = tf.keras.optimizers.Adam(learning_rate=hp_learning_rate)
model.compile(optimizer=opt,
loss=loss_fn,
metrics=[tf.keras.metrics.categorical_accuracy])
return model
def _reduction_a_cell(ip, p, filters, block_id=None):
"""Adds a Reduction cell for NASNet-A (Fig. 4 in the paper).
Arguments:
ip: Input tensor `x`
p: Input tensor `p`
filters: Number of output filters
block_id: String block_id
Returns:
A Keras tensor
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
with K.name_scope('reduction_A_block_%s' % block_id):
p = _adjust_block(p, ip, filters, block_id)
h = Activation('relu')(ip)
h = Conv2D(filters, (1, 1),
strides=(1, 1),
padding='same',
name='reduction_conv_1_%s' % block_id,
use_bias=False,
kernel_initializer='he_normal')(h)
h = BatchNormalization(axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='reduction_bn_1_%s' % block_id)(h)
with K.name_scope('block_1'):
x1_1 = _separable_conv_block(h,
filters, (5, 5),
strides=(2, 2),
block_id='reduction_left1_%s' %
block_id)
x1_2 = _separable_conv_block(p,
filters, (7, 7),
strides=(2, 2),
block_id='reduction_1_%s' % block_id)
x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % block_id)
with K.name_scope('block_2'):
x2_1 = MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='reduction_left2_%s' % block_id)(h)
x2_2 = _separable_conv_block(p,
filters, (7, 7),
strides=(2, 2),
block_id='reduction_right2_%s' %
block_id)
x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % block_id)
with K.name_scope('block_3'):
x3_1 = AveragePooling2D((3, 3),
strides=(2, 2),
padding='same',
name='reduction_left3_%s' % block_id)(h)
x3_2 = _separable_conv_block(p,
filters, (5, 5),
strides=(2, 2),
block_id='reduction_right3_%s' %
block_id)
x3 = add([x3_1, x3_2], name='reduction_add3_%s' % block_id)
with K.name_scope('block_4'):
x4 = AveragePooling2D((3, 3),
strides=(1, 1),
padding='same',
name='reduction_left4_%s' % block_id)(x1)
x4 = add([x2, x4])
with K.name_scope('block_5'):
x5_1 = _separable_conv_block(x1,
filters, (3, 3),
block_id='reduction_left4_%s' %
block_id)
x5_2 = MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='reduction_right5_%s' % block_id)(h)
x5 = add([x5_1, x5_2], name='reduction_add4_%s' % block_id)
x = concatenate([x2, x3, x4, x5],
axis=channel_dim,
name='reduction_concat_%s' % block_id)
return x, ip
def fusion_network(shape, batch_size):
input_img = Input(shape=(*shape, 1), name='input_img')
input_img_3 = Lambda(lambda x: tf.tile(x, [1, 1, 1, 3]),
name='input_tile')(input_img)
bbox = Input(shape=(4, MAX_INSTANCES), name='bbox')
mask = Input(shape=(*shape, MAX_INSTANCES), name='mask')
# VGG16 without top layers
VGG_model = applications.vgg16.VGG16(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
vgg_model_3_pre = Model(VGG_model.input,
VGG_model.layers[-6].output,
name='model_3')(input_img_3)
fg_model_3 = Input(shape=(*vgg_model_3_pre.get_shape().as_list()[1:],
MAX_INSTANCES),
name='fg_model_3') # <-
vgg_model_3 = WeightGenerator(64, batch_size, name='weight_generator_1')(
[fg_model_3, vgg_model_3_pre, bbox, mask]) # <-
# Global features
conv2d_6 = Conv2D(512, (3, 3),
padding='same',
strides=(2, 2),
activation='relu',
name='conv2d_6')(vgg_model_3)
batch_normalization_1 = BatchNormalization(
name='batch_normalization_1')(conv2d_6)
conv2d_7 = Conv2D(512, (3, 3),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_7')(batch_normalization_1)
batch_normalization_2 = BatchNormalization(
name='batch_normalization_2')(conv2d_7)
conv2d_8 = Conv2D(512, (3, 3),
padding='same',
strides=(2, 2),
activation='relu',
name='conv2d_8')(batch_normalization_2)
batch_normalization_3 = BatchNormalization(
name='batch_normalization_3')(conv2d_8)
conv2d_9 = Conv2D(512, (3, 3),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_9')(batch_normalization_3)
batch_normalization_4 = BatchNormalization(
name='batch_normalization_4')(conv2d_9)
# Classification
flatten_2 = Flatten(name='flatten_2')(batch_normalization_4)
dense_4 = Dense(4096, activation='relu', name='dense_4')(flatten_2)
dense_5 = Dense(4096, activation='relu', name='dense_5')(dense_4)
dense_6 = Dense(1000, activation='softmax', name='dense_6')(dense_5)
# Global feature pass back to colorization + classification
flatten_1 = Flatten(name='flatten_1')(batch_normalization_4)
dense_1 = Dense(1024, activation='relu', name='dense_1')(flatten_1)
dense_2 = Dense(512, activation='relu', name='dense_2')(dense_1)
dense_3 = Dense(256, activation='relu', name='dense_3')(dense_2)
repeat_vector_1 = RepeatVector(28 * 28, name='repeat_vector_1')(dense_3)
reshape_1 = Reshape((28, 28, 256), name='reshape_1')(repeat_vector_1)
# Mid-level features
conv2d_10 = Conv2D(512, (3, 3),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_10')(vgg_model_3)
batch_normalization_5 = BatchNormalization(
name='batch_normalization_5')(conv2d_10)
conv2d_11_pre = Conv2D(256, (3, 3),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_11')(batch_normalization_5)
fg_conv2d_11 = Input(shape=(*conv2d_11_pre.get_shape().as_list()[1:],
MAX_INSTANCES),
name='fg_conv2d_11') # <-
conv2d_11 = WeightGenerator(32, batch_size, name='weight_generator_2')(
[fg_conv2d_11, conv2d_11_pre, bbox, mask]) # <-
batch_normalization_6 = BatchNormalization(
name='batch_normalization_6')(conv2d_11)
# Fusion of (VGG16 -> Mid-level) + (VGG16 -> Global) + Colorization
concatenate_2 = concatenate([batch_normalization_6, reshape_1],
name='concatenate_2')
conv2d_12 = Conv2D(256, (1, 1),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_12')(concatenate_2)
conv2d_13_pre = Conv2D(128, (3, 3),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_13')(conv2d_12)
fg_conv2d_13 = Input(shape=(*conv2d_13_pre.get_shape().as_list()[1:],
MAX_INSTANCES),
name='fg_conv2d_13') # <-
conv2d_13 = WeightGenerator(16, batch_size, name='weight_generator_3')(
[fg_conv2d_13, conv2d_13_pre, bbox, mask]) # <-
# conv2dt_1 = Conv2DTranspose(64, (4, 4), padding='same', strides=(2, 2), name='conv2dt_1')(conv2d_13)
up_sampling2d_1 = UpSampling2D(size=(2, 2),
name='up_sampling2d_1',
interpolation='bilinear')(conv2d_13)
conv2d_14 = Conv2D(64, (3, 3),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_14')(up_sampling2d_1)
conv2d_15_pre = Conv2D(64, (3, 3),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_15')(conv2d_14)
fg_conv2d_15 = Input(shape=(*conv2d_15_pre.get_shape().as_list()[1:],
MAX_INSTANCES),
name='fg_conv2d_15') # <-
conv2d_15 = WeightGenerator(16, batch_size, name='weight_generator_4')(
[fg_conv2d_15, conv2d_15_pre, bbox, mask]) # <-
# conv2dt_2 = Conv2DTranspose(32, (4, 4), padding='same', strides=(2, 2), name='conv2dt_2')(conv2d_15)
up_sampling2d_2 = UpSampling2D(size=(2, 2),
name='up_sampling2d_2',
interpolation='bilinear')(conv2d_15)
conv2d_16 = Conv2D(32, (3, 3),
padding='same',
strides=(1, 1),
activation='relu',
name='conv2d_16')(up_sampling2d_2)
conv2d_17_pre = Conv2D(2, (3, 3),
padding='same',
strides=(1, 1),
activation='sigmoid',
name='conv2d_17')(conv2d_16)
fg_conv2d_17 = Input(shape=(*conv2d_17_pre.get_shape().as_list()[1:],
MAX_INSTANCES),
name='fg_conv2d_17') # <-
conv2d_17 = WeightGenerator(16, batch_size, name='weight_generator_5')(
[fg_conv2d_17, conv2d_17_pre, bbox, mask]) # <-
# conv2dt_3 = Conv2DTranspose(2, (4, 4), padding='same', strides=(2, 2), name='conv2dt_3')(conv2d_17)
up_sampling2d_3 = UpSampling2D(size=(2, 2),
name='up_sampling2d_3',
interpolation='bilinear')(conv2d_17)
return Model(inputs=[
input_img, fg_model_3, fg_conv2d_11, fg_conv2d_13, fg_conv2d_15,
fg_conv2d_17, bbox, mask
],
outputs=[up_sampling2d_3, dense_6])
def build_unet(input_shape, num_classes):
inputs = tf.keras.Input(shape=input_shape)
conv1 = Conv2D(64,
3,
activation='relu',
dilation_rate=2,
padding='same',
kernel_initializer='he_normal')(inputs)
conv1 = BatchNormalization()(conv1)
conv1 = Conv2D(64,
3,
activation='relu',
dilation_rate=2,
padding='same',
kernel_initializer='he_normal')(conv1)
conv1 = BatchNormalization()(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(128,
3,
activation='relu',
dilation_rate=2,
padding='same',
kernel_initializer='he_normal')(pool1)
conv2 = BatchNormalization()(conv2)
conv2 = Conv2D(128,
3,
activation='relu',
dilation_rate=2,
padding='same',
kernel_initializer='he_normal')(conv2)
conv2 = BatchNormalization()(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool2)
conv3 = BatchNormalization()(conv3)
conv3 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv3)
conv3 = BatchNormalization()(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool3)
conv4 = BatchNormalization()(conv4)
conv4 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv4)
conv4 = BatchNormalization()(conv4)
drop4 = Dropout(0.5)(conv4, training=True)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv5 = Conv2D(1024,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool4)
conv5 = BatchNormalization()(conv5)
conv5 = Conv2D(1024,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv5)
conv5 = BatchNormalization()(conv5)
drop5 = Dropout(0.5)(conv5, training=True)
up6 = Conv2D(512,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(drop5))
merge6 = concatenate([drop4, up6], axis=3)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge6)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv6)
up7 = Conv2D(256,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv6))
merge7 = concatenate([conv3, up7], axis=3)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge7)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv7)
up8 = Conv2D(128,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv7))
merge8 = concatenate([conv2, up8], axis=3)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge8)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv8)
up9 = Conv2D(64,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv8))
merge9 = concatenate([conv1, up9], axis=3)
conv9 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge9)
conv9 = Conv2D(32,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
x = Flatten()(conv9)
x = Dense(512, activation='relu', kernel_initializer='he_uniform')(x)
x = Dropout(0.3)(x)
x = Dense(num_classes, activation='softmax', name='predictions')(x)
return Model(inputs=inputs, outputs=x)
return model
def Unet(num_class, image_size):
inputs = Input(shape=[image_size, image_size, 3])
conv1 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(inputs)
#conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same')(conv1)
conv1 = Conv2D(64,
3,
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv1)
nor1 = BatchNormalization(momentum=.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones')(conv1)
act1 = Activation('relu')(nor1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(pool1)
#conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same')(conv2)
conv2 = Conv2D(128,
3,
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv2)
nor2 = BatchNormalization(momentum=.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(nor2)
conv3 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(pool2)
#conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same')(conv3)
conv3 = Conv2D(256,
3,
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv3)
nor3 = BatchNormalization(momentum=.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(nor3)
conv4 = Conv2D(512, 3, activation='relu', padding='same')(pool3)
#conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same')(conv4)
conv4 = Conv2D(512, 3, padding='same')(conv4)
nor4 = BatchNormalization(momentum=.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones')(conv4)
#drop4 = Dropout(0.5)(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(nor4)
conv5 = Conv2D(1024,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(pool4)
#conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same')(conv5)
flatten1 = Flatten()(conv5)
dense1 = Dense(1024, activation='relu')(flatten1)
dense2 = Dense(256, activation='relu')(dense1)
dense3 = Dense(3, activation='softmax')(dense2)
conv5 = Conv2D(1024,
3,
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv5)
nor5 = BatchNormalization(momentum=.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones')(conv5)
#drop5 = Dropout(0.5)(nor5)
up6 = Conv2D(512,
2,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(UpSampling2D(size=(2, 2))(nor5))
merge6 = concatenate([nor4, up6], axis=3)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(merge6)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv6)
up7 = Conv2D(256,
2,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(UpSampling2D(size=(2, 2))(conv6))
merge7 = concatenate([nor3, up7], axis=3)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(merge7)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv7)
up8 = Conv2D(128,
2,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(UpSampling2D(size=(2, 2))(conv7))
merge8 = concatenate([nor2, up8], axis=3)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(merge8)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv8)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv8)
up9 = Conv2D(64,
2,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(UpSampling2D(size=(2, 2))(conv8))
merge9 = concatenate([nor1, up9], axis=3)
conv9 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(merge9)
conv9 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='glorot_normal',
bias_initializer='zeros')(conv9)
# conv9 = Conv2D(2, 3, activation = 'relu', padding = 'same')(conv9)
conv10 = Conv2D(num_class, 1, activation='sigmoid')(conv9)
model = Model(inputs=inputs, outputs=conv10)
model.compile(
optimizer='adam',
loss=dice_coef_loss,
metrics=[dice_Score_0, dice_Score_1, dice_Score_2, dice_Score_3])
'''
model = Model(inputs = [inputs], outputs =[conv10,dense3])
model.compile(optimizer = 'adam',
loss={
'conv2d_23':dice_coef_loss,
'dense_2':'sparse_categorical_crossentropy'},
loss_weights={
'conv2d_23':0.5,
'dense_2':0.5},
metrics = {
'conv2d':[dice_Score_0,dice_Score_1,dice_Score_2,dice_Score_3],
'dense_2':['accuracy']})
'''
return model
def make_model_simple(classes, points_per_sample, channel_mode='channels_last'):
# creates the Time Distributed CNN for range Doppler heatmap ##########################
mmw_rdpl_input = (int(points_per_sample),) + rd_shape + (1,) if channel_mode == 'channels_last' else (points_per_sample, 1) + rd_shape
# range doppler shape here
mmw_rdpl_TDCNN = Sequential()
mmw_rdpl_TDCNN.add(
TimeDistributed(
Conv2D(filters=4, kernel_size=(3, 3), data_format=channel_mode,
# kernel_regularizer=l2(0.0005),
kernel_initializer='random_uniform'),
input_shape=mmw_rdpl_input))
# mmw_rdpl_TDCNN.add(TimeDistributed(LeakyReLU(alpha=0.1)))
mmw_rdpl_TDCNN.add(TimeDistributed(BatchNormalization()))
mmw_rdpl_TDCNN.add(TimeDistributed(Flatten())) # this should be where layers meets
# creates the Time Distributed CNN for range Azimuth heatmap ###########################
mmw_razi_input = (int(points_per_sample),) + ra_shape + (1,) if channel_mode == 'channels_last' else (points_per_sample, 1) + ra_shape
mmw_razi_TDCNN = Sequential()
mmw_razi_TDCNN.add(
TimeDistributed(
Conv2D(filters=4, kernel_size=(3, 3), data_format=channel_mode,
# kernel_regularizer=l2(0.0005),
kernel_initializer='random_uniform'),
input_shape=mmw_razi_input))
# mmw_rdpl_TDCNN.add(TimeDistributed(LeakyReLU(alpha=0.1)))
mmw_razi_TDCNN.add(TimeDistributed(BatchNormalization()))
mmw_razi_TDCNN.add(TimeDistributed(Flatten())) # this should be where layers meets
merged = concatenate([mmw_rdpl_TDCNN.output, mmw_razi_TDCNN.output]) # concatenate two feature extractors
regressive_tensor = LSTM(units=32, return_sequences=False, kernel_initializer='random_uniform')(merged)
regressive_tensor = Dropout(rate=0.2)(regressive_tensor)
regressive_tensor = Dense(units=32)(regressive_tensor)
regressive_tensor = Dropout(rate=0.2)(regressive_tensor)
regressive_tensor = Dense(len(classes), activation='softmax', kernel_initializer='random_uniform')(regressive_tensor)
model = Model(inputs=[mmw_rdpl_TDCNN.input, mmw_razi_TDCNN.input], outputs=regressive_tensor)
adam = tf.keras.optimizers.Adam(lr=1e-5, decay=1e-7)
model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])
return model
# def make_model_simple(classes, points_per_sample, channel_mode='channels_last'):
# # creates the Time Distributed CNN for range Doppler heatmap ##########################
# mmw_rdpl_input = (points_per_sample,) + rd_shape + (1,) if channel_mode == 'channels_last' else (points_per_sample, 1) + rd_shape
# # range doppler shape here
# mmw_rdpl_TDCNN = Sequential()
# mmw_rdpl_TDCNN.add(
# TimeDistributed(
# Conv2D(filters=4, kernel_size=(3, 3), data_format=channel_mode,
# # kernel_regularizer=l2(0.0005),
# kernel_initializer='random_uniform'),
# input_shape=mmw_rdpl_input))
# # mmw_rdpl_TDCNN.add(TimeDistributed(LeakyReLU(alpha=0.1)))
# mmw_rdpl_TDCNN.add(TimeDistributed(BatchNormalization()))
# mmw_rdpl_TDCNN.add(TimeDistributed(Flatten())) # this should be where layers meets
#
# # creates the Time Distributed CNN for range Azimuth heatmap ###########################
# mmw_razi_input = (points_per_sample,) + ra_shape + (1,) if channel_mode == 'channels_last' else (points_per_sample, 1) + ra_shape
# mmw_razi_TDCNN = Sequential()
# mmw_razi_TDCNN.add(
# TimeDistributed(
# Conv2D(filters=4, kernel_size=(3, 3), data_format=channel_mode,
# # kernel_regularizer=l2(0.0005),
# kernel_initializer='random_uniform'),
# input_shape=mmw_razi_input))
# # mmw_rdpl_TDCNN.add(TimeDistributed(LeakyReLU(alpha=0.1)))
# mmw_razi_TDCNN.add(TimeDistributed(BatchNormalization()))
# mmw_razi_TDCNN.add(TimeDistributed(Flatten())) # this should be where layers meets
#
# merged = concatenate([mmw_rdpl_TDCNN.output, mmw_razi_TDCNN.output]) # concatenate two feature extractors
# regressive_tensor = LSTM(units=32, return_sequences=False, kernel_initializer='random_uniform')(merged)
# regressive_tensor = Dropout(rate=0.2)(regressive_tensor)
#
# regressive_tensor = Dense(units=32)(regressive_tensor)
# regressive_tensor = Dropout(rate=0.2)(regressive_tensor)
# regressive_tensor = Dense(len(classes), activation='softmax', kernel_initializer='random_uniform')(regressive_tensor)
#
# model = Model(inputs=[mmw_rdpl_TDCNN.input, mmw_razi_TDCNN.input], outputs=regressive_tensor)
# adam = tf.keras.optimizers.Adam(lr=1e-5, decay=1e-7)
# model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])
# return model
def UNet64(input_shape,
n_predictions=1,
lossfunction="mean_squared_error",
simpleclassification=None,
flatten_output=True,
optimizer="adam",
activation_hidden="relu",
activation_output="relu",
metrics=None):
inputs = Input(shape=input_shape)
conv01 = Conv2D(10, kernel_size=(3, 3),
padding="same")(inputs) # 10 x 64x64
conv01 = Activation(activation_hidden)(conv01)
conv01_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv01) # 10 x 32x32
print("0)", conv01_pool.shape, "10 x 32x32")
conv02 = Conv2D(20, kernel_size=(3, 3),
padding="same")(conv01_pool) # 20 x 32x32
conv02 = Activation(activation_hidden)(conv02)
conv02_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv02) # 20 x 16x16
print("1)", conv02_pool.shape, "20 x 16x16")
conv03 = Conv2D(20, kernel_size=(3, 3),
padding="same")(conv02_pool) # 20 x 16x16
conv03 = Activation(activation_hidden)(conv03)
conv03_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv03) # 20 x 8x8
print("2)", conv03_pool.shape, "20 x 8x8")
conv04 = Conv2D(20, kernel_size=(3, 3),
padding="same")(conv03_pool) # 20 x 8x8
conv04 = Activation(activation_hidden)(conv04)
conv04_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv04) # 20 x 4x4
print("3)", conv04_pool.shape, "20 x 4x4")
### UPSAMPLING:
up04 = UpSampling2D((2, 2))(conv04_pool) # 20 x 8x8
up04 = concatenate([conv04, up04], axis=3) # 20+20 x 8x8
print("4)", up04.shape, "40 x 8x8")
up03 = UpSampling2D((2, 2))(up04) # 40 x 16x16
up03 = concatenate([conv03, up03], axis=3) # 20+40 x 16x16
print("5)", up03.shape, "60 x 16x16")
up02 = UpSampling2D((2, 2))(up03) # 60 x 32x32
up02 = concatenate([conv02, up02], axis=3) # 20+60 x 32x32
print("6)", up02.shape, "80 x 32x32")
up01 = UpSampling2D((2, 2))(up02) # 80 x 64x64
up01 = concatenate([conv01, up01], axis=3) # 10+80 x 64x64
print("7)", up01.shape, "90 x 64x64")
output = Conv2D(n_predictions, (1, 1),
activation=activation_output)(up01) # 1 x 64x64
print("8)", output.shape, "{} x 64x64".format(n_predictions))
if flatten_output:
output = Flatten()(output)
print("output flattened to {}".format(output.shape))
if simpleclassification is not None:
output = Dense(simpleclassification, activation='softmax')(output)
print(
"9)", output.shape,
"zur Klassifikation von {} Klassen (mit softmax)".format(
simpleclassification))
model = Model(inputs=inputs, outputs=output)
if metrics is not None:
model.compile(loss=lossfunction, optimizer=optimizer, metrics=metrics)
else:
model.compile(loss=lossfunction, optimizer=optimizer)
return model
def UNet64_2x2core_large(input_shape):
"""wie UNet64_out_expansed, aber downsampling bis 2x2"""
inputs = Input(shape=input_shape)
conv01 = Conv2D(10, kernel_size=(3, 3),
padding="same")(inputs) # 10 x 64x64
conv01 = Activation('relu')(conv01)
conv01_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv01) # 10 x 32x32
print("0)", conv01_pool.shape, "10 x 32x32")
conv02 = Conv2D(20, kernel_size=(3, 3),
padding="same")(conv01_pool) # 20 x 32x32
conv02 = Activation('relu')(conv02)
conv02_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv02) # 20 x 16x16
print("1)", conv02_pool.shape, "20 x 16x16")
conv03 = Conv2D(20, kernel_size=(3, 3),
padding="same")(conv02_pool) # 20 x 16x16
conv03 = Activation('relu')(conv03)
conv03_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv03) # 20 x 8x8
print("2)", conv03_pool.shape, "20 x 8x8")
conv04 = Conv2D(20, kernel_size=(3, 3),
padding="same")(conv03_pool) # 20 x 8x8
conv04 = Activation('relu')(conv04)
conv04_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv04) # 20 x 4x4
print("3)", conv04_pool.shape, "20 x 4x4")
conv05 = Conv2D(20, kernel_size=(3, 3),
padding="same")(conv04_pool) # 20 x 4x4
conv05 = Activation('relu')(conv05)
conv05_pool = MaxPooling2D((2, 2), strides=(2, 2))(conv05) # 20 x 2x2
print("4)", conv05_pool.shape, "20 x 2x2")
### UPSAMPLING:
up05 = UpSampling2D((2, 2))(conv05_pool) # 20 x 4x4
up05 = concatenate([conv05, up05], axis=3) # 40 x 4x4
print("4)", up05.shape, "40 x 4x4")
up04 = UpSampling2D((2, 2))(up05) # 10 x 8x8
up04 = concatenate([conv04, up04], axis=3) # 20+40 x 8x8
print("4)", up04.shape, "60 x 8x8")
up03 = UpSampling2D((2, 2))(up04) # 30 x 16x16
up03 = concatenate([conv03, up03], axis=3) # 20+60 x 16x16
print("5)", up03.shape, "80 x 16x16")
up02 = UpSampling2D((2, 2))(up03) # 80 x 32x32
up02 = concatenate([conv02, up02], axis=3) # 20+80 x 32x32
print("6)", up02.shape, "100 x 32x32")
up01 = UpSampling2D((2, 2))(up02) # 100 x 64x64
up01 = concatenate([conv01, up01], axis=3) # 10+100 x 64x64
print("7)", up01.shape, "110 x 64x64")
output = Conv2D(1, (3, 3), activation='relu',
padding="same")(up01) # 1 x 64x64
print("8)", output.shape, "1 x 64x64")
output = Flatten()(output)
model = Model(inputs=inputs, outputs=output)
model.compile(loss="mean_squared_error", optimizer='adam')
return model
def InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input image size for this model is 299x299.
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=299,
min_size=139,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(
branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, strides=(2, 2), padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(
branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate(
[branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate(
[branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='inception_v3')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
file_hash='9a0d58056eeedaa3f26cb7ebd46da564')
else:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='bcbd6486424b2319ff4ef7d526e38f63')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def bulid_model(self):
inputs = Input(shape=self.input_shape)
net = inputs
# block1
net = Conv2D(filters=32,
kernel_size=3,
strides=2,
activation='relu',
padding='same',
name='bock1_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net = BatchNormalization()(net)
net = Conv2D(filters=32,
kernel_size=3,
activation='relu',
padding='same',
name='block1_conv2',
kernel_regularizer=regularizers.l2(
self.weight_decay)(net))
net = BatchNormalization()(net)
net = Conv2D(filters=64,
kernel_size=3,
activation='relu',
padding='same',
name='block1_conv3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net = BatchNormalization()(net)
net = MaxPooling2D(pool_size=3,
strides=2,
padding='same',
name='block1_pool')(net)
#block2
net = Conv2D(filters=80,
kernel_size=1,
activation='relu',
padding='same',
name='block2_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net = BatchNormalization()(net)
net = Conv2D(filters=192,
kernel_size=3,
activation='relu',
padding='same',
name='block2_conv2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net = BatchNormalization()(net)
net = MaxPooling2D(pool_size=3,
strides=2,
padding='same',
name='block2_pool')(net)
net = self.block1_module1(net, 'block1_module1')
net = self.block1_module2(net, "block1_module2")
net = self.block1_module2(net, 'block1_module2_1')
net = self.block2_module1(net)
# 1x1
net_1x1 = Conv2D(filters=128,
kernel_size=1,
padding='same',
activation='relu',
name='block2_module2_1x1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_1x1 = BatchNormalization()(net_1x1)
# 1x7
net_1x7 = Conv2D(filters=128,
kernel_size=(1, 1),
padding='same',
activation='relu',
name='block2_module2_1x7_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_1x7 = BatchNormalization()(net_1x7)
net_1x7 = Conv2D(filters=128,
kernel_size=(1, 7),
padding='same',
activation='relu',
name='block2_module2_1x7_conv2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_1x7)
net_1x7 = BatchNormalization()(net_1x7)
net_1x7 = Conv2D(filters=192,
kernel_size=(7, 1),
padding='same',
activation='relu',
name='block2_module2_1x7_conv3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_1x7)
net_1x7 = BatchNormalization()(net_1x7)
net_7x1 = Conv2D(filters=128,
kernel_size=(1, 1),
padding='same',
activation='relu',
name='block2_module2_7x1_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_7x1 = Conv2D(filters=128,
kernel_size=(7, 1),
padding='same',
activation='relu',
name='block2_module2_7x1_conv2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_7x1 = Conv2D(filters=128,
kernel_size=(7, 1),
padding='same',
activation='relu',
name='block2_module2_7x1_conv3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_7x1 = Conv2D(filters=192,
kernel_size=(1, 7),
padding='same',
activation='relu',
name='block2_module2_7x1_conv4',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_avg = AvgPool2D(pool_size=3,
strides=1,
padding='same',
name='block2_module2_1x1_avg')(net)
net_avg = Conv2D(filters=192,
kernel_size=1,
padding='same',
activation='relu',
name='block2_module2_1x1_avg_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_avg)
net = concatenate([net_1x1, net_1x7, net_7x1, net_avg], axis=-1)
net = self.block2_modul3_4(net, 'block2_module3')
net = self.block2_modul3_4(net, 'block2_module4')
# 1x1
net_1x1 = Conv2D(filters=192,
kernel_size=1,
padding='same',
activation='relu',
name='block2_module5_1x1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_1x1 = BatchNormalization()(net_1x1)
# 1x7
net_1x7 = Conv2D(filters=192,
kernel_size=(1, 1),
padding='same',
activation='relu',
name='block2_module5_1x7_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_1x7 = BatchNormalization()(net_1x7)
net_1x7 = Conv2D(filters=192,
kernel_size=(1, 7),
padding='same',
activation='relu',
name='block2_module5_1x7_conv2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_1x7)
net_1x7 = BatchNormalization()(net_1x7)
net_1x7 = Conv2D(filters=192,
kernel_size=(7, 1),
padding='same',
activation='relu',
name='block2_module5_1x7_conv3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_1x7)
net_1x7 = BatchNormalization()(net_1x7)
net_7x1 = Conv2D(filters=192,
kernel_size=(1, 1),
padding='same',
activation='relu',
name='block2_module5_7x1_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_7x1 = Conv2D(filters=192,
kernel_size=(7, 1),
padding='same',
activation='relu',
name='block2_module5_7x1_conv2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_7x1 = Conv2D(filters=192,
kernel_size=(7, 1),
padding='same',
activation='relu',
name='block2_module5_7x1_conv3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_7x1 = Conv2D(filters=192,
kernel_size=(1, 7),
padding='same',
activation='relu',
name='block2_module5_7x1_conv4',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_avg = AvgPool2D(pool_size=3,
strides=1,
padding='same',
name='block2_module5_1x1_avg')(net)
net_avg = Conv2D(filters=192,
kernel_size=1,
padding='same',
activation='relu',
name='block2_module5_1x1_avg_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_avg)
net = concatenate([net_1x1, net_1x7, net_7x1, net_avg], axis=-1)
def Inception_Inflated3d(include_top=True,
weights=None,
input_tensor=None,
input_shape=None,
dropout_prob=0.0,
endpoint_logit=True,
classes=400):
"""Instantiates the Inflated 3D Inception v1 architecture.
Optionally loads weights pre-trained
on Kinetics. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input frame(image) size for this model is 224x224.
# Arguments
include_top: whether to include the the classification
layer at the top of the network.
weights: one of `None` (random initialization)
or 'kinetics_only' (pre-training on Kinetics dataset only).
or 'imagenet_and_kinetics' (pre-training on ImageNet and Kinetics datasets).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(NUM_FRAMES, 224, 224, 3)` (with `channels_last` data format)
or `(NUM_FRAMES, 3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 inputs channels.
NUM_FRAMES should be no smaller than 8. The authors used 64
frames per example for training and testing on kinetics dataset
Also, Width and height should be no smaller than 32.
E.g. `(64, 150, 150, 3)` would be one valid value.
dropout_prob: optional, dropout probability applied in dropout layer
after global average pooling layer.
0.0 means no dropout is applied, 1.0 means dropout is applied to all features.
Note: Since Dropout is applied just before the classification
layer, it is only useful when `include_top` is set to True.
endpoint_logit: (boolean) optional. If True, the model's forward pass
will end at producing logits. Otherwise, softmax is applied after producing
the logits to produce the class probabilities prediction. Setting this parameter
to True is particularly useful when you want to combine results of rgb model
and optical flow model.
- `True` end model forward pass at logit output
- `False` go further after logit to produce softmax predictions
Note: This parameter is only useful when `include_top` is set to True.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in WEIGHTS_NAME or weights is None or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or %s' %
str(WEIGHTS_NAME) + ' '
'or a valid path to a file containing `weights` values')
if weights in WEIGHTS_NAME and include_top and classes != 400:
raise ValueError('If using `weights` as one of these %s, with `include_top`'
' as true, `classes` should be 400' % str(WEIGHTS_NAME))
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_frame_size=224,
min_frame_size=32,
default_num_frames=64,
min_num_frames=8,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 4
# Downsampling via convolution (spatial and temporal)
x = conv3d_bn(img_input, 64, 7, 7, 7, strides=(2, 2, 2), padding='same', name='Conv3d_1a_7x7')
# Downsampling (spatial only)
x = MaxPooling3D((1, 3, 3), strides=(1, 2, 2), padding='same', name='MaxPool2d_2a_3x3')(x)
x = conv3d_bn(x, 64, 1, 1, 1, strides=(1, 1, 1), padding='same', name='Conv3d_2b_1x1')
x = conv3d_bn(x, 192, 3, 3, 3, strides=(1, 1, 1), padding='same', name='Conv3d_2c_3x3')
# Downsampling (spatial only)
x = MaxPooling3D((1, 3, 3), strides=(1, 2, 2), padding='same', name='MaxPool2d_3a_3x3')(x)
# Mixed 3b
branch_0 = conv3d_bn(x, 64, 1, 1, 1, padding='same', name='Conv3d_3b_0a_1x1')
branch_1 = conv3d_bn(x, 96, 1, 1, 1, padding='same', name='Conv3d_3b_1a_1x1')
branch_1 = conv3d_bn(branch_1, 128, 3, 3, 3, padding='same', name='Conv3d_3b_1b_3x3')
branch_2 = conv3d_bn(x, 16, 1, 1, 1, padding='same', name='Conv3d_3b_2a_1x1')
branch_2 = conv3d_bn(branch_2, 32, 3, 3, 3, padding='same', name='Conv3d_3b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_3b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 32, 1, 1, 1, padding='same', name='Conv3d_3b_3b_1x1')
######edit##############
q = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_3b_3a_3x3')(x)
q = conv3d_bn(q, 32, 1, 1, 1, padding='same', name='Conv3d_3b_3b_1x1')
########################
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_3b')
# Mixed 3c
branch_0 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_3c_0a_1x1')
branch_1 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_3c_1a_1x1')
branch_1 = conv3d_bn(branch_1, 192, 3, 3, 3, padding='same', name='Conv3d_3c_1b_3x3')
branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_3c_2a_1x1')
branch_2 = conv3d_bn(branch_2, 96, 3, 3, 3, padding='same', name='Conv3d_3c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_3c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_3c_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_3c')
# Downsampling (spatial and temporal)
x = MaxPooling3D((3, 3, 3), strides=(2, 2, 2), padding='same', name='MaxPool2d_4a_3x3')(x)
# Mixed 4b
branch_0 = conv3d_bn(x, 192, 1, 1, 1, padding='same', name='Conv3d_4b_0a_1x1')
branch_1 = conv3d_bn(x, 96, 1, 1, 1, padding='same', name='Conv3d_4b_1a_1x1')
branch_1 = conv3d_bn(branch_1, 208, 3, 3, 3, padding='same', name='Conv3d_4b_1b_3x3')
branch_2 = conv3d_bn(x, 16, 1, 1, 1, padding='same', name='Conv3d_4b_2a_1x1')
branch_2 = conv3d_bn(branch_2, 48, 3, 3, 3, padding='same', name='Conv3d_4b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4b_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4b')
# Mixed 4c
branch_0 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_4c_0a_1x1')
branch_1 = conv3d_bn(x, 112, 1, 1, 1, padding='same', name='Conv3d_4c_1a_1x1')
branch_1 = conv3d_bn(branch_1, 224, 3, 3, 3, padding='same', name='Conv3d_4c_1b_3x3')
branch_2 = conv3d_bn(x, 24, 1, 1, 1, padding='same', name='Conv3d_4c_2a_1x1')
branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4c_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4c')
# Mixed 4d
branch_0 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_4d_0a_1x1')
branch_1 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_4d_1a_1x1')
branch_1 = conv3d_bn(branch_1, 256, 3, 3, 3, padding='same', name='Conv3d_4d_1b_3x3')
branch_2 = conv3d_bn(x, 24, 1, 1, 1, padding='same', name='Conv3d_4d_2a_1x1')
branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4d_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4d_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4d_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4d')
# Mixed 4e
branch_0 = conv3d_bn(x, 112, 1, 1, 1, padding='same', name='Conv3d_4e_0a_1x1')
branch_1 = conv3d_bn(x, 144, 1, 1, 1, padding='same', name='Conv3d_4e_1a_1x1')
branch_1 = conv3d_bn(branch_1, 288, 3, 3, 3, padding='same', name='Conv3d_4e_1b_3x3')
branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_4e_2a_1x1')
branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4e_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4e_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4e_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4e')
# Mixed 4f
branch_0 = conv3d_bn(x, 256, 1, 1, 1, padding='same', name='Conv3d_4f_0a_1x1')
branch_1 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_4f_1a_1x1')
branch_1 = conv3d_bn(branch_1, 320, 3, 3, 3, padding='same', name='Conv3d_4f_1b_3x3')
branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_4f_2a_1x1')
branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_4f_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4f_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_4f_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4f')
# Downsampling (spatial and temporal)
x = MaxPooling3D((2, 2, 2), strides=(2, 2, 2), padding='same', name='MaxPool2d_5a_2x2')(x)
# Mixed 5b
branch_0 = conv3d_bn(x, 256, 1, 1, 1, padding='same', name='Conv3d_5b_0a_1x1')
branch_1 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_5b_1a_1x1')
branch_1 = conv3d_bn(branch_1, 320, 3, 3, 3, padding='same', name='Conv3d_5b_1b_3x3')
branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_5b_2a_1x1')
branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_5b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_5b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_5b_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_5b')
# Mixed 5c
branch_0 = conv3d_bn(x, 384, 1, 1, 1, padding='same', name='Conv3d_5c_0a_1x1')
branch_1 = conv3d_bn(x, 192, 1, 1, 1, padding='same', name='Conv3d_5c_1a_1x1')
branch_1 = conv3d_bn(branch_1, 384, 3, 3, 3, padding='same', name='Conv3d_5c_1b_3x3')
branch_2 = conv3d_bn(x, 48, 1, 1, 1, padding='same', name='Conv3d_5c_2a_1x1')
branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_5c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_5c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_5c_3b_1x1')
x = layers.concatenate(
[branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_5c')
if include_top:
# Classification block
x = AveragePooling3D((2, 7, 7), strides=(1, 1, 1), padding='valid', name='global_avg_pool')(x)
x = Dropout(dropout_prob)(x)
x = conv3d_bn(x, classes, 1, 1, 1, padding='same',
use_bias=True, use_activation_fn=False, use_bn=False, name='Conv3d_6a_1x1')
num_frames_remaining = int(x.shape[1])
x = Reshape((num_frames_remaining, classes))(x)
# logits (raw scores for each class)
x = Lambda(lambda x: K.mean(x, axis=1, keepdims=False),
output_shape=lambda s: (s[0], s[2]))(x)
if not endpoint_logit:
x = Activation('softmax', name='prediction')(x)
else:
h = int(x.shape[2])
w = int(x.shape[3])
x = AveragePooling3D((2, h, w), strides=(1, 1, 1), padding='valid', name='global_avg_pool')(x)
inputs = img_input
# create model
model = Model(inputs, x, name='i3d_inception')
# load weights
if weights in WEIGHTS_NAME:
if weights == WEIGHTS_NAME[0]: # rgb_kinetics_only
if include_top:
weights_url = WEIGHTS_PATH['rgb_kinetics_only']
model_name = 'i3d_inception_rgb_kinetics_only.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['rgb_kinetics_only']
model_name = 'i3d_inception_rgb_kinetics_only_no_top.h5'
elif weights == WEIGHTS_NAME[1]: # flow_kinetics_only
if include_top:
weights_url = WEIGHTS_PATH['flow_kinetics_only']
model_name = 'i3d_inception_flow_kinetics_only.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['flow_kinetics_only']
model_name = 'i3d_inception_flow_kinetics_only_no_top.h5'
elif weights == WEIGHTS_NAME[2]: # rgb_imagenet_and_kinetics
if include_top:
weights_url = WEIGHTS_PATH['rgb_imagenet_and_kinetics']
model_name = 'i3d_inception_rgb_imagenet_and_kinetics.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['rgb_imagenet_and_kinetics']
model_name = 'i3d_inception_rgb_imagenet_and_kinetics_no_top.h5'
elif weights == WEIGHTS_NAME[3]: # flow_imagenet_and_kinetics
if include_top:
weights_url = WEIGHTS_PATH['flow_imagenet_and_kinetics']
model_name = 'i3d_inception_flow_imagenet_and_kinetics.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['flow_imagenet_and_kinetics']
model_name = 'i3d_inception_flow_imagenet_and_kinetics_no_top.h5'
downloaded_weights_path = get_file(model_name, weights_url, cache_subdir='models')
model.load_weights(downloaded_weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first' and K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your keras config '
'at ~/.keras/keras.json.')
elif weights is not None:
model.load_weights(weights)
return model
def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5, growth_rate=12,
reduction=0.0, dropout_rate=None, weight_decay=1e-4,
nb_layers_per_block=4, nb_upsampling_conv=128, upsampling_type='upsampling',
init_conv_filters=48, input_shape=None, activation='deconv'):
''' Build the DenseNet model
Args:
nb_classes: number of classes
img_input: tuple of shape (channels, rows, columns) or (rows, columns, channels)
include_top: flag to include the final Dense layer
nb_dense_block: number of dense blocks to add to end (generally = 3)
growth_rate: number of filters to add per dense block
reduction: reduction factor of transition blocks. Note : reduction value is inverted to compute compression
dropout_rate: dropout rate
weight_decay: weight decay
nb_layers_per_block: number of layers in each dense block.
Can be a positive integer or a list.
If positive integer, a set number of layers per dense block.
If list, nb_layer is used as provided. Note that list size must
be (nb_dense_block + 1)
nb_upsampling_conv: number of convolutional layers in upsampling via subpixel convolution
upsampling_type: Can be one of 'upsampling', 'deconv' and 'subpixel'. Defines
type of upsampling algorithm used.
input_shape: Only used for shape inference in fully convolutional networks.
activation: Type of activation at the top layer. Can be one of 'softmax' or 'sigmoid'.
Note that if sigmoid is used, classes must be 1.
Returns: keras tensor with nb_layers of conv_block appended
'''
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
if concat_axis == 1: # channels_first dim ordering
_, rows, cols = input_shape
else:
rows, cols, _ = input_shape
if reduction != 0.0:
assert reduction <= 1.0 and reduction > 0.0, 'reduction value must lie between 0.0 and 1.0'
# check if upsampling_conv has minimum number of filters
# minimum is set to 12, as at least 3 color channels are needed for correct upsampling
assert nb_upsampling_conv > 12 and nb_upsampling_conv % 4 == 0, 'Parameter `upsampling_conv` number of channels must ' \
'be a positive number divisible by 4 and greater ' \
'than 12'
# layers in each dense block
if type(nb_layers_per_block) is list or type(nb_layers_per_block) is tuple:
nb_layers = list(nb_layers_per_block) # Convert tuple to list
assert len(nb_layers) == (nb_dense_block + 1), 'If list, nb_layer is used as provided. ' \
'Note that list size must be (nb_dense_block + 1)'
bottleneck_nb_layers = nb_layers[-1]
rev_layers = nb_layers[::-1]
nb_layers.extend(rev_layers[1:])
else:
bottleneck_nb_layers = nb_layers_per_block
nb_layers = [nb_layers_per_block] * (2 * nb_dense_block + 1)
# compute compression factor
compression = 1.0 - reduction
# Initial convolution
x = Conv2D(init_conv_filters, (7, 7), kernel_initializer='he_normal', padding='same', name='initial_conv2D',
use_bias=False, kernel_regularizer=l2(weight_decay))(img_input)
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = Activation('relu')(x)
nb_filter = init_conv_filters
skip_list = []
# Add dense blocks and transition down block
for block_idx in range(nb_dense_block):
x, nb_filter = __dense_block(x, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Skip connection
skip_list.append(x)
# add transition_block
x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay)
nb_filter = int(nb_filter * compression) # this is calculated inside transition_down_block
# The last dense_block does not have a transition_down_block
# return the concatenated feature maps without the concatenation of the input
_, nb_filter, concat_list = __dense_block(x, bottleneck_nb_layers, nb_filter, growth_rate,
dropout_rate=dropout_rate, weight_decay=weight_decay,
return_concat_list=True)
skip_list = skip_list[::-1] # reverse the skip list
# Add dense blocks and transition up block
for block_idx in range(nb_dense_block):
n_filters_keep = growth_rate * nb_layers[nb_dense_block + block_idx]
# upsampling block must upsample only the feature maps (concat_list[1:]),
# not the concatenation of the input with the feature maps (concat_list[0].
l = concatenate(concat_list[1:], axis=concat_axis)
t = __transition_up_block(l, nb_filters=n_filters_keep, type=upsampling_type, weight_decay=weight_decay)
# concatenate the skip connection with the transition block
x = concatenate([t, skip_list[block_idx]], axis=concat_axis)
# Dont allow the feature map size to grow in upsampling dense blocks
x_up, nb_filter, concat_list = __dense_block(x, nb_layers[nb_dense_block + block_idx + 1], nb_filter=growth_rate,
growth_rate=growth_rate, dropout_rate=dropout_rate,
weight_decay=weight_decay, return_concat_list=True,
grow_nb_filters=False)
if include_top:
x = Conv2D(nb_classes, (1, 1), activation='linear', padding='same', use_bias=False)(x_up)
if K.image_data_format() == 'channels_first':
channel, row, col = input_shape
else:
row, col, channel = input_shape
x = Reshape((row * col, nb_classes))(x)
x = Activation(activation)(x)
x = Reshape((row, col, nb_classes))(x)
else:
x = x_up
return x
def block2_modul3_4(self, net, name):
# 1x1
net_1x1 = Conv2D(filters=192,
kernel_size=1,
padding='same',
activation='relu',
name=name + "_1x1_conv1",
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_1x1 = BatchNormalization()(net_1x1)
# 1x7
net_1x7 = Conv2D(filters=160,
kernel_size=(1, 1),
padding='same',
activation='relu',
name=name + '_1x7_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_1x7 = BatchNormalization()(net_1x7)
net_1x7 = Conv2D(filters=160,
kernel_size=(1, 7),
padding='same',
activation='relu',
name=name + '_1x7_conv2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_1x7)
net_1x7 = BatchNormalization()(net_1x7)
net_1x7 = Conv2D(filters=192,
kernel_size=(7, 1),
padding='same',
activation='relu',
name=name + '_1x7_conv3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_1x7)
net_1x7 = BatchNormalization()(net_1x7)
net_7x1 = Conv2D(filters=160,
kernel_size=(1, 1),
padding='same',
activation='relu',
name=name + '_7x1_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_7x1 = Conv2D(filters=160,
kernel_size=(7, 1),
padding='same',
activation='relu',
name=name + '_7x1_conv2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_7x1 = Conv2D(filters=160,
kernel_size=(7, 1),
padding='same',
activation='relu',
name=name + '_7x1_conv3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_7x1 = Conv2D(filters=192,
kernel_size=(1, 7),
padding='same',
activation='relu',
name=name + '_7x1_conv4',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_7x1)
net_avg = AvgPool2D(pool_size=3,
strides=1,
padding='same',
name=name + '_1x1_avg')(net)
net_avg = Conv2D(filters=192,
kernel_size=1,
padding='same',
activation='relu',
name=name + '_1x1_avg_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_avg)
net = concatenate([net_1x1, net_1x7, net_7x1, net_avg], axis=-1)
return net
def create_model(Input):
x = ZeroPadding2D(padding=(3, 3), input_shape=(96, 96, 3))(Input)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn1')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = MaxPooling2D(pool_size=3, strides=2)(x)
x = Lambda(LRN2D, name='lrn_1')(x)
x = Conv2D(64, (1, 1), name='conv2')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn2')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = Conv2D(192, (3, 3), name='conv3')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn3')(x)
x = Activation('relu')(x)
x = Lambda(LRN2D, name='lrn_2')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = MaxPooling2D(pool_size=3, strides=2)(x)
# Inception3a
inception_3a_3x3 = Conv2D(96, (1, 1), name='inception_3a_3x3_conv1')(x)
inception_3a_3x3 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_3x3_bn1')(inception_3a_3x3)
inception_3a_3x3 = Activation('relu')(inception_3a_3x3)
inception_3a_3x3 = ZeroPadding2D(padding=(1, 1))(inception_3a_3x3)
inception_3a_3x3 = Conv2D(128, (3, 3),
name='inception_3a_3x3_conv2')(inception_3a_3x3)
inception_3a_3x3 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_3x3_bn2')(inception_3a_3x3)
inception_3a_3x3 = Activation('relu')(inception_3a_3x3)
inception_3a_5x5 = Conv2D(16, (1, 1), name='inception_3a_5x5_conv1')(x)
inception_3a_5x5 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_5x5_bn1')(inception_3a_5x5)
inception_3a_5x5 = Activation('relu')(inception_3a_5x5)
inception_3a_5x5 = ZeroPadding2D(padding=(2, 2))(inception_3a_5x5)
inception_3a_5x5 = Conv2D(32, (5, 5),
name='inception_3a_5x5_conv2')(inception_3a_5x5)
inception_3a_5x5 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_5x5_bn2')(inception_3a_5x5)
inception_3a_5x5 = Activation('relu')(inception_3a_5x5)
inception_3a_pool = MaxPooling2D(pool_size=3, strides=2)(x)
inception_3a_pool = Conv2D(
32, (1, 1), name='inception_3a_pool_conv')(inception_3a_pool)
inception_3a_pool = BatchNormalization(
axis=3, epsilon=0.00001,
name='inception_3a_pool_bn')(inception_3a_pool)
inception_3a_pool = Activation('relu')(inception_3a_pool)
inception_3a_pool = ZeroPadding2D(padding=((3, 4), (3,
4)))(inception_3a_pool)
inception_3a_1x1 = Conv2D(64, (1, 1), name='inception_3a_1x1_conv')(x)
inception_3a_1x1 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_1x1_bn')(inception_3a_1x1)
inception_3a_1x1 = Activation('relu')(inception_3a_1x1)
inception_3a = concatenate([
inception_3a_3x3, inception_3a_5x5, inception_3a_pool, inception_3a_1x1
],
axis=3)
# Inception3b
inception_3b_3x3 = Conv2D(96, (1, 1),
name='inception_3b_3x3_conv1')(inception_3a)
inception_3b_3x3 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_3x3_bn1')(inception_3b_3x3)
inception_3b_3x3 = Activation('relu')(inception_3b_3x3)
inception_3b_3x3 = ZeroPadding2D(padding=(1, 1))(inception_3b_3x3)
inception_3b_3x3 = Conv2D(128, (3, 3),
name='inception_3b_3x3_conv2')(inception_3b_3x3)
inception_3b_3x3 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_3x3_bn2')(inception_3b_3x3)
inception_3b_3x3 = Activation('relu')(inception_3b_3x3)
inception_3b_5x5 = Conv2D(32, (1, 1),
name='inception_3b_5x5_conv1')(inception_3a)
inception_3b_5x5 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_5x5_bn1')(inception_3b_5x5)
inception_3b_5x5 = Activation('relu')(inception_3b_5x5)
inception_3b_5x5 = ZeroPadding2D(padding=(2, 2))(inception_3b_5x5)
inception_3b_5x5 = Conv2D(64, (5, 5),
name='inception_3b_5x5_conv2')(inception_3b_5x5)
inception_3b_5x5 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_5x5_bn2')(inception_3b_5x5)
inception_3b_5x5 = Activation('relu')(inception_3b_5x5)
inception_3b_pool = Lambda(lambda x: x**2, name='power2_3b')(inception_3a)
inception_3b_pool = AveragePooling2D(pool_size=(3, 3),
strides=(3, 3))(inception_3b_pool)
inception_3b_pool = Lambda(lambda x: x * 9,
name='mult9_3b')(inception_3b_pool)
inception_3b_pool = Lambda(lambda x: K.sqrt(x),
name='sqrt_3b')(inception_3b_pool)
inception_3b_pool = Conv2D(
64, (1, 1), name='inception_3b_pool_conv')(inception_3b_pool)
inception_3b_pool = BatchNormalization(
axis=3, epsilon=0.00001,
name='inception_3b_pool_bn')(inception_3b_pool)
inception_3b_pool = Activation('relu')(inception_3b_pool)
inception_3b_pool = ZeroPadding2D(padding=(4, 4))(inception_3b_pool)
inception_3b_1x1 = Conv2D(64, (1, 1),
name='inception_3b_1x1_conv')(inception_3a)
inception_3b_1x1 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_1x1_bn')(inception_3b_1x1)
inception_3b_1x1 = Activation('relu')(inception_3b_1x1)
inception_3b = concatenate([
inception_3b_3x3, inception_3b_5x5, inception_3b_pool, inception_3b_1x1
],
axis=3)
# Inception3c
inception_3c_3x3 = utils.conv2d_bn(inception_3b,
layer='inception_3c_3x3',
cv1_out=128,
cv1_filter=(1, 1),
cv2_out=256,
cv2_filter=(3, 3),
cv2_strides=(2, 2),
padding=(1, 1))
inception_3c_5x5 = utils.conv2d_bn(inception_3b,
layer='inception_3c_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(2, 2),
padding=(2, 2))
inception_3c_pool = MaxPooling2D(pool_size=3, strides=2)(inception_3b)
inception_3c_pool = ZeroPadding2D(padding=((0, 1), (0,
1)))(inception_3c_pool)
inception_3c = concatenate(
[inception_3c_3x3, inception_3c_5x5, inception_3c_pool], axis=3)
# inception 4a
inception_4a_3x3 = utils.conv2d_bn(inception_3c,
layer='inception_4a_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=192,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_4a_5x5 = utils.conv2d_bn(inception_3c,
layer='inception_4a_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(1, 1),
padding=(2, 2))
inception_4a_pool = Lambda(lambda x: x**2, name='power2_4a')(inception_3c)
inception_4a_pool = AveragePooling2D(pool_size=(3, 3),
strides=(3, 3))(inception_4a_pool)
inception_4a_pool = Lambda(lambda x: x * 9,
name='mult9_4a')(inception_4a_pool)
inception_4a_pool = Lambda(lambda x: K.sqrt(x),
name='sqrt_4a')(inception_4a_pool)
inception_4a_pool = utils.conv2d_bn(inception_4a_pool,
layer='inception_4a_pool',
cv1_out=128,
cv1_filter=(1, 1),
padding=(2, 2))
inception_4a_1x1 = utils.conv2d_bn(inception_3c,
layer='inception_4a_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_4a = concatenate([
inception_4a_3x3, inception_4a_5x5, inception_4a_pool, inception_4a_1x1
],
axis=3)
# inception4e
inception_4e_3x3 = utils.conv2d_bn(inception_4a,
layer='inception_4e_3x3',
cv1_out=160,
cv1_filter=(1, 1),
cv2_out=256,
cv2_filter=(3, 3),
cv2_strides=(2, 2),
padding=(1, 1))
inception_4e_5x5 = utils.conv2d_bn(inception_4a,
layer='inception_4e_5x5',
cv1_out=64,
cv1_filter=(1, 1),
cv2_out=128,
cv2_filter=(5, 5),
cv2_strides=(2, 2),
padding=(2, 2))
inception_4e_pool = MaxPooling2D(pool_size=3, strides=2)(inception_4a)
inception_4e_pool = ZeroPadding2D(padding=((0, 1), (0,
1)))(inception_4e_pool)
inception_4e = concatenate(
[inception_4e_3x3, inception_4e_5x5, inception_4e_pool], axis=3)
# inception5a
inception_5a_3x3 = utils.conv2d_bn(inception_4e,
layer='inception_5a_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=384,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_5a_pool = Lambda(lambda x: x**2, name='power2_5a')(inception_4e)
inception_5a_pool = AveragePooling2D(pool_size=(3, 3),
strides=(3, 3))(inception_5a_pool)
inception_5a_pool = Lambda(lambda x: x * 9,
name='mult9_5a')(inception_5a_pool)
inception_5a_pool = Lambda(lambda x: K.sqrt(x),
name='sqrt_5a')(inception_5a_pool)
inception_5a_pool = utils.conv2d_bn(inception_5a_pool,
layer='inception_5a_pool',
cv1_out=96,
cv1_filter=(1, 1),
padding=(1, 1))
inception_5a_1x1 = utils.conv2d_bn(inception_4e,
layer='inception_5a_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_5a = concatenate(
[inception_5a_3x3, inception_5a_pool, inception_5a_1x1], axis=3)
# inception_5b
inception_5b_3x3 = utils.conv2d_bn(inception_5a,
layer='inception_5b_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=384,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_5b_pool = MaxPooling2D(pool_size=3, strides=2)(inception_5a)
inception_5b_pool = utils.conv2d_bn(inception_5b_pool,
layer='inception_5b_pool',
cv1_out=96,
cv1_filter=(1, 1))
inception_5b_pool = ZeroPadding2D(padding=(1, 1))(inception_5b_pool)
inception_5b_1x1 = utils.conv2d_bn(inception_5a,
layer='inception_5b_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_5b = concatenate(
[inception_5b_3x3, inception_5b_pool, inception_5b_1x1], axis=3)
av_pool = AveragePooling2D(pool_size=(3, 3), strides=(1, 1))(inception_5b)
reshape_layer = Flatten()(av_pool)
dense_layer = Dense(128, name='dense_layer')(reshape_layer)
norm_layer = Lambda(lambda x: K.l2_normalize(x, axis=1),
name='norm_layer')(dense_layer)
# Final Model
model = Model(inputs=[Input], outputs=norm_layer)
return model
def block1_module2(self, net, name):
# 1x1
net_1x1 = Conv2D(filters=64,
kernel_size=1,
padding='same',
activation='relu',
name=name + '_1x1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_1x1 = BatchNormalization()(net_1x1)
# 5x5
net_5x5 = Conv2D(filters=48,
kernel_size=1,
padding='same',
activation='relu',
name=name + '_5x5',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_5x5 = BatchNormalization(net_5x5)
net_5x5 = Conv2D(filters=64,
kernel_size=5,
padding='same',
activation='relu',
name=name + '_5x5_2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_5x5)
net_5x5 = BatchNormalization()(net_5x5)
# 3x3
net_3x3 = Conv2D(filters=64,
kernel_size=1,
padding='same',
activation='relu',
name=name + '_3x3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net)
net_3x3 = BatchNormalization()(net_3x3)
net_3x3 = Conv2D(filters=96,
kernel_size=3,
activation='relu',
padding='same',
name=name + '_3x3_2',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_3x3)
net_3x3 = BatchNormalization()(net_3x3)
net_3x3 = Conv2D(filters=96,
kernel_size=3,
activation='relu',
padding='same',
name=name + '_3x3_3',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_3x3)
net_3x3 = BatchNormalization()(net_3x3)
# 1x1xavg
net_1x1_avg = AvgPool2D(pool_size=3,
strides=1,
padding='same',
name=name + '_net_1x1_avg')(net)
net_1x1_avg = Conv2D(filters=64,
kernel_size=1,
activation='relu',
padding='same',
name=name + '_net_1x1_avg_conv1',
kernel_regularizer=regularizers.l2(
self.weight_decay))(net_1x1_avg)
net = concatenate([net_1x1, net_5x5, net_3x3, net_1x1_avg],
axis=-1,
name=name + '_mixed')
return net
def unet_seg(input_size=(128, 128, 1)):
inputs = tf.keras.Input(shape=input_size)
conv1 = Conv2D(64,
3,
activation='relu',
dilation_rate=2,
padding='same',
kernel_initializer='he_normal')(inputs)
conv1 = BatchNormalization()(conv1)
conv1 = Conv2D(64,
3,
activation='relu',
dilation_rate=2,
padding='same',
kernel_initializer='he_normal')(conv1)
conv1 = BatchNormalization()(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(128,
3,
activation='relu',
dilation_rate=2,
padding='same',
kernel_initializer='he_normal')(pool1)
conv2 = BatchNormalization()(conv2)
conv2 = Conv2D(128,
3,
activation='relu',
dilation_rate=2,
padding='same',
kernel_initializer='he_normal')(conv2)
conv2 = BatchNormalization()(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool2)
conv3 = BatchNormalization()(conv3)
conv3 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv3)
conv3 = BatchNormalization()(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool3)
conv4 = BatchNormalization()(conv4)
conv4 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv4)
conv4 = BatchNormalization()(conv4)
drop4 = Dropout(0.5)(conv4, training=True)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv5 = Conv2D(1024,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool4)
conv5 = BatchNormalization()(conv5)
conv5 = Conv2D(1024,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv5)
conv5 = BatchNormalization()(conv5)
drop5 = Dropout(0.5)(conv5, training=True)
up6 = Conv2D(512,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(drop5))
merge6 = concatenate([drop4, up6], axis=3)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge6)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv6)
up7 = Conv2D(256,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv6))
merge7 = concatenate([conv3, up7], axis=3)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge7)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv7)
up8 = Conv2D(128,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv7))
merge8 = concatenate([conv2, up8], axis=3)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge8)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv8)
up9 = Conv2D(64,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv8))
merge9 = concatenate([conv1, up9], axis=3)
conv9 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge9)
conv9 = Conv2D(32,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv10 = Conv2D(input_size[2], 1, activation='sigmoid')(conv9)
model = tf.keras.Model(inputs=inputs, outputs=conv10)
return model
def ASPP(tensor):
'''atrous spatial pyramid pooling'''
dims = K.int_shape(tensor)
y_pool = AveragePooling2D(pool_size=(dims[1], dims[2]),
name='average_pooling')(tensor)
y_pool = Conv2D(filters=2048,
kernel_size=1,
padding='same',
kernel_initializer='he_normal',
name='pool_1x1conv2d',
use_bias=False)(y_pool)
y_pool = BatchNormalization(name=f'bn_1')(y_pool)
y_pool = Activation('relu', name=f'relu_1')(y_pool)
y_pool = Upsample(tensor=y_pool, size=[dims[1], dims[2]])
y_1 = Conv2D(filters=2048,
kernel_size=1,
dilation_rate=1,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_d1',
use_bias=False)(tensor)
y_1 = BatchNormalization(name=f'bn_2')(y_1)
y_1 = Activation('relu', name=f'relu_2')(y_1)
y_6 = Conv2D(filters=2048,
kernel_size=3,
dilation_rate=6,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_d6',
use_bias=False)(tensor)
y_6 = BatchNormalization(name=f'bn_3')(y_6)
y_6 = Activation('relu', name=f'relu_3')(y_6)
y_12 = Conv2D(filters=2048,
kernel_size=3,
dilation_rate=12,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_d12',
use_bias=False)(tensor)
y_12 = BatchNormalization(name=f'bn_4')(y_12)
y_12 = Activation('relu', name=f'relu_4')(y_12)
y_18 = Conv2D(filters=2048,
kernel_size=3,
dilation_rate=18,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_d18',
use_bias=False)(tensor)
y_18 = BatchNormalization(name=f'bn_5')(y_18)
y_18 = Activation('relu', name=f'relu_5')(y_18)
y = concatenate([y_pool, y_1, y_6, y_12, y_18], name='ASPP_concat')
y = Conv2D(filters=2048,
kernel_size=1,
dilation_rate=1,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_final',
use_bias=False)(y)
y = BatchNormalization(name=f'bn_final')(y)
y = Activation('relu', name=f'relu_final')(y)
return y
def InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input image size for this model is 299x299.
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=299,
min_size=139,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl,
96,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3,
320,
3,
3,
strides=(2, 2),
padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(branch7x7x3,
192,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch7x7x3, branch_pool],
axis=channel_axis,
name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate([branch3x3_1, branch3x3_2],
axis=channel_axis,
name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate([branch3x3dbl_1, branch3x3dbl_2],
axis=channel_axis)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
x = Flatten(name='custom')(x) ##DB
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='inception_v3')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
file_hash='9a0d58056eeedaa3f26cb7ebd46da564')
else:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='bcbd6486424b2319ff4ef7d526e38f63')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def InceptionV3(input_shape=None):
input_shape = imagenet_utils.obtain_input_shape(
input_shape,
default_size=299,
min_size=75,
data_format=backend.image_data_format(),
require_flatten=True)
img_input = layers.Input(shape=input_shape)
if backend.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 288
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 288
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl,
96,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3,
320,
3,
3,
strides=(2, 2),
padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(branch7x7x3,
192,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch7x7x3, branch_pool],
axis=channel_axis,
name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate([branch3x3_1, branch3x3_2],
axis=channel_axis,
name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate([branch3x3dbl_1, branch3x3dbl_2],
axis=channel_axis)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
# Classification block
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
imagenet_utils.validate_activation('softmax', None)
x = layers.Dense(NUM_CLASSES, activation='softmax', name='predictions')(x)
# Create model.
model = training.Model(img_input, x, name='inception_v3')
return model
y_batch = to_categorical(y_batch, num_classes=len(POSSIBLE_LABELS))
yield x_batch, y_batch
x_in = Input(shape=(257, 98, 2))
x = BatchNormalization()(x_in)
for i in range(4):
x = Conv2D(16 * (2**i), (3, 3))(x)
x = Activation('elu')(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(128, (1, 1))(x)
x_branch_1 = GlobalAveragePooling2D()(x)
x_branch_2 = GlobalMaxPool2D()(x)
x = concatenate([x_branch_1, x_branch_2])
x = Dense(256, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(len(POSSIBLE_LABELS), activation='sigmoid')(x)
model = Model(inputs=x_in, outputs=x)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
from keras_tqdm import TQDMNotebookCallback
from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
callbacks = [
EarlyStopping(monitor='val_loss',
patience=5,
def transformer3_pooling(ih, iw, nb_conv, size_conv, nb_gpu):
"""
The cnn image transformation model with 3 times of downsampling. The downsampling uses maxpooling.
Parameters
----------
ih, iw : int
The input image dimension
nb_conv : int
Number of convolution kernels for each layer
size_conv : int
The size of convolution kernel
Returns
-------
mdl
Description.
"""
inputs = Input((ih, iw, 1))
conv1 = Conv2D(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(inputs)
conv1 = Conv2D(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(pool1)
conv2 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(pool2)
conv3 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(pool3)
conv4 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(conv4)
conv4 = Conv2D(1, (size_conv, size_conv),
activation='relu',
padding='same')(conv4)
#
fc1 = Flatten()(conv4)
fc1 = Dense(iw * ih / 128, activation='relu')(fc1)
fc1 = Dropout(0.2)(fc1)
fc1 = Dense(iw * ih / 128, activation='relu')(fc1)
fc1 = Dropout(0.25)(fc1)
fc1 = Dense(iw * ih / 64, activation='relu')(fc1)
fc1 = Dropout(0.25)(fc1)
fc1 = Reshape((int(ih // 8), int(iw // 8), 1))(fc1)
fc2 = Conv2DTranspose(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(fc1)
fc2 = Conv2DTranspose(nb_conv * 8, (size_conv, size_conv),
activation='relu',
padding='same')(fc2)
up1 = concatenate([UpSampling2D(size=(2, 2))(fc2), conv3], axis=3)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up1)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv6)
up2 = concatenate([UpSampling2D(size=(2, 2))(conv6), conv2], axis=3)
conv7 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up2)
conv7 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv7)
up3 = concatenate([UpSampling2D(size=(2, 2))(conv7), conv1], axis=3)
conv8 = Conv2DTranspose(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(up3)
conv8 = Conv2DTranspose(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(conv8)
conv8 = Conv2DTranspose(1, (3, 3), activation='relu',
padding='same')(conv8)
mdl = Model(inputs=inputs, outputs=conv8)
# if nb_gpu > 1:
# mdl = multi_gpu_model(mdl, nb_gpu)
mdl.compile(loss='mse', optimizer='Adam')
return mdl
def create_network(
Zt,
Ct,
channels=1,
upscaling_filters=[512, 256, 256, 128, 64],
upscaling_ks=[5, 5, 5, 5, 5],
upscaling_strides=[2, 2, 2, 2, 2, 1],
encoder_filters=[64, 128, 256],
encoder_ks=[3, 3, 3],
resblock_filters=[256, 256, 256, 256, 256],
resblock_ks=[3, 3, 3, 3, 3, 3],
decoder_filters=[256, 256, 128, 64],
decoder_ks=[5, 5, 5, 5, 5],
):
with tf.name_scope("Gen"):
Z = kl.Input((
None,
None,
channels,
), tensor=Zt, name="Z")
C = kl.Input((
None,
None,
channels,
), tensor=Ct, name="C")
layer = Z
# Upscaling
for l in range(len(upscaling_filters)):
layer = kl.Conv2DTranspose(filters=upscaling_filters[l],
kernel_size=upscaling_ks[l],
padding="same",
strides=upscaling_strides[l],
kernel_regularizer=kr.l2(),
activation="relu")(layer)
layer = kl.BatchNormalization()(layer)
layer = kl.Conv2DTranspose(filters=channels,
kernel_size=upscaling_ks[-1],
padding="same",
strides=1,
activation="relu",
kernel_regularizer=kr.l2())(layer)
layer = kl.concatenate([layer, C])
# Encoder
skips = [C]
for l in range(len(encoder_filters)):
layer = kl.Conv2D(
filters=encoder_filters[l],
kernel_size=encoder_ks[l],
padding="same",
activation="relu",
# strides=2,
kernel_regularizer=kr.l2())(layer)
layer = kl.AveragePooling2D()(layer)
layer = kl.BatchNormalization()(layer)
skips.append(layer)
# Residual blocks
for l in range(len(resblock_filters)):
layer = ResidualBlock(resblock_filters[l],
nb_layers=3,
kernel_size=resblock_ks[l])(layer)
# Decoder
layer = kl.Conv2DTranspose(filters=decoder_filters[0],
kernel_size=decoder_ks[0],
padding="same",
strides=2,
kernel_regularizer=kr.l2(),
activation="relu")(layer)
layer = kl.BatchNormalization()(layer)
skips = skips[::-1]
for l in range(1, len(decoder_filters)):
layer = kl.concatenate([layer, skips[l]])
layer = kl.Conv2DTranspose(filters=decoder_filters[l],
kernel_size=decoder_ks[l],
padding="same",
strides=2,
kernel_regularizer=kr.l2(),
activation="relu")(layer)
layer = kl.BatchNormalization()(layer)
G_out = kl.Conv2D(filters=channels,
kernel_size=5,
activation="tanh",
padding="same",
kernel_regularizer=kr.l2())(layer)
model = k.Model(inputs=[Z, C], outputs=G_out, name="G")
return model
def transformer3_direct(ih, iw, nb_conv, size_conv):
"""
The cnn model for image transformation with 3 times downsampling. The downsampling uses strides. It does not have
merged layers. It will lose resolution but possible to generate more different images.
Parameters
----------
ih, iw : int
The input image dimension
nb_conv : int
Number of convolution kernels for each layer
size_conv : int
The size of convolution kernel
Returns
-------
mdl
Description.
"""
inputs = Input((ih, iw, 1))
conv1 = Conv2D(nb_conv, (size_conv, size_conv),
activation='relu',
padding='same')(inputs)
conv1a = Conv2D(nb_conv, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv1)
conv2 = Conv2D(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv1a)
conv2a = Conv2D(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv2)
conv3 = Conv2D(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(conv2a)
conv3 = Conv2D(nb_conv * 4, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv3)
fc1 = Flatten()(conv3)
fc1 = Dense(iw * ih / 64, activation='relu')(fc1)
fc1 = Dropout(0.2)(fc1)
fc1 = Dense(iw * ih / 16, activation='relu')(fc1)
fc1 = Dropout(0.25)(fc1)
fc1 = Reshape((ih // 4, iw // 4, 1))(fc1)
fc2 = Conv2DTranspose(nb_conv * 4, (size_conv, size_conv),
activation='relu',
padding='same')(fc1)
fc2 = Conv2DTranspose(nb_conv * 4, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(fc2)
up1 = concatenate([fc2, conv2], axis=3)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up1)
conv6 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
strides=(2, 2),
activation='relu',
padding='same')(conv6)
up2 = concatenate([conv6, conv1], axis=3)
conv7 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(up2)
conv7 = Conv2DTranspose(nb_conv * 2, (size_conv, size_conv),
activation='relu',
padding='same')(conv7)
conv8 = Conv2DTranspose(1, (3, 3), activation='relu',
padding='same')(conv7)
mdl = Model(inputs=inputs, outputs=conv8)
mdl.compile(loss=psnr, optimizer='Adam', metrics=['mse'])
return mdl
def test_model():
s_cols = ['C1', 'C2', 'C3']
d_cols = ['I1', 'I2']
# sparse
print('----------- sparse features ------------')
sparse_input = []
lr_embedding = []
for s_col in s_cols:
_input = Input(shape=(1, ))
sparse_input.append(_input)
print('sparse input: ', sparse_input)
nums = pd.concat((x_train[s_col], x_valid[s_col])).nunique() + 1
embed = Flatten()(Embedding(
nums,
4,
input_length=1,
embeddings_regularizer=tf.keras.regularizers.l2(0.5))(
_input)) # shape=(None, 1, 4) -> shape=(None, 4)
lr_embedding.append(embed)
print('\nsparse emb: ', lr_embedding)
fst_order_sparse_layer = concatenate(lr_embedding)
print('\nsparse layer: ', fst_order_sparse_layer)
# dense
print('----------- dense features ------------')
dense_input = []
for d_col in d_cols:
_input = Input(shape=(1, ))
dense_input.append(_input)
print('dense input: ', dense_input)
concat_dense_input = concatenate(dense_input)
print('\nfinal dense input: ', concat_dense_input)
fst_order_dense_layer = Dense(4, activation='relu')(concat_dense_input)
print('\ndense layer: ', fst_order_dense_layer)
# linear concat
print('----------- linear part ------------')
linear_part = concatenate([fst_order_dense_layer, fst_order_sparse_layer])
print('linear part: ', linear_part)
#######dnn layer##########
print('----------- dnn part ------------')
print('dnn input layer: ', linear_part)
fc_layer = Dropout(0.2)(Activation(activation="relu")(BatchNormalization()(
Dense(128)(linear_part))))
print('full conection layer1: ', fc_layer)
fc_layer = Dropout(0.2)(Activation(activation="relu")(BatchNormalization()(
Dense(64)(fc_layer))))
print('full conection layer2: ', fc_layer)
fc_layer = Dropout(0.2)(Activation(activation="relu")(BatchNormalization()(
Dense(32)(fc_layer))))
print('full conection layer3: ', fc_layer)
######## output layer ##########
print('----------- output part ------------')
print('linear layer: ', linear_part)
print('dnn layer: ', fc_layer)
output_layer = concatenate([linear_part, fc_layer])
print('hidden layer to output: ', output_layer)
output_layer = Dense(1, activation='sigmoid')(output_layer)
print('output layer: ', output_layer)
######## model ##########
print('----------- model ------------')
model = Model(inputs=sparse_input + dense_input, outputs=output_layer)
print('input: ', sparse_input + dense_input)
print('\noutput: ', output_layer)
print('\nmodel: ', model)
def unet_512(input_shape=(512, 512, 3), num_classes=1, dropout=0.25):
inputs = Input(shape=input_shape)
# s = Lambda(lambda x: x / 255) (inputs)
#
# 512
down0a = Conv2D(16, (3, 3), padding='same')(inputs)
down0a = BatchNormalization()(down0a)
down0a = Activation('relu')(down0a)
down0a = Conv2D(16, (3, 3), padding='same')(down0a)
down0a = BatchNormalization()(down0a)
down0a = Activation('relu')(down0a)
down0a_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0a)
# 256
down0 = Conv2D(32, (3, 3), padding='same')(down0a_pool)
down0 = BatchNormalization()(down0)
down0 = Activation('relu')(down0)
down0 = Conv2D(32, (3, 3), padding='same')(down0)
down0 = BatchNormalization()(down0)
down0 = Activation('relu')(down0)
down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)
p2 = Dropout(dropout)(down0_pool)
# 128
down1 = Conv2D(64, (3, 3), padding='same')(p2)
down1 = BatchNormalization()(down1)
down1 = Activation('relu')(down1)
down1 = Conv2D(64, (3, 3), padding='same')(down1)
down1 = BatchNormalization()(down1)
down1 = Activation('relu')(down1)
down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
p3 = Dropout(dropout)(down1_pool)
# 64
down2 = Conv2D(128, (3, 3), padding='same')(p3)
down2 = BatchNormalization()(down2)
down2 = Activation('relu')(down2)
down2 = Conv2D(128, (3, 3), padding='same')(down2)
down2 = BatchNormalization()(down2)
down2 = Activation('relu')(down2)
down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
p4 = Dropout(dropout)(down2_pool)
# 32
down3 = Conv2D(256, (3, 3), padding='same')(p4)
down3 = BatchNormalization()(down3)
down3 = Activation('relu')(down3)
down3 = Conv2D(256, (3, 3), padding='same')(down3)
down3 = BatchNormalization()(down3)
down3 = Activation('relu')(down3)
down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)
p5 = Dropout(dropout)(down3_pool)
# 16
down4 = Conv2D(512, (3, 3), padding='same')(p5)
down4 = BatchNormalization()(down4)
down4 = Activation('relu')(down4)
down4 = Conv2D(512, (3, 3), padding='same')(down4)
down4 = BatchNormalization()(down4)
down4 = Activation('relu')(down4)
down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)
p6 = Dropout(dropout)(down4_pool)
# 8
center = Conv2D(1024, (3, 3), padding='same')(p6)
center = BatchNormalization()(center)
center = Activation('relu')(center)
center = Conv2D(1024, (3, 3), padding='same')(center)
center = BatchNormalization()(center)
center = Activation('relu')(center)
# center
up4 = UpSampling2D((2, 2))(center)
up4 = concatenate([down4, up4], axis=3)
p7 = Dropout(dropout)(up4)
up4 = Conv2D(512, (3, 3), padding='same')(p7)
up4 = BatchNormalization()(up4)
up4 = Activation('relu')(up4)
up4 = Conv2D(512, (3, 3), padding='same')(up4)
up4 = BatchNormalization()(up4)
up4 = Activation('relu')(up4)
up4 = Conv2D(512, (3, 3), padding='same')(up4)
up4 = BatchNormalization()(up4)
up4 = Activation('relu')(up4)
# 16
up3 = UpSampling2D((2, 2))(up4)
up3 = concatenate([down3, up3], axis=3)
p8 = Dropout(dropout)(up3)
up3 = Conv2D(256, (3, 3), padding='same')(p8)
up3 = BatchNormalization()(up3)
up3 = Activation('relu')(up3)
up3 = Conv2D(256, (3, 3), padding='same')(up3)
up3 = BatchNormalization()(up3)
up3 = Activation('relu')(up3)
up3 = Conv2D(256, (3, 3), padding='same')(up3)
up3 = BatchNormalization()(up3)
up3 = Activation('relu')(up3)
# 32
up2 = UpSampling2D((2, 2))(up3)
up2 = concatenate([down2, up2], axis=3)
p9 = Dropout(dropout)(up2)
up2 = Conv2D(128, (3, 3), padding='same')(p9)
up2 = BatchNormalization()(up2)
up2 = Activation('relu')(up2)
up2 = Conv2D(128, (3, 3), padding='same')(up2)
up2 = BatchNormalization()(up2)
up2 = Activation('relu')(up2)
up2 = Conv2D(128, (3, 3), padding='same')(up2)
up2 = BatchNormalization()(up2)
up2 = Activation('relu')(up2)
# 64
up1 = UpSampling2D((2, 2))(up2)
up1 = concatenate([down1, up1], axis=3)
p10 = Dropout(dropout)(up1)
up1 = Conv2D(64, (3, 3), padding='same')(p10)
up1 = BatchNormalization()(up1)
up1 = Activation('relu')(up1)
up1 = Conv2D(64, (3, 3), padding='same')(up1)
up1 = BatchNormalization()(up1)
up1 = Activation('relu')(up1)
up1 = Conv2D(64, (3, 3), padding='same')(up1)
up1 = BatchNormalization()(up1)
up1 = Activation('relu')(up1)
# 128
up0 = UpSampling2D((2, 2))(up1)
up0 = concatenate([down0, up0], axis=3)
p11 = Dropout(dropout)(up0)
up0 = Conv2D(32, (3, 3), padding='same')(p11)
up0 = BatchNormalization()(up0)
up0 = Activation('relu')(up0)
up0 = Conv2D(32, (3, 3), padding='same')(up0)
up0 = BatchNormalization()(up0)
up0 = Activation('relu')(up0)
up0 = Conv2D(32, (3, 3), padding='same')(up0)
up0 = BatchNormalization()(up0)
up0 = Activation('relu')(up0)
# 256
up0a = UpSampling2D((2, 2))(up0)
up0a = concatenate([down0a, up0a], axis=3)
up0a = Conv2D(16, (3, 3), padding='same')(up0a)
up0a = BatchNormalization()(up0a)
up0a = Activation('relu')(up0a)
up0a = Conv2D(16, (3, 3), padding='same')(up0a)
up0a = BatchNormalization()(up0a)
up0a = Activation('relu')(up0a)
up0a = Conv2D(16, (3, 3), padding='same')(up0a)
up0a = BatchNormalization()(up0a)
up0a = Activation('relu')(up0a)
# 512
classify = Conv2D(num_classes, (1, 1), activation='sigmoid')(up0a)
model = Model(inputs=[inputs], outputs=[classify])
model.load_weights(
'/home/tonee/.config/spyder-py3/deeplearning practise/line_crop_best_9000.h5'
)
# model.compile(optimizer = RMSprop(lr=0.001), loss = [jacard_coef_loss], metrics = [jacard_coef])
# model.compile(optimizer = Adam(lr = 1e-4), loss = [jacard_coef_loss], metrics = [jacard_coef])
model.compile(optimizer=RMSprop(lr=0.0001),
loss=bce_dice_loss,
metrics=[dice_coeff, jacard_coef])
# model.summary()
return model
def InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
model_input=None,
pooling=None,
classes=1000,
model_path=""):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input image size for this model is 299x299.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or 'imagenet' (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
img_input = model_input
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl,
96,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3,
320,
3,
3,
strides=(2, 2),
padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(branch7x7x3,
192,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch7x7x3, branch_pool],
axis=channel_axis,
name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate([branch3x3_1, branch3x3_2],
axis=channel_axis,
name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate([branch3x3dbl_1, branch3x3dbl_2],
axis=channel_axis)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
inputs = img_input
# Create model.
model = Model(inputs, x, name='inception_v3')
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
if include_top:
weights_path = model_path
model.load_weights(weights_path)
else:
weights_path = ""
elif (weights == "trained"):
weights_path = model_path
model.load_weights(weights_path)
return model
def _reduction_a_cell(ip, p, filters, block_id=None):
"""Adds a Reduction cell for NASNet-A (Fig. 4 in the paper).
Arguments:
ip: Input tensor `x`
p: Input tensor `p`
filters: Number of output filters
block_id: String block_id
Returns:
A Keras tensor
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
with K.name_scope('reduction_A_block_%s' % block_id):
p = _adjust_block(p, ip, filters, block_id)
h = Activation('relu')(ip)
h = Conv2D(
filters, (1, 1),
strides=(1, 1),
padding='same',
name='reduction_conv_1_%s' % block_id,
use_bias=False,
kernel_initializer='he_normal')(
h)
h = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='reduction_bn_1_%s' % block_id)(
h)
with K.name_scope('block_1'):
x1_1 = _separable_conv_block(
h,
filters, (5, 5),
strides=(2, 2),
block_id='reduction_left1_%s' % block_id)
x1_2 = _separable_conv_block(
p,
filters, (7, 7),
strides=(2, 2),
block_id='reduction_1_%s' % block_id)
x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % block_id)
with K.name_scope('block_2'):
x2_1 = MaxPooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_left2_%s' % block_id)(
h)
x2_2 = _separable_conv_block(
p,
filters, (7, 7),
strides=(2, 2),
block_id='reduction_right2_%s' % block_id)
x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % block_id)
with K.name_scope('block_3'):
x3_1 = AveragePooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_left3_%s' % block_id)(
h)
x3_2 = _separable_conv_block(
p,
filters, (5, 5),
strides=(2, 2),
block_id='reduction_right3_%s' % block_id)
x3 = add([x3_1, x3_2], name='reduction_add3_%s' % block_id)
with K.name_scope('block_4'):
x4 = AveragePooling2D(
(3, 3),
strides=(1, 1),
padding='same',
name='reduction_left4_%s' % block_id)(
x1)
x4 = add([x2, x4])
with K.name_scope('block_5'):
x5_1 = _separable_conv_block(
x1, filters, (3, 3), block_id='reduction_left4_%s' % block_id)
x5_2 = MaxPooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_right5_%s' % block_id)(
h)
x5 = add([x5_1, x5_2], name='reduction_add4_%s' % block_id)
x = concatenate(
[x2, x3, x4, x5],
axis=channel_dim,
name='reduction_concat_%s' % block_id)
return x, ip
def compute_cost_volume(inputs,
t_s_ids,
min_disp=None,
max_disp=None,
labels=None,
move_path=None,
logger=None):
# 1.initialization
if isinstance(inputs, (list, )):
View_n = len(inputs)
_, H, W, C = K.int_shape(inputs[int(
(View_n - 1) / 2)]) # batch, height, width, channel of features
# reference (-> central view)
reference = inputs[int((View_n - 1) / 2)]
# init spatial coordinates (of central view)
cords = tf.zeros_like(inputs[0])[..., :2]
# angular coordinate (of central view)
cent_t_id = int((View_n - 1) / 2)
cent_s_id = int((View_n - 1) / 2)
# 1 label = the number of disparity
label_disp = (max_disp - min_disp) / labels
# camera moving/image view path
if move_path == "LT":
t_sign = 1
s_sign = 1
# cost list (contains a list of cost slices at all disparity offsets)
cost_l = []
# 2.iterate from the minimum to maximum disparity by every disparity unit (label_disp)
for disp in np.arange(min_disp, max_disp, label_disp):
feature_maps = []
id = 0
for t_id, s_id in zip(t_s_ids[0], t_s_ids[1]):
# append (translated) feature maps
if t_id == cent_t_id and s_id == cent_s_id:
# self
feature_maps.append(reference)
continue
else:
# add 1 dimension: W; tile: repeat to W columns.
tmp0 = tf.cast(
tf.tile(
tf.expand_dims(
tf.clip_by_value(
tf.range(H) + t_sign *
(cent_t_id - t_id) * disp, 0, H), 1), [1, W]),
tf.float32)
# add 0 dimension: H
tmp1 = tf.cast(
tf.tile(
tf.expand_dims(
tf.clip_by_value(
tf.range(W) + s_sign *
(cent_s_id - s_id) * disp, 0, W), 0), [H, 1]),
tf.float32)
cords = tf.cast(
tf.tile(
tf.expand_dims(tf.stack([tmp0, tmp1], axis=2), axis=0),
[tf.shape(cords)[0], 1, 1, 1]), tf.float32)
# shift: shift/translate feature maps by disparities
target = Lambda(lambda x: bilinear_sampler(*x))(
[inputs[id], cords])
# others
feature_maps.append(target)
id += 1
# get a cost slice
cost = concatenate(feature_maps, name='cost_d') # DxHxWx(NC)
cost_l.append(cost)
# 3.stack all cost slices to get a [cost volume]
cost_volume = K.stack(cost_l, axis=1)
return cost_volume
def _adjust_block(p, ip, filters, block_id=None):
"""Adjusts the input `previous path` to match the shape of the `input`.
Used in situations where the output number of filters needs to be changed.
Arguments:
p: Input tensor which needs to be modified
ip: Input tensor whose shape needs to be matched
filters: Number of output filters to be matched
block_id: String block_id
Returns:
Adjusted Keras tensor
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
img_dim = 2 if K.image_data_format() == 'channels_first' else -2
ip_shape = K.int_shape(ip)
if p is not None:
p_shape = K.int_shape(p)
with K.name_scope('adjust_block'):
if p is None:
p = ip
elif p_shape[img_dim] != ip_shape[img_dim]:
with K.name_scope('adjust_reduction_block_%s' % block_id):
p = Activation('relu', name='adjust_relu_1_%s' % block_id)(p)
p1 = AveragePooling2D(
(1, 1),
strides=(2, 2),
padding='valid',
name='adjust_avg_pool_1_%s' % block_id)(p)
p1 = Conv2D(filters // 2, (1, 1),
padding='same',
use_bias=False,
name='adjust_conv_1_%s' % block_id,
kernel_initializer='he_normal')(p1)
p2 = ZeroPadding2D(padding=((0, 1), (0, 1)))(p)
p2 = Cropping2D(cropping=((1, 0), (1, 0)))(p2)
p2 = AveragePooling2D(
(1, 1),
strides=(2, 2),
padding='valid',
name='adjust_avg_pool_2_%s' % block_id)(p2)
p2 = Conv2D(filters // 2, (1, 1),
padding='same',
use_bias=False,
name='adjust_conv_2_%s' % block_id,
kernel_initializer='he_normal')(p2)
p = concatenate([p1, p2], axis=channel_dim)
p = BatchNormalization(axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='adjust_bn_%s' % block_id)(p)
elif p_shape[channel_dim] != filters:
with K.name_scope('adjust_projection_block_%s' % block_id):
p = Activation('relu')(p)
p = Conv2D(filters, (1, 1),
strides=(1, 1),
padding='same',
name='adjust_conv_projection_%s' % block_id,
use_bias=False,
kernel_initializer='he_normal')(p)
p = BatchNormalization(axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='adjust_bn_%s' % block_id)(p)
return p
def InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
model_input=None,
pooling=None,
classes=1000, model_path=""):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input image size for this model is 299x299.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or 'imagenet' (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
img_input = model_input
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(
branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D(
(3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3, 320, 3, 3,
strides=(2, 2), padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(
branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate(
[branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate(
[branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis)
branch_pool = AveragePooling2D(
(3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
inputs = img_input
# Create model.
model = Model(inputs, x, name='inception_v3')
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
if include_top:
weights_path = model_path
model.load_weights(weights_path)
else:
weights_path = ""
elif (weights == "trained"):
weights_path = model_path
model.load_weights(weights_path)
return model
