def NormalCell(hi,
h_i1sub,
filter_i,
filter_o,
stride=1,
name="NAS",
use_deform=False,
is_tail=False):
"""
adjust feature size & channel size
"""
hi = Conv2D(filter_o, (1, 1),
padding='same',
name='%s_hi_align' % name,
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(hi)
hi = Activation('relu', name='%s_hi_align_relu' % name)(hi)
hi = BatchNormalization(name='%s_hi_align_bn' % name)(hi)
if use_deform:
hi = SepConvOffset2D(filter_o,
name='%s_deform_conv' % name)(hi,
use_resam=True)
h_i1sub = _adjust_block(h_i1sub, hi, filter_o, 0, name)
"""
SubLayer
"""
l = SeparableConv2D(filter_o, (3, 3),
padding='same',
name='%s_sep3x3_1' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(hi)
l = Activation('relu', name='%s_sep3x3_1_relu' % name)(l)
l = BatchNormalization(name='%s_sep3x3_1_bn' % name)(l)
add1 = Add()([l, hi])
l = SeparableConv2D(filter_o, (5, 5),
padding='same',
name='%s_sep5x5_1' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(hi)
l = Activation('relu', name='%s_sep5x5_1_relu' % name)(l)
l = BatchNormalization(name='%s_sep5x5_1_bn' % name)(l)
if h_i1sub is not None:
l_1sub = SeparableConv2D(
filter_o, (3, 3),
padding='same',
name='%s_sep3x3_sub_1' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(h_i1sub)
l_1sub = Activation('relu',
name='%s_sep3x3_sub_1_relu' % name)(l_1sub)
l_1sub = BatchNormalization(name='%s_sep3x3_sub_1_bn' %
name)(l_1sub)
add2 = Add()([l, l_1sub])
else:
add2 = l
l = SeparableConv2D(filter_o, (3, 3),
padding='same',
name='%s_sep3x3_2' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(hi)
l = Activation('relu', name='%s_sep3x3_2_relu' % name)(l)
l = BatchNormalization(name='%s_sep3x3_2_bn' % name)(l)
if h_i1sub is not None:
l_1sub = SeparableConv2D(
filter_o, (5, 5),
padding='same',
name='%s_sep5x5_sub_1' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(h_i1sub)
l_1sub = Activation('relu',
name='%s_sep5x5_sub_1_relu' % name)(l_1sub)
l_1sub = BatchNormalization(name='%s_sep5x5_sub_1_bn' %
name)(l_1sub)
add3 = Add()([l, l_1sub])
else:
add3 = l
if h_i1sub is not None:
avg_p1 = AveragePooling2D(pool_size=(3, 3),
strides=(stride, stride),
padding='same')(h_i1sub)
# avg_p2 = AveragePooling2D(pool_size=(3, 3), strides=(stride, stride), padding='same')(h_i1sub)
# add4 = Add()([avg_p1, avg_p2])
add4 = avg_p1
l_1sub_1 = SeparableConv2D(
filter_o, (3, 3),
padding='same',
name='%s_sep3x3_sub_2' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(h_i1sub)
l_1sub_1 = Activation('relu',
name='%s_sep3x3_sub_2_relu' % name)(l_1sub_1)
l_1sub_1 = BatchNormalization(name='%s_sep3x3_sub_2_bn' %
name)(l_1sub_1)
l_1sub_2 = SeparableConv2D(
filter_o, (5, 5),
padding='same',
name='%s_sep5x5_sub_2' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(h_i1sub)
l_1sub_2 = Activation('relu',
name='%s_sep5x5_sub_2_relu' % name)(l_1sub_2)
l_1sub_2 = BatchNormalization(name='%s_sep5x5_sub_2_bn' %
name)(l_1sub_2)
add5 = Add()([l_1sub_1, l_1sub_2])
result = concatenate([add1, add2, add3, add4, add5])
if is_tail:
result = Conv2D(filter_o, (1, 1),
strides=(1, 1),
padding='same',
name='%s_result1x1_align' % name,
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(result)
result = Activation('relu',
name='%s_result1x1_align_relu' %
name)(result)
result = BatchNormalization(name='%s_result1x1_align_bn' %
name)(result)
return result
else:
result = concatenate([add1, add2, add3])
if is_tail:
result = Conv2D(filter_o, (1, 1),
strides=(1, 1),
padding='same',
name='%s_result1x1_align' % name,
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(result)
result = Activation('relu',
name='%s_result1x1_align_relu' %
name)(result)
result = BatchNormalization(name='%s_result1x1_align_bn' %
name)(result)
return result
def create_model_duel(window,
input_shape,
num_actions,
init_method,
model_name='q_network'): # noqa: D103
print("Built a dueling sequential DQN")
input_rows, input_cols = input_shape[0], input_shape[1]
input = Input(shape=(window, input_rows, input_cols))
if init_method == 'he':
l_1 = Conv2D(16,
kernel_size=(8, 8),
strides=(4, 4),
padding='same',
kernel_initializer=initializers.he_normal(),
activation='relu',
input_shape=(window, input_rows, input_cols))(input)
l_2 = Conv2D(32,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer=initializers.he_normal(),
activation='relu')(l_1)
l_3 = Flatten()(l_2)
v_1 = Dense(128,
activation='relu',
kernel_initializer=initializers.he_normal())(l_3)
elif init_method == 'default':
l_1 = Conv2D(16,
kernel_size=(8, 8),
strides=(4, 4),
padding='same',
activation='relu',
input_shape=(window, input_rows, input_cols))(input)
l_2 = Conv2D(32,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
activation='relu')(l_1)
l_3 = Flatten()(l_2)
v_1 = Dense(128, activation='relu')(l_3)
v_2 = Dense(1)(v_1)
v_out = Lambda(lambda s: K.expand_dims(s[:, 0], axis=-1),
output_shape=(num_actions, ))(v_2)
if init_method == 'he':
a_1 = Dense(128,
activation='relu',
kernel_initializer=initializers.he_normal())(l_3)
elif init_method == 'default':
a_1 = Dense(128, activation='relu')(l_3)
a_2 = Dense(num_actions)(a_1)
a_out = Lambda(lambda a: a[:, :] - K.mean(a[:, :], keepdims=True),
output_shape=(num_actions, ))(a_2)
# q_out = keras.layers.merge([v_out, a_out], mode='add')
q_out = Add()([v_out, a_out])
model = Model(inputs=input, outputs=q_out)
return model
def decoder(
x,
output_shape,
encoded_shape,
conv_chans=None,
min_h=5,
min_c=4,
prefix='vte_dec',
n_convs_per_stage=1,
include_dropout=False,
ks=3,
include_skips=None,
use_residuals=False,
use_upsample=False,
use_batchnorm=False,
target_vol_sizes=None,
n_samples=1,
kernel_initializer=None,
bias_initializer=None,
):
n_dims = len(output_shape) - 1
if conv_chans is None:
n_convs = int(np.floor(np.log2(output_shape[0] / min_h)))
conv_chans = [min_c * 2] * n_convs
elif not type(conv_chans) == list:
n_convs = int(np.floor(np.log2(output_shape[0] / min_h)))
conv_chans = [conv_chans] * n_convs
elif type(conv_chans) == list:
n_convs = len(conv_chans)
if isinstance(ks, list):
assert len(ks) == (n_convs + 1
) # specify for each conv, as well as the last one
else:
ks = [ks] * (n_convs + 1)
print('Decoding with conv filters {}'.format(conv_chans))
# compute default sizes that we want on the way up, mainly in case we have more convs than stages
# and we upsample past the output size
if n_dims == 2:
# just upsample by a factor of 2 and then crop the final volume to the desired volume
default_target_vol_sizes = np.asarray(
[(int(encoded_shape[0] * 2.**(i + 1)),
int(encoded_shape[1] * 2.**(i + 1)))
for i in range(n_convs - 1)] + [output_shape[:2]])
else:
print(output_shape)
print(encoded_shape)
# just upsample by a factor of 2 and then crop the final volume to the desired volume
default_target_vol_sizes = np.asarray(
[(min(output_shape[0], int(encoded_shape[0] * 2.**(i + 1))),
min(output_shape[1], int(encoded_shape[1] * 2.**(i + 1))),
min(output_shape[2], int(encoded_shape[2] * 2.**(i + 1))))
for i in range(n_convs - 1)] + [output_shape[:3]])
# automatically stop when we reach the desired image shape
for vi, vs in enumerate(default_target_vol_sizes):
if np.all(vs >= output_shape[:-1]):
default_target_vol_sizes[vi] = output_shape[:-1]
print('Automatically computed target output sizes: {}'.format(
default_target_vol_sizes))
if target_vol_sizes is None:
target_vol_sizes = default_target_vol_sizes
else:
print('Target concat vols to match shapes to: {}'.format(
target_vol_sizes))
# TODO: check that this logic makes sense for more convs
# fill in any Nones that we might have in our target_vol_sizes
filled_target_vol_sizes = [None] * len(target_vol_sizes)
for i in range(n_convs):
if i < len(target_vol_sizes) and target_vol_sizes[i] is not None:
filled_target_vol_sizes[i] = target_vol_sizes[i]
target_vol_sizes = filled_target_vol_sizes
if include_skips is not None:
print('Concatentating padded/cropped shapes {} with skips {}'.format(
target_vol_sizes, include_skips))
curr_shape = np.asarray(encoded_shape[:n_dims])
for i in range(n_convs):
print(target_vol_sizes[i])
if i < len(target_vol_sizes) and target_vol_sizes[i] is not None:
x = _pad_or_crop_to_shape(x, curr_shape, target_vol_sizes[i])
curr_shape = np.asarray(
target_vol_sizes[i]) # we will upsample first thing next stage
# if we want to concatenate with another volume (e.g. from encoder, or a downsampled input)...
if include_skips is not None and i < len(
include_skips) and include_skips[i] is not None:
x_shape = x.get_shape().as_list()
skip_shape = include_skips[i].get_shape().as_list()
print(
'Attempting to concatenate current layer {} with previous skip connection {}'
.format(x_shape, skip_shape))
# input size might not match in time dimension, so just tile it
if n_samples > 1:
tile_factor = [1] + [n_samples] + [1] * (len(x_shape) - 1)
print('Tiling by {}'.format(tile_factor))
print(target_vol_sizes[i])
skip = Lambda(lambda y: K.expand_dims(y, axis=1))(
include_skips[i])
skip = Lambda(lambda y: tf.tile(y, tile_factor),
name='{}_lambdatilesamples_{}'.format(prefix, i),
output_shape=[n_samples] + skip_shape[1:])(skip)
skip = Lambda(lambda y: tf.reshape(y, [-1] + skip_shape[1:]),
output_shape=skip_shape[1:])(skip)
else:
skip = include_skips[i]
x = Concatenate(axis=-1,
name='{}_concatskip_{}'.format(prefix,
i))([x, skip])
for ci in range(n_convs_per_stage):
x = myConv(
conv_chans[i],
ks=ks[i],
strides=1,
n_dims=n_dims,
prefix=prefix,
suffix='{}_{}'.format(i, ci + 1),
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
)(x)
if use_batchnorm: # TODO: check to see if this should go before the residual
x = BatchNormalization()(x)
# if we want residuals, store them here
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_{}_add_residual'.format(prefix, i))(
[residual_input, x])
x = LeakyReLU(0.2,
name='{}_leakyrelu_{}_{}'.format(prefix, i,
ci + 1))(x)
if include_dropout and i < 2:
x = Dropout(0.3)(x)
# if we are not at the output resolution yet, upsample or do a transposed convolution
if not np.all(curr_shape == output_shape[:len(curr_shape)]):
if not use_upsample:
# if we have convolutional filters left at the end, just apply them at full resolution
x = myConvTranspose(
conv_chans[i],
n_dims=n_dims,
ks=ks[i],
strides=2,
prefix=prefix,
suffix=i,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
)(x)
if use_batchnorm:
x = BatchNormalization()(x)
x = LeakyReLU(0.2, name='{}_leakyrelu_{}'.format(prefix, i))(
x) # changed 5/15/2018, will break old models
else:
x = myUpsample(size=2, n_dims=n_dims, prefix=prefix,
suffix=i)(x)
curr_shape *= 2
# last stage of convolutions, no more upsampling
x = myConv(
output_shape[-1],
ks=ks[-1],
n_dims=n_dims,
strides=1,
prefix=prefix,
suffix='final',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
)(x)
return x
def get_large_deform_cnn(class_num, trainable=False):
# init = Orthogonal(gain=1.0, seed=None)
init = lecun_normal()
inputs = l = Input((200, 200, 3), name='input')
norm_input = ImageNorm()(inputs)
# norm_input = inputs
# conv11
l = Conv2D(32, (3, 3),
padding='same',
name='conv11',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l = Activation('selu', name='conv11_relu')(l)
l = BatchNormalization(name='conv11_bn', center=False, scale=False)(l)
l2 = InvConv2D(32, (3, 3),
padding='same',
name='inv_conv11',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l2 = Activation('selu', name='inv_conv11_relu')(l2)
l2 = BatchNormalization(name='inv_conv11_bn', center=False,
scale=False)(l2)
l3 = Conv2D(32, (3, 1),
padding='same',
name='conv11_2',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l3 = Activation('selu', name='conv11_2_relu')(l3)
l3 = BatchNormalization(name='conv11_2_bn', center=False, scale=False)(l3)
l5 = Conv2D(32, (1, 3),
padding='same',
name='conv11_3',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l5 = Activation('selu', name='conv11_3_relu')(l5)
l5 = BatchNormalization(name='conv11_3_bn', center=False, scale=False)(l5)
l = concatenate([l, l2, l3, l5])
# conv12
# l_offset = ConvOffset2D(32, name='conv12_offset')(l)
l = Conv2D(128, (3, 3),
padding='same',
strides=(2, 2),
name='pool11_12',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='pool11_12_relu')(l)
# l = BatchNormalization(name='conv12_bn', center=False, scale=False)(l)
l3 = Conv2D(32, (3, 1),
padding='same',
name='conv12_2',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l3 = Activation('selu', name='conv12_2_relu')(l3)
l3 = BatchNormalization(name='conv12_2_bn', center=False, scale=False)(l3)
l5 = Conv2D(32, (1, 3),
padding='same',
name='conv12_3',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l5 = Activation('selu', name='conv12_3_relu')(l5)
l5 = BatchNormalization(name='conv12_3_bn', center=False, scale=False)(l5)
l = Conv2D(128, (3, 3),
padding='same',
name='conv12',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv12_relu')(l)
l = BatchNormalization(name='conv12_bn', center=False, scale=False)(l)
l = concatenate([l, l3, l5])
l = Conv2D(128, (3, 3),
padding='same',
name='conv13',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv13_relu')(l)
l = BatchNormalization(name='conv13_bn', center=False, scale=False)(l)
l = Conv2D(128, (3, 3),
padding='same',
strides=(2, 2),
name='conv14',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv14_relu')(l)
l = BatchNormalization(name='conv14_bn', center=False, scale=False)(l)
# l = Conv2D(192, (3, 3), padding='same', name='conv21', trainable=trainable, kernel_initializer=init, kernel_regularizer=OrthLocalReg2D)(l)
# l = Activation('selu', name='conv21_relu')(l)
# l = BatchNormalization(name='conv21_bn', center=False, scale=False)(l)
#
# l = Conv2D(192, (3, 3), padding='same', name='conv22', trainable=trainable, kernel_initializer=init, kernel_regularizer=OrthLocalReg2D)(l)
# l = Activation('selu', name='conv22_relu')(l)
# l = BatchNormalization(name='conv22_bn', center=False, scale=False)(l)
# conv22
# l_offset = ConvOffset2D(192, name='conv32_offset')(l)
# l = Conv2D(192, (3, 3), padding='same', strides=(2, 2), name='conv23', trainable=trainable, kernel_initializer=init, kernel_regularizer=OrthLocalReg2D)(l)
# l = Activation('selu', name='conv23_relu')(l)
# l = BatchNormalization(name='conv23_bn', center=False, scale=False)(l)
l = Conv2D(256, (3, 3),
padding='same',
strides=(2, 2),
name='conv31',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv31_relu')(l)
l31 = BatchNormalization(name='conv31_bn', center=False, scale=False)(l)
l = Conv2D(256, (1, 1),
padding='same',
name='conv32',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l31)
l = Activation('selu', name='conv32_relu')(l)
l = BatchNormalization(name='conv32_bn', center=False, scale=False)(l)
l = Conv2D(256, (3, 3),
padding='same',
name='conv33',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv33_relu')(l)
l = BatchNormalization(name='conv33_bn', center=False, scale=False)(l)
l = Add(name='residual_31_32')([l31, l])
l_offset = ConvOffset2D(256, name='conv33_offset')(l)
l = Conv2D(512, (3, 3),
padding='same',
name='conv41',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l_offset)
l = Activation('selu', name='conv41_relu')(l)
l = BatchNormalization(name='conv41_bn', center=False, scale=False)(l)
l = Conv2D(512, (3, 3),
padding='same',
strides=(2, 2),
name='conv42',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv42_relu')(l)
l = BatchNormalization(name='conv42_bn', center=False, scale=False)(l)
# l_offset = ConvOffset2D(512, name='conv35_offset')(l)
l = Conv2D(512, (3, 3),
padding='same',
name='conv43',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv43_relu')(l)
l = BatchNormalization(name='conv43_bn', center=False, scale=False)(l)
l = Conv2D(1024, (3, 3),
padding='same',
strides=(2, 2),
name='conv51',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv43_relu')(l)
l = BatchNormalization(name='conv43_bn', center=False, scale=False)(l)
l = Conv2D(1024, (3, 3),
padding='same',
name='conv52',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv43_relu')(l)
l = BatchNormalization(name='conv43_bn', center=False, scale=False)(l)
# out
# l = GlobalAvgPool2D(name='avg_pool')(l)
l = MaxPooling2D(name='max_pool_final')(l)
l = Flatten(name='flatten_maxpool')(l)
l = Dense(768, name='fc1', trainable=trainable, kernel_initializer=init)(l)
l = Activation('selu', name='fc1_relu')(l)
l = Dense(256, name='fc2', trainable=trainable, kernel_initializer=init)(l)
l = Activation('selu', name='fc2_relu')(l)
l = Dense(class_num, name='fc3', trainable=trainable)(l)
outputs = l = Activation('softmax', name='out')(l)
return inputs, outputs
deact6 = Activation('relu')(deconv6)
decoder = Model(input=[code_lr], output=[deact6])
decoder.compile(optimizer=adam, loss='mse', metrics=['accuracy'])
codec_input = Input(shape=(im_size, im_size, 1),
dtype='float32',
name='encode_input')
codec_input_next = Input(shape=(im_size, im_size, 1),
dtype='float32',
name='encode_input_next')
decoder_input = encoder(codec_input)
decoder_input_next = encoder(codec_input_next)
codec_output = decoder(decoder_input)
negated = Lambda(lambda x: -x)(decoder_input_next)
code_diff = Add()([decoder_input, negated])
codec = Model(input=[codec_input, codec_input_next],
output=[codec_output, code_diff])
codec.compile(loss='mse', optimizer=adam)
codec_only = Model(input=[codec_input], output=[codec_output])
codec_only.compile(loss='mse', optimizer=adam)
train_images_np = np.reshape(np.asarray(train_images),
(-1, im_size, im_size, 1))
test_images_np = np.reshape(np.asarray(test_images),
(-1, im_size, im_size, 1))
nbEpoch = 50
batchSize = 1
numBatches = len(train_list) / batchSize - 1
def get_large_res_deform_cnn2(class_num, trainable=False):
inputs = l = Input((None, None, 3), name='input')
"""
Block 1
"""
l = Conv2D(32, (3, 3),
padding='same',
name='conv11',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv11_relu')(l)
l = BatchNormalization(name='conv11_bn')(l)
l2 = InvConv2D(32, (3, 3),
padding='same',
name='inv_conv11',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l2 = Activation('relu', name='inv_conv11_relu')(l2)
l2 = BatchNormalization(name='inv_conv11_bn')(l2)
l3 = Conv2D(32, (5, 5),
padding='same',
name='conv11_2',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l3 = Activation('relu', name='conv11_2_relu')(l3)
l3 = BatchNormalization(name='conv11_2_bn')(l3)
l5 = InvConv2D(32, (5, 5),
padding='same',
name='conv11_3',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l5 = Activation('relu', name='conv11_3_relu')(l5)
l5 = BatchNormalization(name='conv11_3_bn')(l5)
l = concatenate([l, l2, l3, l5])
# conv12
# l_offset = ConvOffset2D(32, name='conv12_offset')(l)
l = SeparableConv2D(128, (3, 3),
padding='same',
strides=(2, 2),
name='pool11_12',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='pool11_12_relu')(l)
# l = BatchNormalization(name='conv11_12_bn')(l)
"""
Block 2
"""
l3 = SeparableConv2D(128, (5, 3),
padding='same',
name='conv12_2',
trainable=trainable)(l)
l3 = Activation('relu', name='conv12_2_relu')(l3)
l3 = BatchNormalization(name='conv12_2_bn')(l3)
l5 = SeparableConv2D(128, (3, 5),
padding='same',
name='conv12_3',
trainable=trainable)(l)
l5 = Activation('relu', name='conv12_3_relu')(l5)
l5 = BatchNormalization(name='conv12_3_bn')(l5)
l = SeparableConv2D(128, (3, 3),
padding='same',
name='conv12',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv12_relu')(l)
l = BatchNormalization(name='conv12_bn')(l)
l = concatenate([l, l3, l5])
"""
Normal Convs
"""
l = Conv2D(192, (3, 3),
padding='same',
strides=(2, 2),
name='conv13',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv13_relu')(l)
l = BatchNormalization(name='conv13_bn')(l)
l = SeparableConv2D(192, (3, 3),
padding='same',
name='conv14',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv14_relu')(l)
l_res_h = BatchNormalization(name='conv14_bn')(l)
# l5 = SeparableConv2D(192, (5, 5), padding='same', name='conv14_5', trainable=trainable, depthwise_regularizer=OrthLocalRegSep2D)(l)
# l5 = Activation('relu', name='conv14_5_relu')(l5)
# l5 = BatchNormalization(name='conv14_5_bn')(l5)
l = SeparableConv2D(192, (3, 3),
padding='same',
name='conv21',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l_res_h)
l = Activation('relu', name='conv21_relu')(l)
l = BatchNormalization(name='conv21_bn')(l)
# l = SeparableConv2D(192, (3, 3), padding='same', name='conv22', trainable=trainable, depthwise_regularizer=OrthLocalRegSep2D)(l)
# l = Activation('relu', name='conv22_relu')(l)
# l = BatchNormalization(name='conv22_bn')(l)
l = Add(name='residual_21_23')([l_res_h, l])
# l = concatenate([l, l5])
# l5 = SeparableConv2D(384, (5, 5), padding='same', strides=(2, 2), name='conv23_5', trainable=trainable, depthwise_regularizer=OrthLocalRegSep2D)(l)
# l5 = Activation('relu', name='conv23_5_relu')(l5)
# l5 = BatchNormalization(name='conv23_5_bn')(l5)
l = SeparableConv2D(384, (3, 3),
padding='same',
strides=(2, 2),
name='conv23',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv23_relu')(l)
l = BatchNormalization(name='conv23_bn')(l)
l = SeparableConv2D(384, (3, 3),
padding='same',
name='conv31',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv31_relu')(l)
l_res_h = BatchNormalization(name='conv31_bn')(l)
l = SeparableConv2D(384, (3, 3),
padding='same',
name='conv32',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l_res_h)
l = Activation('relu', name='conv32_relu')(l)
l = BatchNormalization(name='conv32_bn')(l)
# l = SeparableConv2D(384, (3, 3), padding='same', name='conv33', trainable=trainable, depthwise_regularizer=OrthLocalRegSep2D)(l)
# l = Activation('relu', name='conv33_relu')(l)
# l = BatchNormalization(name='conv33_bn')(l)
# l = Add(name='residual_31_33')([l_res_h, l, l5])
# l = concatenate([l, l5])
l_offset = ConvOffset2D(384,
name='conv33_offset',
kernel_regularizer=OrthLocalReg2D)(l,
use_resam=True)
l = Conv2D(512, (3, 3),
padding='same',
name='conv41',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l_offset)
l = Activation('relu', name='conv41_relu')(l)
l = BatchNormalization(name='conv41_bn')(l)
l = SeparableConv2D(512, (3, 3),
padding='same',
strides=(2, 2),
depth_multiplier=1,
name='conv42',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv42_relu')(l)
l_res_h = BatchNormalization(name='conv42_bn')(l)
# l_offset = ConvOffset2D(512, name='conv35_offset')(l)
l = SeparableConv2D(512, (3, 3),
padding='same',
name='conv43',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l_res_h)
l = Activation('relu', name='conv43_relu')(l)
l = BatchNormalization(name='conv43_bn')(l)
# l = SeparableConv2D(512, (3, 3), padding='same', name='conv44', trainable=trainable, depthwise_regularizer=OrthLocalRegSep2D)(l)
# l = Activation('relu', name='conv44_relu')(l)
# l = BatchNormalization(name='conv44_bn')(l)
l = Add(name='residual_42_44')([l_res_h, l])
l = SeparableConv2D(1024, (3, 3),
padding='same',
strides=(2, 2),
name='conv51',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv51_relu')(l)
l_res_h = BatchNormalization(name='conv51_bn')(l)
l = SeparableConv2D(1024, (3, 3),
padding='same',
name='conv52',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l_res_h)
l = Activation('relu', name='conv52_relu')(l)
l = BatchNormalization(name='conv52_bn')(l)
# l = SeparableConv2D(1024, (3, 3), padding='same', name='conv53', trainable=trainable, depthwise_regularizer=OrthLocalRegSep2D)(l)
# l = Activation('relu', name='conv53_relu')(l)
# l = BatchNormalization(name='conv53_bn')(l)
# l = Add(name='residual_51_53')([l_res_h, l])
# out
l = SpatialPyramidPooling([1, 2, 4], input_shape=[None, None, 1024])(l)
# l = Flatten(name='flatten_maxpool')(l)
l = Dense(768,
name='fc1',
trainable=trainable,
kernel_regularizer=OrthLocalReg1D)(l)
l = Activation('relu', name='fc1_relu')(l)
l = Dense(256,
name='fc2',
trainable=trainable,
kernel_regularizer=OrthLocalReg1D)(l)
l = Activation('relu', name='fc2_relu')(l)
outputs_1 = Dense(class_num, name='fc3', trainable=trainable)(l)
outputs_1 = Activation('softmax', name='out')(outputs_1)
outputs_2 = Dense(2, name='fc3_super', trainable=trainable)(l)
outputs_2 = Activation('softmax')(outputs_2)
# outputs = concatenate([outputs_1, outputs_2], name='out')
outputs = [outputs_1, outputs_2]
return inputs, outputs
def get_large_res_deform_cnn(class_num, trainable=False):
inputs = l = Input((200, 200, 3), name='input')
"""
Block 1
"""
l = Conv2D(32, (3, 3),
padding='same',
name='conv11',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv11_relu')(l)
l = BatchNormalization(name='conv11_bn')(l)
l2 = InvConv2D(32, (3, 3),
padding='same',
name='inv_conv11',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l2 = Activation('relu', name='inv_conv11_relu')(l2)
l2 = BatchNormalization(name='inv_conv11_bn')(l2)
l3 = Conv2D(32, (3, 1),
padding='same',
name='conv11_2',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l3 = Activation('relu', name='conv11_2_relu')(l3)
l3 = BatchNormalization(name='conv11_2_bn')(l3)
l5 = Conv2D(32, (1, 3),
padding='same',
name='conv11_3',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l5 = Activation('relu', name='conv11_3_relu')(l5)
l5 = BatchNormalization(name='conv11_3_bn')(l5)
l = concatenate([l, l2, l3, l5])
# conv12
# l_offset = ConvOffset2D(32, name='conv12_offset')(l)
l = SeparableConv2D(128, (3, 3),
padding='same',
strides=(2, 2),
name='pool11_12',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='pool11_12_relu')(l)
# l = BatchNormalization(name='conv11_12_bn')(l)
"""
Block 2
"""
l3 = SeparableConv2D(128, (3, 1),
padding='same',
name='conv12_2',
trainable=trainable)(l)
l3 = Activation('relu', name='conv12_2_relu')(l3)
l3 = BatchNormalization(name='conv12_2_bn')(l3)
l5 = SeparableConv2D(128, (1, 3),
padding='same',
name='conv12_3',
trainable=trainable)(l)
l5 = Activation('relu', name='conv12_3_relu')(l5)
l5 = BatchNormalization(name='conv12_3_bn')(l5)
l = SeparableConv2D(128, (3, 3),
padding='same',
name='conv12',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv12_relu')(l)
l = BatchNormalization(name='conv12_bn')(l)
l = concatenate([l, l3, l5])
"""
Normal Convs
"""
l = Conv2D(192, (3, 3),
padding='same',
name='conv13',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv13_relu')(l)
l = BatchNormalization(name='conv13_bn')(l)
l = SeparableConv2D(192, (3, 3),
padding='same',
strides=(2, 2),
name='conv14',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv14_relu')(l)
# l = BatchNormalization(name='conv14_bn')(l)
l = SeparableConv2D(192, (3, 3),
padding='same',
name='conv21',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv21_relu')(l)
l_res_h = BatchNormalization(name='conv21_bn')(l)
l = SeparableConv2D(192, (3, 3),
padding='same',
name='conv22',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l_res_h)
l = Activation('relu', name='conv22_relu')(l)
l = BatchNormalization(name='conv22_bn')(l)
l = Add(name='residual_21_23')([l_res_h, l])
# conv22
# l_offset = ConvOffset2D(192, name='conv32_offset')(l)
l = SeparableConv2D(192, (3, 3),
padding='same',
strides=(2, 2),
name='conv23',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv23_relu')(l)
l = BatchNormalization(name='conv23_bn')(l)
l = SeparableConv2D(256, (3, 3),
padding='same',
name='conv31',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv31_relu')(l)
l_res_h = BatchNormalization(name='conv31_bn')(l)
l = SeparableConv2D(256, (3, 3),
padding='same',
name='conv32',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l_res_h)
l = Activation('relu', name='conv32_relu')(l)
l = BatchNormalization(name='conv32_bn')(l)
l = SeparableConv2D(256, (3, 3),
padding='same',
name='conv33',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv33_relu')(l)
l = BatchNormalization(name='conv33_bn')(l)
l = Add(name='residual_31_33')([l_res_h, l])
l_offset = ConvOffset2D(256, name='conv33_offset')(l)
l = Conv2D(256, (3, 3), padding='same', name='conv41',
trainable=trainable)(l_offset)
l = Activation('relu', name='conv41_relu')(l)
l = BatchNormalization(name='conv41_bn')(l)
l = SeparableConv2D(512, (3, 3),
padding='same',
strides=(2, 2),
depth_multiplier=2,
name='conv42',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv42_relu')(l)
l_res_h = BatchNormalization(name='conv42_bn')(l)
# l_offset = ConvOffset2D(512, name='conv35_offset')(l)
l = SeparableConv2D(512, (3, 3),
padding='same',
name='conv43',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l_res_h)
l = Activation('relu', name='conv43_relu')(l)
l = BatchNormalization(name='conv43_bn')(l)
l = SeparableConv2D(512, (3, 3),
padding='same',
name='conv44',
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(l)
l = Activation('relu', name='conv44_relu')(l)
l = BatchNormalization(name='conv44_bn')(l)
l = Add(name='residual_42_44')([l_res_h, l])
# out
# l = GlobalAvgPool2D(name='avg_pool')(l)
# l = MaxPooling2D(name='max_pool_final')(l)
# l = SeparableConv2D(512, (3, 3), padding='same', name='conv51', trainable=trainable, kernel_regularizer=OrthLocalRegSep2D)(l)
# l = Activation('relu', name='conv51_relu')(l)
# l = BatchNormalization(name='conv51_bn')(l)
l = SpatialPyramidPooling([1, 2, 4], input_shape=[None, None, 512])(l)
# l = Flatten(name='flatten_maxpool')(l)
l = Dense(768,
name='fc1',
trainable=trainable,
kernel_regularizer=OrthLocalReg1D)(l)
l = Activation('relu', name='fc1_relu')(l)
l = Dense(256, name='fc2', trainable=trainable)(l)
l = Activation('relu', name='fc2_relu')(l)
l = Dense(class_num, name='fc3', trainable=trainable)(l)
outputs = l = Activation('softmax', name='out')(l)
return inputs, outputs
def FCN_8s_pretrained(classes, in_shape, l2_reg, nopad=False, test=False):
vgg16 = VGG16(include_top=False, weights="imagenet", input_shape=in_shape)
inputs = Input(shape=in_shape)
if nopad:
x = inputs
else:
x = ZeroPadding2D(padding=(100, 100))(inputs)
for layer in vgg16.layers[1:]:
if "conv" in layer.name:
W, b = layer.get_weights()
config = layer.get_config()
config["kernel_regularizer"] = l2(l2_reg)
config["kernel_initializer"] = Constant(W)
config["bias_initializer"] = Constant(b)
conv = Conv2D.from_config(config)
x = conv(x)
elif "pool" in layer.name:
x = MaxPooling2D()(x)
if layer.name == "block3_pool":
feat3 = x
elif layer.name == "block4_pool":
feat4 = x
x = Conv2D(filters=4096,
kernel_size=(7, 7),
padding="valid",
activation="relu",
kernel_regularizer=l2(l2_reg),
name="fc1")(x)
x = Dropout(0.5)(x)
x = Conv2D(filters=4096,
kernel_size=(1, 1),
padding="valid",
activation="relu",
kernel_regularizer=l2(l2_reg),
name="fc2")(x)
x = Dropout(0.5)(x)
score5 = Conv2D(filters=classes,
kernel_size=(1, 1),
kernel_regularizer=l2(l2_reg),
activation="relu")(x)
if nopad:
score5 = Conv2DTranspose(filters=classes,
kernel_size=(14, 14),
strides=(1, 1),
padding="valid",
activation="linear",
kernel_regularizer=l2(l2_reg),
kernel_initializer=Constant(
bilinear_upsample_weights(
"full", classes)
))(score5)
else:
score5 = Conv2DTranspose(filters=classes,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
activation="linear",
kernel_regularizer=l2(l2_reg),
kernel_initializer=Constant(
bilinear_upsample_weights(2, classes)
))(score5)
# pool3 のfeature mapを次元圧縮
score3 = Conv2D(filters=classes,
kernel_size=(1, 1),
kernel_regularizer=l2(l2_reg),
activation='relu')(feat3)
# pool4のfeature mapを次元圧縮
score4 = Conv2D(filters=classes,
kernel_size=(1, 1),
kernel_regularizer=l2(l2_reg),
activation="relu")(feat4)
# merge p4 and p5
score4 = CroppingLike2D(K.int_shape(score5))(score4)
score45 = Add()([score4, score5])
# p4+p5 を x2 upsampling
if not nopad:
score45 = ZeroPadding2D(padding=(1, 1))(score45)
score45 = Conv2DTranspose(filters=classes,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
activation="linear",
kernel_regularizer=l2(l2_reg),
kernel_initializer=Constant(
bilinear_upsample_weights(2, classes)
))(score45)
# p3とp45をmerge
score3 = CroppingLike2D(K.int_shape(score45))(score3)
score345 = Add()([score3, score45])
# p3+p4+p5を x8 upsampling
if not nopad:
score345 = ZeroPadding2D(padding=(1, 1))(score345)
x = Conv2DTranspose(filters=classes,
kernel_size=(16, 16),
strides=(8, 8),
padding="same",
activation="linear",
kernel_regularizer=l2(l2_reg),
kernel_initializer=Constant(
bilinear_upsample_weights(8, classes)
))(score345)
x = CroppingLike2D(K.int_shape(inputs))(x)
if test:
x = Activation("softmax")(x)
model = Model(inputs=inputs, outputs=x)
return model
def FCN_8s(classes, in_shape, l2_reg, nopad=False, test=False):
"""
VGG16 based FCN model,
classes: int, number of classes
return: keras Model object
"""
inputs = Input(shape=in_shape)
if nopad:
x = inputs
else:
x = ZeroPadding2D(padding=(100, 100))(inputs)
x = Conv2D(filters=64,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Conv2D(filters=64,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = MaxPooling2D()(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = MaxPooling2D()(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = MaxPooling2D()(x)
# pool3のfeature mapを取得
p3 = x
x = Conv2D(filters=512,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = MaxPooling2D()(x)
# pool4のfeature mapを取得
p4 = x
x = Conv2D(filters=512,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
padding="same",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = MaxPooling2D()(x)
x = Conv2D(filters=4096,
kernel_size=(7, 7),
padding="valid",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
x = Conv2D(filters=4096,
kernel_size=(1, 1),
padding="valid",
kernel_regularizer=l2(l2_reg))(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
score_p5 = Conv2D(filters=classes,
kernel_size=(1, 1),
kernel_regularizer=l2(l2_reg),
activation="relu")(x)
if nopad:
score_p5 = Conv2DTranspose(filters=classes,
kernel_size=(14, 14),
strides=(1, 1),
padding="valid",
activation="linear",
kernel_regularizer=l2(l2_reg),
kernel_initializer=Constant(
bilinear_upsample_weights(
"full", classes)
))(score_p5)
else:
score_p5 = Conv2DTranspose(filters=classes,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
activation="linear",
kernel_regularizer=l2(l2_reg),
kernel_initializer=Constant(
bilinear_upsample_weights(2, classes)
))(score_p5)
# pool3 のfeature mapを次元圧縮
score_p3 = Conv2D(filters=classes,
kernel_size=(1, 1),
kernel_regularizer=l2(l2_reg),
activation='relu')(p3)
# pool4のfeature mapを次元圧縮
score_p4 = Conv2D(filters=classes,
kernel_size=(1, 1),
kernel_regularizer=l2(l2_reg),
activation="relu")(p4)
# merge p4 and p5
score_p4 = CroppingLike2D(K.int_shape(score_p5))(score_p4)
score_p45 = Add()([score_p4, score_p5])
# p4+p5 を x2 upsampling
if not nopad:
score_p45 = ZeroPadding2D(padding=(1, 1))(score_p45)
score_p45 = Conv2DTranspose(filters=classes,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
activation="linear",
kernel_regularizer=l2(l2_reg),
kernel_initializer=Constant(
bilinear_upsample_weights(2, classes)
))(score_p45)
# p3とp45をmerge
score_p3 = CroppingLike2D(K.int_shape(score_p45))(score_p3)
score_p345 = Add()([score_p3, score_p45])
# p3+p4+p5を x8 upsampling
if not nopad:
score_p345 = ZeroPadding2D(padding=(1, 1))(score_p345)
x = Conv2DTranspose(filters=classes,
kernel_size=(16, 16),
strides=(8, 8),
padding="same",
activation="linear",
kernel_regularizer=l2(l2_reg),
kernel_initializer=Constant(
bilinear_upsample_weights(8, classes)
))(score_p345)
x = CroppingLike2D(K.int_shape(inputs))(x)
if test:
x = Activation("softmax")(x)
model = Model(inputs=inputs, outputs=x)
return model
def loupe_model(input_shape=(256, 256, 1),
filt=64,
kern=3,
sparsity=None,
pmask_slope=5,
pmask_init=None,
sample_slope=12,
acti=None,
model_type='v2'):
"""
loupe_model
Parameters:
input_shape: input shape
filt: number of base filters
kern: kernel size
model_type: 'unet', 'v1', 'v2'
sparsity: desired sparsity (only for model type 'v2')
pmask_slope: slope of logistic parameter in probability mask
acti=None: activation
Returns:
keras model
UNet leaky two channel
"""
if model_type == 'unet': # Creates a single U-Net
inputs = Input(shape=input_shape, name='input')
last_tensor = inputs
elif model_type == 'loupe_mask_input':
# inputs
inputs = [
Input(shape=input_shape, name='input'),
Input(shape=input_shape, name='mask_input')
]
last_tensor = inputs[0]
# if necessary, concatenate with zeros for FFT
if input_shape[-1] == 1:
last_tensor = layers.ConcatenateZero(
name='concat_zero')(last_tensor)
# input -> kspace via FFT
last_tensor = layers.FFT(name='fft')(last_tensor)
# input mask
last_tensor_mask = inputs[1]
# Under-sample and back to image space via IFFT
last_tensor = layers.UnderSample(name='under_sample_kspace')(
[last_tensor, last_tensor_mask])
last_tensor = layers.IFFT(name='under_sample_img')(last_tensor)
else: # Creates LOUPE
assert model_type in ['v1',
'v2'], 'model_type should be unet, v1 or v2'
# inputs
inputs = Input(shape=input_shape, name='input')
last_tensor = inputs
# if necessary, concatenate with zeros for FFT
if input_shape[-1] == 1:
last_tensor = layers.ConcatenateZero(
name='concat_zero')(last_tensor)
# input -> kspace via FFT
last_tensor = layers.FFT(name='fft')(last_tensor)
# build probability mask
prob_mask_tensor = layers.ProbMask(name='prob_mask',
slope=pmask_slope,
initializer=pmask_init)(last_tensor)
if model_type == 'v2':
assert sparsity is not None, 'for this model, need desired sparsity to be specified'
prob_mask_tensor = layers.RescaleProbMap(
sparsity, name='prob_mask_scaled')(prob_mask_tensor)
else:
assert sparsity is None, 'for v1 model, cannot specify sparsity'
# Realization of probability mask
thresh_tensor = layers.RandomMask(name='random_mask')(prob_mask_tensor)
last_tensor_mask = layers.ThresholdRandomMask(
slope=sample_slope,
name='sampled_mask')([prob_mask_tensor, thresh_tensor])
# Under-sample and back to image space via IFFT
last_tensor = layers.UnderSample(name='under_sample_kspace')(
[last_tensor, last_tensor_mask])
last_tensor = layers.IFFT(name='under_sample_img')(last_tensor)
# hard-coded UNet
unet_tensor = _unet_from_tensor(last_tensor,
filt,
kern,
acti,
output_nb_feats=input_shape[-1])
# complex absolute layer
abs_tensor = layers.ComplexAbs(name='complex_addition')(last_tensor)
# final output from model
add_tensor = Add(name='unet_output')([abs_tensor, unet_tensor])
# prepare and output a model as necessary
outputs = [add_tensor]
if model_type == 'v1':
outputs += [last_tensor_mask]
return keras.models.Model(inputs, outputs)
def main():
print('============================================================')
print('Read Data')
movies, all_genres = read_movie(DATA_DIR + '/movies.csv')
genders, ages, occupations = read_user(DATA_DIR + '/users.csv')
train = read_train(DATA_DIR + '/train.csv')
print('============================================================')
print('Preprocess Data')
userID, movieID, userGender, userAge, userOccu, movieGenre, Y = \
preprocess(train, genders, ages, occupations, movies)
n_users = np.max(userID) + 1
n_movies = np.max(movieID) + 1
print('============================================================')
print('Construct Model')
EMB_DIM = 128
print('Embedding Dimension:', EMB_DIM)
# inputs
in_userID = Input(shape=(1, )) # user id
in_movieID = Input(shape=(1, )) # movie id
# embeddings
emb_userID = Embedding(n_users, EMB_DIM)(in_userID)
emb_movieID = Embedding(n_movies, EMB_DIM)(in_movieID)
vec_userID = Dropout(0.5)(Flatten()(emb_userID))
vec_movieID = Dropout(0.5)(Flatten()(emb_movieID))
# dot
dot1 = Dot(axes=1)([vec_userID, vec_movieID])
# bias
emb2_userID = Embedding(n_users, 1,
embeddings_initializer='zeros')(in_userID)
emb2_movieID = Embedding(n_movies, 1,
embeddings_initializer='zeros')(in_movieID)
bias_userID = Flatten()(emb2_userID)
bias_movieID = Flatten()(emb2_movieID)
# output
out = Add()([bias_userID, bias_movieID, dot1])
# model (userID * movieID only)
model = Model(inputs=[in_userID, in_movieID], outputs=out)
model.summary()
def rmse(y_true, y_pred):
return K.sqrt(K.mean((y_pred - y_true)**2))
model.compile(optimizer='adam', loss='mse', metrics=[rmse])
print('============================================================')
print('Train Model')
es = EarlyStopping(monitor='val_rmse', patience=30, verbose=1, mode='min')
cp = ModelCheckpoint(monitor='val_rmse', save_best_only=True, save_weights_only=False, \
mode='min', filepath='mf_simple_model.h5')
history = model.fit([userID, movieID], Y, \
epochs=500, verbose=1, batch_size=10000, callbacks=[es, cp], \
validation_split=0.05)
H = history.history
best_val = str(round(np.min(H['val_rmse']), 6))
print('Best Val RMSE:', best_val)
print('============================================================')
print('Test Model')
model = load_model('mf_simple_model.h5', custom_objects={'rmse': rmse})
test = read_test(DATA_DIR + '/test.csv')
ID = np.array(test[:, 0]).reshape(-1, 1)
print('Test data len:', len(test))
userID, movieID, userGender, userAge, userOccu, movieGenre, _Y = \
preprocess(test, genders, ages, occupations, movies)
result = model.predict([userID, movieID])
print('Output Result')
rating = np.clip(result, 1, 5).reshape(-1, 1)
output = np.array(np.concatenate((ID, rating), axis=1))
print('============================================================')
print('Save Result')
write_result(PRED_DIR + '/mf_simple_bias.csv', output)
np.savez(HIS_DIR + '/mf_simple_' + best_val + '_his.npz',
rmse=H['rmse'],
val_rmse=H['val_rmse'])
os.rename('mf_simple_model.h5',
MODEL_DIR + '/mf_simple_' + best_val + '.h5')
def resblock(x, kernel_size, resample, nfilters, norm=BatchNormalization):
assert resample in ["UP", "SAME", "DOWN"]
if resample == "UP":
shortcut = UpSampling2D(size=(2, 2))(x)
shortcut = Conv2D(nfilters,
kernel_size,
padding='same',
kernel_initializer='he_uniform',
use_bias=True)(shortcut)
convpath = norm()(x)
convpath = Activation('relu')(convpath)
convpath = UpSampling2D(size=(2, 2))(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=False,
padding='same')(convpath)
convpath = norm()(convpath)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=True,
padding='same')(convpath)
y = Add()([shortcut, convpath])
elif resample == "SAME":
shortcut = Conv2D(nfilters,
kernel_size,
padding='same',
kernel_initializer='he_uniform',
use_bias=True)(x)
convpath = norm()(x)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=False,
padding='same')(convpath)
convpath = norm()(convpath)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=True,
padding='same')(convpath)
y = Add()([shortcut, convpath])
else:
shortcut = AveragePooling2D(pool_size=(2, 2))(x)
shortcut = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
padding='same',
use_bias=True)(shortcut)
convpath = norm()(x)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=False,
padding='same')(convpath)
convpath = AveragePooling2D(pool_size=(2, 2))(convpath)
convpath = norm()(convpath)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=True,
padding='same')(convpath)
y = Add()([shortcut, convpath])
return y
def segnet_res(nClasses, optimizer=None, input_height=360, input_width=480):
kernel = 3
filter_size = 64
pad = 1
pool_size = 2
img_input = Input(shape=(input_height, input_width, 3))
# encoder
x = ZeroPadding2D(padding=(pad, pad))(img_input)
x = Convolution2D(filter_size, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
l1 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(128, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
l2 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(256, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
l3 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(512, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
l4 = x
x = Activation('relu')(x)
# decoder
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(512, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l4, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(256, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l3, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(128, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l2, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(filter_size, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l1, x])
x = Convolution2D(nClasses, (1, 1), padding='valid')(x)
out = x
a = Model(inputs=img_input, outputs=out)
model = []
a.outputHeight = a.output_shape[1]
a.outputWidth = a.output_shape[2]
out = Reshape((a.outputHeight * a.outputWidth, nClasses),
input_shape=(nClasses, a.outputHeight, a.outputWidth))(out)
out = Activation('softmax')(out)
model = Model(inputs=img_input, outputs=out)
model.outputHeight = a.outputHeight
model.outputWidth = a.outputWidth
return model
def resunet(input_shape,
n_filters,
loss_function,
optimizer_function,
batch_norm=False):
# Convolutional block: BN -> ReLU -> Conv3x3
def conv_block(inputs,
n_filters,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
batch_norm=True,
padding='same'):
if batch_norm:
x = BatchNormalization()(inputs)
else:
x = inputs
if activation:
x = Activation('relu')(x)
x = Conv2D(filters=n_filters,
kernel_size=kernel_size,
strides=strides,
padding=padding)(x)
return x
inputs = Input((*input_shape, 1))
# Encoding
short1 = inputs
conv1 = conv_block(inputs, n_filters, activation=None, batch_norm=False)
conv1 = conv_block(conv1, n_filters)
short1 = conv_block(short1, n_filters, activation=None)
conv1 = Add()([conv1, short1])
short2 = conv1
conv2 = conv_block(conv1, n_filters * 2, strides=(2, 2))
conv2 = conv_block(conv2, n_filters * 2)
short2 = conv_block(short2, n_filters * 2, strides=(2, 2), activation=None)
conv2 = Add()([conv2, short2])
short3 = conv2
conv3 = conv_block(conv2, n_filters * 4, strides=(2, 2))
conv3 = conv_block(conv3, n_filters * 4)
short3 = conv_block(short3, n_filters * 4, strides=(2, 2), activation=None)
conv3 = Add()([conv3, short3])
# Bridge
short4 = conv3
conv4 = conv_block(conv3, n_filters * 8, strides=(2, 2))
conv4 = conv_block(conv4, n_filters * 8)
short4 = conv_block(short4, n_filters * 8, strides=(2, 2), activation=None)
conv4 = Add()([conv4, short4])
# Decoding
up5 = Conv2DTranspose(filters=n_filters * 4,
kernel_size=(3, 3),
strides=(2, 2),
padding='same')(conv4)
up5 = concatenate([up5, conv3])
short5 = up5
conv5 = conv_block(up5, n_filters * 4)
conv5 = conv_block(conv5, n_filters * 4)
short5 = conv_block(short5, n_filters * 4, activation=None)
conv5 = Add()([conv5, short5])
up6 = Conv2DTranspose(filters=n_filters * 2,
kernel_size=(3, 3),
strides=(2, 2),
padding='same')(conv5)
up6 = concatenate([up6, conv2])
short6 = up6
conv6 = conv_block(up6, n_filters * 2)
conv6 = conv_block(conv6, n_filters * 2)
short6 = conv_block(short6, n_filters * 2, activation=None)
conv6 = Add()([conv6, short6])
up7 = Conv2DTranspose(filters=n_filters,
kernel_size=(3, 3),
strides=(2, 2),
padding='same')(conv6)
up7 = concatenate([up7, conv1])
short7 = up7
conv7 = conv_block(up7, n_filters)
conv7 = conv_block(conv7, n_filters)
short7 = conv_block(short7, n_filters, activation=None)
conv7 = Add()([conv7, short7])
outputs = Conv2D(filters=1, kernel_size=(1, 1),
activation='sigmoid')(conv7)
model = Model(inputs=[inputs], outputs=[outputs], name='ResUnet')
model.compile(optimizer=optimizer_function,
loss=loss_function,
metrics=['accuracy'])
return model
def two_way_interactions(self, collapsed_type, deep_out,
deep_kernel_constraint):
# Calculate interactions with a dot product
inputs = [None] * len(self.feature_names)
biases = [None] * len(self.feature_names)
factors = [None] * len(self.feature_names)
interactions = []
for i, groups in enumerate(self.deep_weight_groups):
for grp, (feature_i, feature_j) in enumerate(groups):
#factor_i = factors[self.feature_names.index(feature_i)]
index_i = self.feature_names.index(feature_i)
factor_i = self.build_variables(index_i, inputs, biases,
factors)
if isinstance(feature_j, str) or isinstance(
feature_j, unicode):
#factor_j = factors[ self.feature_names.index(feature_j) ]
index_j = self.feature_names.index(feature_j)
factor_j = self.build_variables(index_j, inputs, biases,
factors)
name_j = feature_j
elif isinstance(feature_j, list):
name_j = "grp_{}".format("{:03d}".format(i))
if collapsed_type is not None:
#constant = np.zeros( (self.embedding_dimensions,) )
#k_constant = K.constant(constant, shape=(self.embedding_dimensions,))
collapsed_input = Input(shape=(1, ),
name="input_{}".format(name_j))
embedding = Embedding(
input_dim=collapsed_type,
name="embedding_{}".format(name_j),
output_dim=self.embedding_dimensions)(
collapsed_input)
inputs.append(collapsed_input)
factor_j = embedding
else:
factors_j = []
for feature_j_n in feature_j:
#factor = factors[ self.feature_names.index(feature_j_n) ]
index_j_n = self.feature_names.index(feature_j_n)
factor = self.build_variables(
index_j_n, inputs, biases, factors)
if deep_out:
if self.dropout_layer > 0:
factor = Dropout(
self.dropout_layer,
name="dropout_terms_{}_{}".format(
feature_j_n, i))(factor)
factor = Scaler(
name="scaler_{}_{}".format(feature_j_n, i),
constraint=deep_kernel_constraint)(factor)
factors_j.append(factor)
factor_j = Add(name=name_j)(factors_j) # collapse them
if factor_i is None:
print("Warning... {} does not have an embedding".format(
feature_i))
continue
if factor_j is None:
print("Warning... {} does not have an embedding".format(
feature_j))
continue
dot_product = Dot(axes=-1,
name="interaction_{}X{}".format(
feature_i, name_j))
interactions.append(dot_product([factor_i, factor_j]))
# Add whatever you have now:
if len(interactions) == 0:
two_way = None
elif len(interactions) == 1:
two_way = interactions[0:]
else:
two_way = Add(name="factors_term")(interactions)
return inputs, biases, two_way
#Dliated Block1 (DB1)
conv4 = BatchNormalization(momentum=0.99)(pool3)
conv4_1 = Conv2D(256,
3,
activation='relu',
padding='same',
dilation_rate=(2, 2),
kernel_initializer='he_normal')(conv4)
conv4 = BatchNormalization(momentum=0.99)(conv4_1)
conv4_2 = Conv2D(256,
1,
activation='relu',
padding='same',
dilation_rate=(2, 2),
kernel_initializer='he_normal')(conv4)
conv4 = Add()([conv4_1, conv4_2]) #Feature reuse module
conv4 = Dropout(0.02)(conv4)
#Dliated Block2 (DB2)
conv5 = BatchNormalization(momentum=0.99)(conv4)
conv5_1 = Conv2D(512,
3,
activation='relu',
padding='same',
dilation_rate=(4, 4),
kernel_initializer='he_normal')(conv5)
conv5 = BatchNormalization(momentum=0.99)(conv5_1)
conv5_2 = Conv2D(512,
1,
activation='relu',
padding='same',
def build_model(self,
embedding_dimensions,
collapsed_type=None,
l2_bias=0.0,
l2_factors=0.0,
l2_deep=0.0,
deep_out=True,
bias_only=None,
embeddings_only=None,
deep_weight_groups=None,
deep_out_bias=True,
deep_out_activation='linear',
deep_kernel_constraint=None,
dropout_input=0.,
dropout_layer=0.,
**kwargs):
"""
Builds the FM model in Keras Network
:param embedding_dimensions: The number of dimensions of the embeddings
:param collapsed_type: None if the model shouldn't collapsed, or # of embedding types if it should
:param l2_bias: L2 regularization for bias terms
:param l2_factors: L2 regularization for interaction terms
:param l2_deep: L2 regularization for the deep layer
:param deep_out: Whether to have a fully connected "deep" layer to weight the factors
:param deep_out_bias: whether to use bias terms in the "deep" intermediate layer. Only applies if the deep=True
:paran deep_out_activation: the activation function for the "deep" intermediate layer. Only applies if the deep=True.
should be a keyword keras recognizes
:param deep_kernel_constraint whether to have a constraint on the deep layer
:param weight_groups: an ordered list of integers representing weight groups for each feature; these correspond to the
t() function in the DeepFM paper(section II.B.1). Default is each input feature gets there own unique weight for the deep layer.
NOTE: my current implementation , for dropout in the deep layer, when using weight groups, will equally weight each unique weight group
cross-sectionm rather than individual interactions.
:param bias_only: a list of booleans of length # features that determines whether to only use the bias and not allow the designated
feature to be interacted
:param dropout_input: Dropout rate for the features,
:param dropout_layer: Dropout rate for the factors entering the deep layer. Only applies if the deep=True
:param
:param kwargs: any additional arguments passed directly to the Model Keras class initializer
Returns a Keras model
"""
self.embedding_dimensions = embedding_dimensions
self.check_build_params(l2_bias, l2_factors, l2_deep, bias_only,
embeddings_only, deep_weight_groups,
deep_out_bias, deep_out_activation,
dropout_input, dropout_layer)
features, biases, factors = self.two_way_interactions(
collapsed_type, deep_out, deep_kernel_constraint)
features = [f for f in features if f is not None]
biases = [f for f in biases if f is not None]
#1-way interactions:
if len(biases) == 0:
bias_term = None
elif len(biases) == 1:
bias_term = biases[0]
else:
bias_term = Add(name="biases_terms")(biases)
#2-way interactions
# interact embedding layers corresponding to indiv. features together via dot-product
# Combine 1-way and 2-way interactions:
if (bias_term is not None) and (factors is not None):
output_layer = Add(name="output_layer")([bias_term, factors])
elif factors is not None:
output_layer = factors
else:
output_layer = bias_term # Lambda(lambda x: K.sum(x, axis=-1, keepdims=True), name="output_layer")(biases)
assert output_layer is not None
if self.obj == 'ns':
output = Reshape(
(1, ),
name='final_output')(Activation('sigmoid',
name="sigmoid")(output_layer))
elif self.obj == 'nce':
noise_probs = Input(batch_shape=(None, 1), name='noise_probs')
features.append(noise_probs)
#noise_mult_tensor = K.constant([self.noise_multiplier],name='noise_mult_tens')
#noise_multiplier = Input(batch_shape=(None,1),name='noise_multiplier',tensor=noise_mult_tensor)
output = Lambda(NCEobj,
name='nce_obj')([output_layer, noise_probs])
global graph
graph = tf.get_default_graph()
return Model(inputs=features,
outputs=output,
name="Factorization Machine",
**kwargs)
def FCN_8s(input_shape,
n_class,
encoder_name,
encoder_weights=None,
fc_num=4096,
weight_decay=1e-4,
kernel_initializer="he_normal",
bn_epsilon=1e-3,
bn_momentum=0.99,
dropout=0.5):
""" implementation of FCN-8s for semantic segmentation.
ref: Long J, Shelhamer E, Darrell T. Fully Convolutional Networks for Semantic Segmentation[J].
arXiv preprint arXiv:1411.4038, 2014.
:param input_shape: tuple, i.e., (height, width, channel).
:param n_class: int, number of class, must >= 2.
:param encoder_name: string, name of encoder.
:param encoder_weights: string, path of weights, default None.
:param fc_num: int, number of filters of fully convolutions, default 4096.
:param weight_decay: float, default 1e-4.
:param kernel_initializer: string, default "he_normal".
:param bn_epsilon: float, default 1e-3.
:param bn_momentum: float, default 0.99.
:param dropout: float, default 0.5.
:return: a Keras Model instance.
"""
# adapted from https://github.com/shelhamer/fcn.berkeleyvision.org/blob/master/voc-fcn8s/net.py
encoder = build_encoder(input_shape=input_shape,
encoder_name=encoder_name,
encoder_weights=encoder_weights,
weight_decay=weight_decay,
kernel_initializer=kernel_initializer,
bn_epsilon=bn_epsilon,
bn_momentum=bn_momentum)
p3 = encoder.get_layer(scope_table["pool3"]).output
p4 = encoder.get_layer(scope_table["pool4"]).output
p5 = encoder.get_layer(scope_table["pool5"]).output
# # # 1. merge pool5 & pool4
# upsample prediction from pool5
x1 = Conv2D(fc_num, (7, 7),
padding="same",
activation="relu",
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(p5)
x1 = Dropout(dropout)(x1)
x1 = Conv2D(fc_num, (1, 1),
padding="same",
activation="relu",
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(x1)
x1 = Dropout(dropout)(x1)
x1 = Conv2D(n_class, (1, 1),
padding="same",
activation=None,
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(x1)
x1 = Conv2DTranspose(n_class, (4, 4),
strides=(2, 2),
use_bias=False,
activation=None,
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(x1)
# upsample prediction from pool4
x2 = Conv2D(n_class, (1, 1),
padding="same",
activation=None,
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(p4)
x1 = Add()([x1, x2])
# # # 2. merge pool4 & pool3
x1 = Conv2DTranspose(n_class, (4, 4),
strides=(2, 2),
use_bias=False,
Activation=None,
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(x1)
x2 = Conv2D(n_class, (1, 1),
padding="same",
activation=None,
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(p3)
x1 = Add()([x1, x2])
# # # 3. upsample and predict
x1 = Conv2DTranspose(n_class, (16, 16),
strides=(8, 8),
use_bias=False,
activation=None,
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(x1)
output = Activation("softmax")(x1)
fcn_8s_model = Model(encoder.input, output)
return fcn_8s_model
def get_large_deform_inv_cnn(class_num, trainable=False):
inputs = l = Input((200, 200, 3), name='input')
# conv11
l = Conv2D(32, (3, 3),
padding='same',
name='conv11',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv11_relu')(l)
l = BatchNormalization(name='conv11_bn')(l)
l2 = InvConv2D(32, (3, 3),
padding='same',
name='inv_conv11',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l2 = Activation('relu', name='inv_conv11_relu')(l2)
l2 = BatchNormalization(name='inv_conv11_bn')(l2)
l3 = Conv2D(32, (3, 5),
padding='same',
name='conv11_2',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l3 = Activation('relu', name='conv11_2_relu')(l3)
l3 = BatchNormalization(name='conv11_2_bn')(l3)
l5 = Conv2D(32, (5, 3),
padding='same',
name='conv11_3',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(inputs)
l5 = Activation('relu', name='conv11_3_relu')(l5)
l5 = BatchNormalization(name='conv11_3_bn')(l5)
l = concatenate([l, l2, l3, l5])
# conv12
# l_offset = ConvOffset2D(32, name='conv12_offset')(l)
l = Conv2D(128, (3, 3),
padding='same',
strides=(2, 2),
name='pool11_12',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='pool11_12_relu')(l)
# l = BatchNormalization(name='conv12_bn')(l)
l3 = Conv2D(32, (3, 5),
padding='same',
name='conv12_2',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l3 = Activation('relu', name='conv12_2_relu')(l3)
l3 = BatchNormalization(name='conv12_2_bn')(l3)
l5 = Conv2D(32, (5, 3),
padding='same',
name='conv12_3',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l5 = Activation('relu', name='conv12_3_relu')(l5)
l5 = BatchNormalization(name='conv12_3_bn')(l5)
l = Conv2D(128, (3, 3),
padding='same',
name='conv12',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv12_relu')(l)
l = BatchNormalization(name='conv12_bn')(l)
l_block2 = l
l = concatenate([l, l3, l5])
l = Conv2D(128, (3, 3),
padding='same',
name='conv13',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv13_relu')(l)
l = BatchNormalization(name='conv13_bn')(l)
l = Conv2D(128, (3, 3),
padding='same',
strides=(2, 2),
name='conv14',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv14_relu')(l)
l = BatchNormalization(name='conv14_bn')(l)
# l = Conv2D(192, (3, 3), padding='same', name='conv21', trainable=trainable, kernel_regularizer=OrthLocalReg2D)(l)
# l = Activation('relu', name='conv21_relu')(l)
# l = BatchNormalization(name='conv21_bn')(l)
# l = Conv2D(192, (3, 3), padding='same', name='conv22', trainable=trainable, kernel_regularizer=OrthLocalReg2D)(l)
# l = Activation('relu', name='conv22_relu')(l)
# l = BatchNormalization(name='conv22_bn')(l)
# conv22
# l_offset = ConvOffset2D(192, name='conv32_offset')(l)
l = Conv2D(192, (3, 3),
padding='same',
name='conv23',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv23_relu')(l)
l = BatchNormalization(name='conv23_bn')(l)
l24 = Conv2D(192, (7, 7),
padding='same',
strides=(2, 2),
name='conv24',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l_block2)
l24 = Activation('relu', name='conv24_relu')(l24)
l24 = BatchNormalization(name='conv24_bn')(l24)
l = concatenate([l, l24])
l = Conv2D(256, (3, 3),
padding='same',
name='conv31',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv31_relu')(l)
l31 = BatchNormalization(name='conv31_bn')(l)
l = Conv2D(256, (3, 3),
padding='same',
name='conv32',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l31)
l = Activation('relu', name='conv32_relu')(l)
l = BatchNormalization(name='conv32_bn')(l)
l = Conv2D(256, (3, 3),
padding='same',
name='conv33',
strides=(2, 2),
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv33_relu')(l)
l = BatchNormalization(name='conv33_bn')(l)
# l = Add(name='residual_31_32')([l31, l])
l_offset = ConvOffset2D(256, name='conv33_offset')(l)
l = Conv2D(512, (3, 3),
padding='same',
name='conv41',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l_offset)
l = Activation('relu', name='conv41_relu')(l)
l = BatchNormalization(name='conv41_bn')(l)
l = Conv2D(512, (3, 3),
padding='same',
strides=(2, 2),
name='conv42',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv42_relu')(l)
l = l_res = BatchNormalization(name='conv42_bn')(l)
# l_offset = ConvOffset2D(512, name='conv35_offset')(l)
l = Conv2D(512, (3, 3),
padding='same',
name='conv43',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv43_relu')(l)
l = BatchNormalization(name='conv43_bn')(l)
l = Conv2D(512, (3, 3),
padding='same',
name='conv44',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv44_relu')(l)
l = BatchNormalization(name='conv44_bn')(l)
l = Add(name='residual_42_44')([l_res, l])
l = MaxPooling2D(name='max_pool_5')(l)
l = Conv2D(1024, (3, 3),
padding='same',
name='conv51',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv51_relu')(l)
l = BatchNormalization(name='conv51_bn', center=False, scale=False)(l)
l = Conv2D(1024, (3, 3),
padding='same',
name='conv52',
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('selu', name='conv52_relu')(l)
l = BatchNormalization(name='conv52_bn', center=False, scale=False)(l)
# out
# l = MaxPooling2D(name='max_pool_final')(l)
# l = Flatten(name='flatten_maxpool')(l)
l = SpatialPyramidPooling([1, 2, 4], input_shape=[None, None, 512])(l)
l = Dense(768,
name='fc1',
trainable=trainable,
kernel_regularizer=OrthLocalReg1D)(l)
l = Activation('relu', name='fc1_relu')(l)
l = Dense(256,
name='fc2',
trainable=trainable,
kernel_regularizer=OrthLocalReg1D)(l)
l = Activation('relu', name='fc2_relu')(l)
l = Dense(class_num, name='fc3', trainable=trainable)(l)
outputs = l = Activation('softmax', name='out')(l)
return inputs, outputs
def generate_model(self, weight_decay=0.0005):
'''
define the model structure
---------------------------------------------------------------------------
INPUT:
weight_decay: all the weights in the layer would be decayed by this factor
OUTPUT:
model: the model structure after being defined
# References
- [An Improved Deep Learning Architecture for Person Re-Identification]
---------------------------------------------------------------------------
'''
self.gg = tf.Graph()
with self.gg.as_default() as g:
config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.1
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))
def upsample_neighbor_function(input_x):
input_x_pad = K.spatial_2d_padding(input_x,
padding=((2, 2), (2, 2)))
x_length = K.int_shape(input_x)[1]
y_length = K.int_shape(input_x)[2]
output_x_list = []
output_y_list = []
for i_x in range(2, x_length + 2):
for i_y in range(2, y_length + 2):
output_y_list.append(input_x_pad[:, i_x - 2:i_x + 3,
i_y - 2:i_y + 3, :])
output_x_list.append(K.concatenate(output_y_list, axis=2))
output_y_list = []
return K.concatenate(output_x_list, axis=1)
max_pooling = MaxPooling2D()
x1_input = Input(shape=(160, 60, 3))
x2_input = Input(shape=(160, 60, 3))
share_conv_1 = Conv2D(20,
5,
kernel_regularizer=l2(weight_decay),
activation="relu")
x1 = share_conv_1(x1_input)
x2 = share_conv_1(x2_input)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
share_conv_2 = Conv2D(25,
5,
kernel_regularizer=l2(weight_decay),
activation="relu")
x1 = share_conv_2(x1)
x2 = share_conv_2(x2)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
upsample_same = UpSampling2D(size=(5, 5))
x1_up = upsample_same(x1)
x2_up = upsample_same(x2)
upsample_neighbor = Lambda(upsample_neighbor_function)
x1_nn = upsample_neighbor(x1)
x2_nn = upsample_neighbor(x2)
negative = Lambda(lambda x: -x)
x1_nn = negative(x1_nn)
x2_nn = negative(x2_nn)
x1 = Add()([x1_up, x2_nn])
x2 = Add()([x2_up, x1_nn])
conv_3_1 = Conv2D(25,
5,
strides=(5, 5),
kernel_regularizer=l2(weight_decay),
activation="relu")
conv_3_2 = Conv2D(25,
5,
strides=(5, 5),
kernel_regularizer=l2(weight_decay),
activation="relu")
x1 = conv_3_1(x1)
x2 = conv_3_2(x2)
conv_4_1 = Conv2D(25,
3,
kernel_regularizer=l2(weight_decay),
activation="relu")
conv_4_2 = Conv2D(25,
3,
kernel_regularizer=l2(weight_decay),
activation="relu")
x1 = conv_4_1(x1)
x2 = conv_4_2(x2)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
y = Concatenate()([x1, x2])
y = Flatten()(y)
y = Dense(500,
kernel_regularizer=l2(weight_decay),
activation='relu')(y)
y = Dense(2,
kernel_regularizer=l2(weight_decay),
activation='softmax')(y)
model = Model(inputs=[x1_input, x2_input], outputs=[y])
#model.summary()
return model
def get_large_deform_cnn2(class_num, trainable=False, GPU=1):
# init = Orthogonal(gain=1.0, seed=None)
init = 'random_normal'
inputs = l = Input((200, 200, 3), name='input')
input_target = Input((1, ), name='input_target')
#norm_input = ImageNorm()(inputs)
norm_input = inputs
# conv11
l = Conv2D(32, (3, 3),
padding='same',
name='conv11',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l = Activation('relu', name='conv11_relu')(l)
l = BatchNormalization(name='conv11_bn')(l)
l2 = InvConv2D(32, (3, 3),
padding='same',
name='inv_conv11',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l2 = Activation('relu', name='inv_conv11_relu')(l2)
l2 = BatchNormalization(name='inv_conv11_bn')(l2)
l3 = Conv2D(32, (3, 1),
padding='same',
name='conv11_2',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l3 = Activation('relu', name='conv11_2_relu')(l3)
l3 = BatchNormalization(name='conv11_2_bn')(l3)
l5 = Conv2D(32, (1, 3),
padding='same',
name='conv11_3',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l5 = Activation('relu', name='conv11_3_relu')(l5)
l5 = BatchNormalization(name='conv11_3_bn')(l5)
l4 = InvConv2D(32, (3, 1),
padding='same',
name='conv11_2i',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l4 = Activation('relu', name='conv11_2i_relu')(l4)
l4 = BatchNormalization(name='conv11_2i_bn')(l4)
l6 = InvConv2D(32, (1, 3),
padding='same',
name='conv11_3i',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(norm_input)
l6 = Activation('relu', name='conv11_3i_relu')(l6)
l6 = BatchNormalization(name='conv11_3i_bn')(l6)
l = concatenate([l, l2, l3, l5, l4, l6])
# conv12
# l_offset = ConvOffset2D(32, name='conv12_offset')(l)
l5 = Conv2D(128, (5, 5),
padding='same',
strides=(2, 2),
name='pool5_11_12',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l5 = Activation('relu', name='pool5_11_12_relu')(l5)
l5 = BatchNormalization(name='pool5_11_12_bn')(l5)
l3 = Conv2D(128, (3, 3),
padding='same',
strides=(2, 2),
name='pool3_11_12',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l3 = Activation('relu', name='pool3_11_12_relu')(l3)
l3 = BatchNormalization(name='pool3_11_12_bn')(l3)
l = concatenate([l3, l5])
l3 = Conv2D(32, (5, 3),
padding='same',
name='conv12_2',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l3 = Activation('relu', name='conv12_2_relu')(l3)
l3 = BatchNormalization(name='conv12_2_bn')(l3)
l5 = Conv2D(32, (3, 5),
padding='same',
name='conv12_3',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l5 = Activation('relu', name='conv12_3_relu')(l5)
l5 = BatchNormalization(name='conv12_3_bn')(l5)
l = Conv2D(128, (3, 3),
padding='same',
name='conv12',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv12_relu')(l)
l = BatchNormalization(name='conv12_bn')(l)
l = concatenate([l, l3, l5])
l = Conv2D(128, (3, 3),
padding='same',
name='conv13',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv13_relu')(l)
l = jump = BatchNormalization(name='conv13_bn')(l)
l = Conv2D(192, (3, 3),
padding='same',
strides=(2, 2),
name='conv14',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv14_relu')(l)
# l = BatchNormalization(name='conv14_bn')(l)
l = Conv2D(192, (3, 3),
padding='same',
name='conv21',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv21_relu')(l)
l = BatchNormalization(name='conv21_bn')(l)
l = Conv2D(192, (3, 3),
padding='same',
name='conv22',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv22_relu')(l)
l = BatchNormalization(name='conv22_bn')(l)
# conv22
# l_offset = ConvOffset2D(192, name='conv32_offset')(l)
l = Conv2D(256, (3, 3),
padding='same',
strides=(2, 2),
name='conv23',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv23_relu')(l)
l = BatchNormalization(name='conv23_bn')(l)
l = Conv2D(256, (3, 3),
padding='same',
name='conv31',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv31_relu')(l)
l31 = BatchNormalization(name='conv31_bn')(l)
# l = Conv2D(256, (1, 1), padding='same', name='conv32', trainable=trainable, kernel_initializer=init, kernel_regularizer=OrthLocalReg2D)(l31)
# l = Activation('relu', name='conv32_relu')(l)
# l = BatchNormalization(name='conv32_bn')(l)
l = Conv2D(256, (3, 3),
padding='same',
name='conv33',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv33_relu')(l)
l = BatchNormalization(name='conv33_bn')(l)
l = Add(name='residual_31_32')([l31, l])
lj = Conv2D(256, (3, 3),
padding='same',
strides=(2, 2),
name='jump_pool',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(jump)
lj = Activation('relu', name='jump_pool_relu')(lj)
lj = BatchNormalization(name='jump_pool_bn')(lj)
lj = Conv2D(256, (3, 3),
padding='same',
strides=(2, 2),
name='jump_pool2',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(lj)
lj = Activation('relu', name='jump_pool2_relu')(lj)
lj = BatchNormalization(name='jump_pool2_bn')(lj)
l = concatenate([l, lj])
l_offset = ConvOffset2D(512, name='conv33_offset')(l)
l = Conv2D(512, (3, 3),
padding='same',
strides=(2, 2),
name='conv41',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l_offset)
l = Activation('relu', name='conv41_relu')(l)
l = BatchNormalization(name='conv41_bn')(l)
l = Conv2D(512, (3, 3),
padding='same',
name='conv42',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv42_relu')(l)
l = BatchNormalization(name='conv42_bn')(l)
# l_offset = ConvOffset2D(512, name='conv35_offset')(l)
l = Conv2D(512, (3, 3),
padding='same',
name='conv43',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg2D)(l)
l = Activation('relu', name='conv43_relu')(l)
l = BatchNormalization(name='conv43_bn')(l)
# out
# l = GlobalAvgPool2D(name='avg_pool')(l)
l = MaxPooling2D(name='max_pool_final')(l)
l = Flatten(name='flatten_maxpool')(l)
l = Dense(768,
name='fc1',
trainable=trainable,
kernel_initializer=init,
kernel_regularizer=OrthLocalReg1D)(l)
l = Activation('relu', name='fc1_relu')(l)
l = feature = Dense(256,
name='fc2',
trainable=trainable,
kernel_initializer=init)(l)
l = Activation('relu', name='fc2_relu')(l)
l = Dense(class_num, name='fc3', trainable=trainable)(l)
outputs = l = Activation('softmax', name='out')(l)
if GPU == 1:
centers = Embedding(class_num, 256)(input_target)
l2_loss = Lambda(
lambda x: K.sum(K.square(x[0] - x[1][:, 0]), 1, keepdims=True),
name='l2_loss')([feature, centers])
Model(inputs=[inputs, input_target], outputs=[outputs, l2_loss])
elif GPU == 0:
return inputs, outputs
else:
BODY = Model(inputs=[inputs, input_target], outputs=[outputs, feature])
BODY = make_parallel(BODY, GPU)
softmax_output = Lambda(lambda x: x, name='output')(BODY.outputs[0])
centers = Embedding(class_num, 256)(input_target)
l2_loss = Lambda(
lambda x: K.sum(K.square(x[0] - x[1][:, 0]), 1, keepdims=True),
name='l2_loss')([BODY.outputs[1], centers])
model_withcneter = Model(inputs=BODY.inputs,
outputs=[softmax_output, l2_loss])
return model_withcneter
def buildNetwork(weight_decay):
max_pooling = MaxPooling2D()
x1_input = Input(shape=(160, 60, 3))
x2_input = Input(shape=(160, 60, 3))
share_conv_1 = Conv2D(20,
5,
kernel_regularizer=l2(weight_decay),
activation="relu")
x1 = share_conv_1(x1_input)
x2 = share_conv_1(x2_input)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
x1 = GaussianNoise(0.1)(x1)
x2 = GaussianNoise(0.1)(x2)
share_conv_2 = Conv2D(25,
5,
kernel_regularizer=l2(weight_decay),
activation="relu")
x1 = share_conv_2(x1)
x2 = share_conv_2(x2)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
x1 = Dropout(0.4)(x1)
x2 = Dropout(0.4)(x2)
upsample_same = UpSampling2D(size=(5, 5))
x1_up = upsample_same(x1)
x2_up = upsample_same(x2)
upsample_neighbor = Lambda(upsample_neighbor_function)
x1_nn = upsample_neighbor(x1)
x2_nn = upsample_neighbor(x2)
negative = Lambda(lambda x: -x)
x1_nn = negative(x1_nn)
x2_nn = negative(x2_nn)
x1 = Add()([x1_up, x2_nn])
x2 = Add()([x2_up, x1_nn])
conv_3_1 = Conv2D(25,
5,
strides=(5, 5),
kernel_regularizer=l2(weight_decay),
activation="relu")
conv_3_2 = Conv2D(25,
5,
strides=(5, 5),
kernel_regularizer=l2(weight_decay),
activation="relu")
x1 = conv_3_1(x1)
x2 = conv_3_2(x2)
x1 = Dropout(0.4)(x1)
x2 = Dropout(0.4)(x2)
conv_4_1 = Conv2D(25,
3,
kernel_regularizer=l2(weight_decay),
activation="relu")
conv_4_2 = Conv2D(25,
3,
kernel_regularizer=l2(weight_decay),
activation="relu")
x1 = conv_4_1(x1)
x2 = conv_4_2(x2)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
x1 = Dropout(0.4)(x1)
x2 = Dropout(0.4)(x2)
y = Concatenate()([x1, x2])
y = Flatten()(y)
y = Dense(500, kernel_regularizer=l2(weight_decay),
activation='relu')(y)
y = Dense(2, kernel_regularizer=l2(weight_decay),
activation='softmax')(y)
return Model(inputs=[x1_input, x2_input], outputs=[y])
def residual_block(inputs,
base_depth,
depth,
kernel_size,
stride=1,
rate=1,
block_name="block1",
unit_name="unit1",
weight_decay=1e-4,
kernel_initializer="he_normal",
bn_epsilon=1e-3,
bn_momentum=0.99):
"""Implementation of a residual block, with 3 conv layers. Each convolutional layer is followed
with a batch normalization layer and a relu layer.
The corresponding kernel sizes are (1, kernel_size, 1),
corresponding strides are (1->stride->1),
corresponding filters are (base_depth, base_depth, depth).
If the depth of the inputs is equal to the 'depth', this is a identity block, else a convolutional
block.
:param inputs: 4-D tensor, shape of (batch_size, height, width, channel).
:param base_depth: int, base depth of the residual block.
:param depth: int, output depth.
:param kernel_size: int.
:param stride: int, default 1.
:param rate: int, dilation rate, default 1.
:param block_name: string, name of the bottleneck block, default "block1".
:param unit_name: string, name of the unit(residual block), default "unit1".
:param weight_decay: float, default 1e-4.
:param kernel_initializer: string, default "he_normal".
:param bn_epsilon: float, default 1e-3.
:param bn_momentum: float, default 0.99.
:return: 4-D tensor, shape of (batch_size, height, width, channel).
"""
depth_in = int(inputs.shape[-1])
conv_name_base = block_name+"_"+unit_name+"_conv"
bn_name_base = block_name+"_"+unit_name+"_bn"
# pre-activation and batch normalization
preact = BatchNormalization(name=bn_name_base+"0", epsilon=bn_epsilon, momentum=bn_momentum)(inputs)
preact = Activation("relu")(preact)
# determine convolutional or identity connection
if depth_in == depth:
x_shortcut = MaxPooling2D(pool_size=(1, 1), strides=stride)(inputs) if stride > 1 else inputs
else:
x_shortcut = Conv2D(depth, (1, 1), strides=(stride, stride), name=conv_name_base + "short",
use_bias=False, activation=None, kernel_initializer=kernel_initializer,
kernel_regularizer=l2(weight_decay))(preact)
x_shortcut = BatchNormalization(name=bn_name_base + "short", epsilon=bn_epsilon,
momentum=bn_momentum)(x_shortcut)
x = Conv2D(base_depth, (1, 1), strides=(1, 1), padding="same", name=conv_name_base + "2a",
use_bias=False, activation=None, kernel_initializer=kernel_initializer,
kernel_regularizer=l2(weight_decay))(preact)
x = BatchNormalization(name=bn_name_base + "2a", epsilon=bn_epsilon, momentum=bn_momentum)(x)
x = Activation("relu")(x)
x = Conv2D(base_depth, (kernel_size, kernel_size), strides=(stride, stride), dilation_rate=rate,
padding="same", name=conv_name_base + "2b", use_bias=False, activation=None,
kernel_initializer=kernel_initializer, kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name=bn_name_base + "2b", epsilon=bn_epsilon, momentum=bn_momentum)(x)
x = Activation("relu")(x)
x = Conv2D(depth, (1, 1), strides=(1, 1), padding="same", name=conv_name_base + "2c", use_bias=False,
activation=None, kernel_initializer=kernel_initializer, kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name=bn_name_base + "2c", epsilon=bn_epsilon, momentum=bn_momentum)(x)
output = Add()([x_shortcut, x])
return output
def create_deepconn_dp(self):
dotproduct = Dot(axes=1)([self.towerU, self.towerM])
output = Add()([self.outNeuron, dotproduct])
self.model = Model(inputs=[self.inputU, self.inputM], outputs=[output])
self.model.compile(optimizer='Adam', loss='mse')
def IRD_block(
input,
filters,
kernel_size=(3, 3),
strides=(1, 1),
):
x = BatchNormalization()(input)
x = Activation('relu')(x)
x = Conv2D(filters=4 * filters,
kernel_size=(1, 1),
strides=strides,
padding='same')(x)
#Tower 1
tower_1 = Conv2D(filters=filters,
kernel_size=(1, 1),
strides=strides,
padding='same')(x)
tower_1 = BatchNormalization()(tower_1)
tower_1 = Activation('relu')(tower_1)
#Tower 2
tower_2 = Conv2D(filters=4 * filters,
kernel_size=(1, 1),
strides=strides,
padding='same')(x)
tower_2 = BatchNormalization()(tower_2)
tower_2 = Activation('relu')(tower_2)
tower_2 = Conv2D(filters=filters,
kernel_size=(3, 3),
strides=strides,
padding='same')(x)
tower_2 = BatchNormalization()(tower_2)
tower_2 = Activation('relu')(tower_2)
#Tower 3
tower_3 = Conv2D(filters=filters,
kernel_size=(1, 1),
strides=strides,
padding='same')(x)
tower_3 = BatchNormalization()(tower_3)
tower_3 = Activation('relu')(tower_3)
tower_3 = Conv2D(filters=4 * filters,
kernel_size=(3, 3),
strides=strides,
padding='same')(x)
tower_3 = BatchNormalization()(tower_3)
tower_3 = Activation('relu')(tower_3)
tower_3 = Conv2D(filters=filters,
kernel_size=(3, 3),
strides=strides,
padding='same')(x)
tower_3 = BatchNormalization()(tower_3)
tower_3 = Activation('relu')(tower_3)
#Tower 4
# tower_4 = MaxPooling2D((2, 2), padding='same')(x)
tower_4 = Conv2D(filters=filters,
kernel_size=(1, 1),
strides=strides,
padding='same')(x)
tower_4 = BatchNormalization()(tower_4)
tower_4 = Activation('relu')(tower_4)
inception = concatenate([tower_1, tower_2, tower_3, tower_4])
inception = Add()([inception, x])
res = BatchNormalization()(inception)
res = Activation('relu')(x)
res = Conv2D(filters=4 * filters,
kernel_size=(1, 1),
strides=strides,
padding='same')(x)
output = Add()([input, res])
return output
def segnet(nClasses, optimizer=None, input_height=360, input_width=480):
kernel = 3
filter_size = 64
pad = 1
pool_size = 2
img_input = Input(shape=(input_height, input_width,3))
# encoder
x = ZeroPadding2D(padding=(pad, pad))(img_input)
x = Convolution2D(filter_size, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu') (x)
l1 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(128, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
l2 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(256, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
l3 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(512, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
l4 = x
x = Activation('relu')(x)
# decoder
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(512, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l4, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(256, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l3, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(128, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l2, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(filter_size, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l1, x])
x = Convolution2D(nClasses, (1, 1), padding='valid') (x)
beforeCrfRNN = x
out = CrfRnnLayer(image_dims=(input_height, input_width),
num_classes=nClasses,
theta_alpha=160.,
theta_beta=3.,
theta_gamma=3.,
num_iterations=5,
name='crfrnn')([x, img_input])
a = Model(inputs=img_input, outputs=out)
model = []
a.outputHeight = a.output_shape[1]
a.outputWidth = a.output_shape[2]
out = Reshape((a.outputHeight * a.outputWidth, nClasses), input_shape=(nClasses, a.outputHeight, a.outputWidth))(out)
out = Activation('softmax')(out)
model = Model(inputs=img_input, outputs=out)
model.outputHeight = a.outputHeight
model.outputWidth = a.outputWidth
model.compile(loss=penalized_loss(bottleNeckFeatures=l4), optimizer="adadelta", metrics=['accuracy'])
return model
def transformer_concat_model(
conditioning_input_shapes,
conditioning_input_names=None,
output_shape=None,
model_name='CVAE_transformer',
transform_latent_shape=(100, ),
enc_params=None,
condition_on_image=True,
n_concat_scales=3,
transform_activation=None,
clip_output_range=None,
source_input_idx=None,
):
# collect conditioning inputs, and concatentate them into a stack
if not isinstance(conditioning_input_shapes, list):
conditioning_input_shapes = [conditioning_input_shapes]
if conditioning_input_names is None:
conditioning_input_names = [
'cond_input_{}'.format(ii)
for ii in range(len(conditioning_input_shapes))
]
conditioning_inputs = []
for ii, input_shape in enumerate(conditioning_input_shapes):
conditioning_inputs.append(
Input(input_shape, name=conditioning_input_names[ii]))
if len(conditioning_inputs) > 1:
conditioning_input_stack = Concatenate(name='concat_cond_inputs',
axis=-1)(conditioning_inputs)
else:
conditioning_input_stack = conditioning_inputs[0]
conditioning_input_shape = tuple(
conditioning_input_stack.get_shape().as_list()[1:])
n_dims = len(conditioning_input_shape) - 1
# we will always give z as a flattened vector
z_input = Input((np.prod(transform_latent_shape), ), name='z_input')
# determine what we should apply the transformation to
if source_input_idx is None:
# the image we want to transform is exactly the input of the conditioning branch
x_source = conditioning_input_stack
source_input_shape = conditioning_input_shape
else:
# slice conditioning input to get the single source im that we will apply the transform to
source_input_shape = conditioning_input_shapes[source_input_idx]
x_source = conditioning_inputs[source_input_idx]
# assume the output is going to be the transformed source input, so it should be the same shape
if output_shape is None:
output_shape = source_input_shape
layer_prefix = 'color'
decoder_output_shape = output_shape
if condition_on_image: # assume we always condition by concat since it's better than other forms
# simply concatenate the conditioning stack (at various scales) with the decoder volumes
include_fullres = True
concat_decoder_outputs_with = [None] * len(enc_params['nf_dec'])
concat_skip_sizes = [None] * len(enc_params['nf_dec'])
# make sure x_I is the same shape as the output, including in the channels dimension
if not np.all(output_shape <= conditioning_input_shape):
tile_factor = [
int(round(output_shape[i] / conditioning_input_shape[i]))
for i in range(len(output_shape))
]
print('Tile factor: {}'.format(tile_factor))
conditioning_input_stack = Lambda(
lambda x: tf.tile(x, [1] + tile_factor),
name='lambda_tile_cond_input')(conditioning_input_stack)
# downscale the conditioning inputs by the specified number of times
xs_downscaled = [conditioning_input_stack]
for si in range(n_concat_scales):
curr_x_scaled = network_utils.Blur_Downsample(
n_chans=conditioning_input_shape[-1],
n_dims=n_dims,
do_blur=True,
name='downsample_scale-1/{}'.format(2**(si + 1)))(
xs_downscaled[-1])
xs_downscaled.append(curr_x_scaled)
if not include_fullres:
xs_downscaled = xs_downscaled[1:] # exclude the full-res volume
print('Including downsampled input sizes {}'.format(
[x.get_shape().as_list() for x in xs_downscaled]))
# the smallest decoder volume will be the same as the smallest encoder volume, so we need to make sure we match volume sizes appropriately
n_enc_scales = len(enc_params['nf_enc'])
n_ds = len(xs_downscaled)
concat_decoder_outputs_with[n_enc_scales - n_ds +
1:n_enc_scales] = list(
reversed(xs_downscaled))
concat_skip_sizes[n_enc_scales - n_ds + 1:n_enc_scales] = list(
reversed([
np.asarray(x.get_shape().as_list()[1:-1])
for x in xs_downscaled if x is not None
]))
else:
# just ignore the conditioning input
concat_decoder_outputs_with = None
concat_skip_sizes = None
if 'ks' not in enc_params:
enc_params['ks'] = 3
# determine what size to reshape the latent vector to
reshape_encoding_to = get_encoded_shape(
img_shape=conditioning_input_shape,
conv_chans=enc_params['nf_enc'],
)
x_enc = Dense(np.prod(reshape_encoding_to),
name='dense_encoding_to_vol')(z_input)
x_enc = LeakyReLU(0.2)(x_enc)
x_enc = Reshape(reshape_encoding_to)(x_enc)
print('Decoder starting shape: {}'.format(reshape_encoding_to))
x_transformation = decoder(
x_enc,
decoder_output_shape,
encoded_shape=reshape_encoding_to,
prefix='{}_dec'.format(layer_prefix),
conv_chans=enc_params['nf_dec'],
ks=enc_params['ks'] if 'ks' in enc_params else 3,
n_convs_per_stage=enc_params['n_convs_per_stage']
if 'n_convs_per_stage' in enc_params else 1,
use_upsample=enc_params['use_upsample']
if 'use_upsample' in enc_params else False,
kernel_initializer=enc_params['kernel_initializer']
if 'kernel_initializer' in enc_params else None,
bias_initializer=enc_params['bias_initializer']
if 'bias_initializer' in enc_params else None,
include_skips=concat_decoder_outputs_with,
target_vol_sizes=concat_skip_sizes)
if transform_activation is not None:
x_transformation = Activation(
transform_activation,
name='activation_transform_{}'.format(transform_activation))(
x_transformation)
if transform_activation == 'tanh':
# TODO: maybe move this logic
# if we are learning a colro delta with a tanh, make sure to multiply it by 2
x_transformation = Lambda(
lambda x: x * 2, name='lambda_scale_tanh')(x_transformation)
im_out = Add()([x_source, x_transformation])
if clip_output_range is not None:
im_out = Lambda(lambda x: tf.clip_by_value(x, clip_output_range[0],
clip_output_range[1]),
name='lambda_clip_output_{}-{}'.format(
clip_output_range[0],
clip_output_range[1]))(im_out)
return Model(inputs=conditioning_inputs + [z_input],
outputs=[im_out, x_transformation],
name=model_name)
def uxnet(input_shape,
n_depth=2,
n_filter_base=16,
kernel_size=(3, 3),
n_conv_per_depth=2,
activation="relu",
last_activation='linear',
batch_norm=False,
dropout=0.0,
pool_size=(2, 2),
residual=True,
odd_to_even=False,
shortcut=None,
shared_idx=[],
prob_out=False,
eps_scale=1e-3):
"""
Multi-body U-Net which learns identity by leaving one plane out in each branch
:param input_shape:
:param n_depth:
:param n_filter_base:
:param kernel_size:
:param n_conv_per_depth:
:param activation:
:param last_activation:
:param batch_norm:
:param dropout:
:param pool_size:
:param prob_out:
:param eps_scale:
:return: Model
"""
# TODO: fill params
# TODO: add odd-to-even mode
# Define vars
channel_axis = -1 if backend_channels_last() else 1
n_planes = input_shape[channel_axis]
if n_planes % 2 != 0 and odd_to_even:
raise ValueError('Odd-to-even mode does not support uneven number of planes')
n_dim = len(kernel_size)
conv = Conv2D if n_dim == 2 else Conv3D
# Define functional model
input = Input(shape=input_shape, name='input_main')
# TODO test new implementation and remove old
# Split planes (preserve channel)
input_x = [Lambda(lambda x: x[..., i:i+1], output_shape=(None, None, 1))(input) for i in range(n_planes)]
# We can train either in odd-to-even mode or in LOO mode
if odd_to_even:
# In this mode we stack together odd and even planes, train the net to predict even from odd and vice versa
# input_x_out = [Concatenate(axis=-1)(input_x[j::2]) for j in range(2)]
input_x_out = [Concatenate(axis=-1)(input_x[j::2]) for j in range(1, -1, -1)]
else:
# Concatenate planes back in leave-one-out way
input_x_out = [Concatenate(axis=-1)([plane for i, plane in enumerate(input_x) if i != j]) for j in range(n_planes)]
# if odd_to_even:
# input_x_out = [Lambda(lambda x: x[..., j::2],
# output_shape=(None, None, n_planes // 2),
# name='{}_planes'.format('even' if j == 0 else 'odd'))(input)
# for j in range(1, -1, -1)]
# else:
# # input_x_out = [Lambda(lambda x: x[..., tf.convert_to_tensor([i for i in range(n_planes) if i != j], dtype=tf.int32)],
# # output_shape=(None, None, n_planes-1),
# # name='leave_{}_plane_out'.format(j))(input)
# # for j in range(n_planes)]
#
# input_x_out = [Lambda(lambda x: K.concatenate([x[..., :j], x[..., (j+1):]], axis=-1),
# output_shape=(None, None, n_planes - 1),
# name='leave_{}_plane_out'.format(j))(input)
# for j in range(n_planes)]
# U-Net parameters depend on mode (odd-to-even or LOO)
n_blocks = 2 if odd_to_even else n_planes
input_planes = n_planes // 2 if odd_to_even else n_planes-1
output_planes = n_planes // 2 if odd_to_even else 1
# Create U-Net blocks (by number of planes)
unet_x = unet_blocks(n_blocks=n_blocks, input_planes=input_planes, output_planes=output_planes,
n_depth=n_depth, n_filter_base=n_filter_base, kernel_size=kernel_size,
activation=activation, dropout=dropout, batch_norm=batch_norm,
n_conv_per_depth=n_conv_per_depth, pool=pool_size, shared_idx=shared_idx)
unet_x = [unet(inp_out) for unet, inp_out in zip(unet_x, input_x_out)]
# Version without weight sharing:
# unet_x = [unet_block(n_depth, n_filter_base, kernel_size,
# activation=activation, dropout=dropout, batch_norm=batch_norm,
# n_conv_per_depth=n_conv_per_depth, pool=pool_size,
# prefix='out_{}_'.format(i))(inp_out) for i, inp_out in enumerate(input_x_out)]
# TODO: rewritten for sharing -- remove commented below
# Convolve n_filter_base to 1 as each U-Net predicts a single plane
# unet_x = [conv(1, (1,) * n_dim, activation=activation)(unet) for unet in unet_x]
if residual:
if odd_to_even:
# For residual U-Net sum up output for odd planes with even planes and vice versa
unet_x = [Add()([unet, inp]) for unet, inp in zip(unet_x, input_x[::-1])]
else:
# For residual U-Net sum up output with its neighbor (next for the first plane, previous for the rest
unet_x = [Add()([unet, inp]) for unet, inp in zip(unet_x, [input_x[1]]+input_x[:-1])]
# Concatenate outputs of blocks, should receive (None, None, None, n_planes)
# TODO assert to check shape?
if odd_to_even:
# Split even and odd, assemble them together in the correct order
# TODO tests
unet_even = [Lambda(lambda x: x[..., i:i+1],
output_shape=(None, None, 1),
name='even_{}'.format(i))(unet_x[0]) for i in range(n_planes // 2)]
unet_odd = [Lambda(lambda x: x[..., i:i+1],
output_shape=(None, None, 1),
name='odd_{}'.format(i))(unet_x[1]) for i in range(n_planes // 2)]
unet_x = list(np.array(list(zip(unet_even, unet_odd))).flatten())
unet = Concatenate(axis=-1)(unet_x)
if shortcut is not None:
# We can create a shortcut without long skip connection to prevent noise memorization
if shortcut == 'unet':
shortcut_block = unet_block(long_skip=False, input_planes=n_planes,
n_depth=n_depth, n_filter_base=n_filter_base, kernel_size=kernel_size,
activation=activation, dropout=dropout, batch_norm=batch_norm,
n_conv_per_depth=n_conv_per_depth, pool=pool_size)(input)
shortcut_block = conv(n_planes, (1,) * n_dim, activation='linear', name='shortcut_final_conv')(shortcut_block)
# Or a simple gaussian blur block
elif shortcut == 'gaussian':
shortcut_block = gaussian_2d(n_planes, k=13, s=7)(input)
else:
raise ValueError('Shortcut should be either unet or gaussian')
# TODO add or concatenate?
unet = Add()([unet, shortcut_block])
# unet = Concatenate(axis=-1)([unet, shortcut_unet])
# Final activation layer
final = Activation(activation=last_activation)(unet)
if prob_out:
scale = conv(n_planes, (1,)*n_dim, activation='softplus')(unet)
scale = Lambda(lambda x: x+np.float32(eps_scale))(scale)
final = Concatenate(axis=channel_axis)([final, scale])
return Model(inputs=input, outputs=final)
def encoder(x,
img_shape,
conv_chans=None,
n_convs_per_stage=1,
min_h=5,
min_c=None,
prefix='',
ks=3,
return_skips=False,
use_residuals=False,
use_maxpool=False,
kernel_initializer=None,
bias_initializer=None):
skip_layers = []
concat_skip_sizes = []
n_dims = len(
img_shape
) - 1 # assume img_shape includes spatial dims, followed by channels
if conv_chans is None:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [min_c * 2] * (n_convs - 1) + [min_c]
elif not type(conv_chans) == list:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [conv_chans] * (n_convs - 1) + [min_c]
else:
n_convs = len(conv_chans)
if isinstance(ks, list):
assert len(ks) == (n_convs + 1
) # specify for each conv, as well as the last one
else:
ks = [ks] * (n_convs + 1)
for i in range(len(conv_chans)):
#if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None:
for ci in range(n_convs_per_stage):
x = myConv(nf=conv_chans[i],
ks=ks[i],
strides=1,
n_dims=n_dims,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
prefix='{}_enc'.format(prefix),
suffix='{}_{}'.format(i, ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))(
[residual_input, x])
x = LeakyReLU(0.2,
name='{}_enc_leakyrelu_{}_{}'.format(
prefix, i, ci + 1))(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1]))
if use_maxpool and i < len(conv_chans) - 1:
# changed 5/30/19, don't pool after our last conv
x = myPool(n_dims=n_dims, prefix=prefix, suffix=i)(x)
else:
x = myConv(conv_chans[i],
ks=ks[i],
strides=2,
n_dims=n_dims,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
prefix='{}_enc'.format(prefix),
suffix=i)(x)
# don't activate right after a maxpool, it makes no sense
if i < len(conv_chans) - 1: # no activation on last convolution
x = LeakyReLU(0.2,
name='{}_enc_leakyrelu_{}'.format(prefix, i))(x)
if min_c is not None and min_c > 0:
# if the last number of channels is specified, convolve to that
if n_convs_per_stage is not None and n_convs_per_stage > 1:
for ci in range(n_convs_per_stage):
# TODO: we might not have enough ks for this
x = myConv(min_c,
ks=ks[-1],
n_dims=n_dims,
strides=1,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
prefix='{}_enc'.format(prefix),
suffix='last_{}'.format(ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(
name='{}_enc_{}_add_residual'.format(prefix, 'last'))(
[residual_input, x])
x = LeakyReLU(0.2,
name='{}_enc_leakyrelu_last'.format(prefix))(x)
x = myConv(min_c,
ks=ks[-1],
strides=1,
n_dims=n_dims,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
prefix='{}_enc'.format(prefix),
suffix='_last')(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1]))
if return_skips:
return x, skip_layers, concat_skip_sizes
else:
return x
def ReductionCell(hi,
h_i1sub,
filter_i,
filter_o,
stride=2,
name="NAS",
is_tail=False):
"""
adjust feature size & channel size
"""
h_i1sub = _adjust_block(h_i1sub, hi, filter_o, 0, name)
hi = Conv2D(filter_o, (1, 1),
padding='same',
name='%s_hi_align' % name,
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(hi)
hi = Activation('relu', name='%s_hi_align_relu' % name)(hi)
hi = BatchNormalization(name='%s_hi_align_bn' % name)(hi)
"""
SubLayer 1
"""
l = SeparableConv2D(filter_o, (5, 5),
strides=(stride, stride),
padding='same',
name='%s_sep5x5_1' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(hi)
l = Activation('relu', name='%s_sep5x5_1_relu' % name)(l)
l = BatchNormalization(name='%s_sep5x5_1_bn' % name)(l)
if h_i1sub is not None:
l_1sub = SeparableConv2D(
filter_o, (7, 7),
strides=(stride, stride),
padding='same',
name='%s_sep7x7_sub_1' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(h_i1sub)
l_1sub = Activation('relu',
name='%s_sep7x7_sub_1_relu' % name)(l_1sub)
l_1sub = BatchNormalization(name='%s_sep7x7_sub_1_bn' %
name)(l_1sub)
add1 = Add()([l, l_1sub])
else:
add1 = l
l = SeparableConv2D(filter_o, (3, 3),
strides=(stride, stride),
padding='same',
name='%s_sep3x3_1' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(hi)
l = Activation('relu', name='%s_sep3x3_1_relu' % name)(l)
l = BatchNormalization(name='%s_sep3x3_1_bn' % name)(l)
if h_i1sub is not None:
l_1sub = SeparableConv2D(
filter_o, (7, 7),
strides=(stride, stride),
padding='same',
name='%s_sep7x7_sub_2' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(h_i1sub)
l_1sub = Activation('relu',
name='%s_sep7x7_sub_2_relu' % name)(l_1sub)
l_1sub = BatchNormalization(name='%s_sep7x7_sub_2_bn' %
name)(l_1sub)
add2 = Add()([l, l_1sub])
else:
add2 = l
l = AveragePooling2D(pool_size=(3, 3),
strides=(stride, stride),
padding='same')(hi)
if filter_i != filter_o:
l = Conv2D(filter_o, (1, 1),
strides=(1, 1),
padding='same',
name='%s_sep1x1_align' % name,
trainable=trainable)(l)
l = Activation('relu', name='%s_sep1x1_align_relu' % name)(l)
l = BatchNormalization(name='%s_sep1x1_align_bn' % name)(l)
if h_i1sub is not None:
l_1sub = SeparableConv2D(
filter_o, (5, 5),
strides=(stride, stride),
padding='same',
name='%s_sep5x5_sub_1' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(h_i1sub)
l_1sub = Activation('relu',
name='%s_sep5x5_sub_1_relu' % name)(l_1sub)
l_1sub = BatchNormalization(name='%s_sep5x5_sub_1_bn' %
name)(l_1sub)
add3 = Add()([l, l_1sub])
else:
add3 = l
"""
SubLayer 2
"""
l = MaxPooling2D(pool_size=(3, 3),
strides=(stride, stride),
padding='same')(hi)
if filter_i != filter_o:
l = Conv2D(filter_o, (1, 1),
strides=(1, 1),
padding='same',
name='%s_sep1x1_align_2' % name,
trainable=trainable)(l)
l = Activation('relu', name='%s_sep1x1_align_2_relu' % name)(l)
l = BatchNormalization(name='%s_sep1x1_align_2_bn' % name)(l)
l_1sub = SeparableConv2D(filter_o, (3, 3),
padding='same',
name='%s_sep3x3_2' % name,
trainable=trainable,
depthwise_regularizer=OrthLocalRegSep2D)(add1)
l_1sub = Activation('relu', name='%s_sep3x3_2_relu' % name)(l_1sub)
l_1sub = BatchNormalization(name='%s_sep3x3_2_bn' % name)(l_1sub)
add4 = Add()([l, l_1sub])
l = AveragePooling2D(pool_size=(3, 3), strides=(1, 1),
padding='same')(add1)
add5 = Add()([l, add2])
result = concatenate([add3, add4, add5])
if is_tail:
result = Conv2D(filter_o, (1, 1),
strides=(1, 1),
padding='same',
name='%s_result1x1_align' % name,
trainable=trainable,
kernel_regularizer=OrthLocalReg2D)(result)
result = Activation('relu',
name='%s_result1x1_align_relu' % name)(result)
result = BatchNormalization(name='%s_result1x1_align_bn' %
name)(result)
return result
