def context_block(x, dilation_array, num_classes, init):
i = 0
for dil in dilation_array:
x = AtrousConvolution2D(num_classes,
3,
3,
atrous_rate=(dil, dil),
name='cb_3_{}'.format(i),
border_mode='same',
dim_ordering=dim_ordering,
init=init)(x)
x = Activation('relu')(x)
i = i + 1
x = AtrousConvolution2D(num_classes,
1,
1,
atrous_rate=(1, 1),
name='cb_final_conv',
border_mode='same',
dim_ordering=dim_ordering,
init=init)(x)
x = Activation('relu')(x)
return x
def sam_resnet(x):
# Dilated Convolutional Network
dcn = dcn_resnet(input_tensor=x[0])
conv_feat = Convolution2D(512, 3, 3, border_mode='same', activation='relu')(dcn.output)
# Attentive Convolutional LSTM
att_convlstm = Lambda(repeat, repeat_shape)(conv_feat)
att_convlstm = AttentiveConvLSTM(nb_filters_in=512, nb_filters_out=512, nb_filters_att=512,
nb_cols=3, nb_rows=3)(att_convlstm)
# Learned Prior (1)
priors1 = LearningPrior(nb_gaussian=nb_gaussian, init=gaussian_priors_init)(x[1])
concateneted = merge([att_convlstm, priors1], mode='concat', concat_axis=1)
learned_priors1 = AtrousConvolution2D(512, 5, 5, border_mode='same', activation='relu',
atrous_rate=(4, 4))(concateneted)
# Learned Prior (2)
priors2 = LearningPrior(nb_gaussian=nb_gaussian, init=gaussian_priors_init)(x[1])
concateneted = merge([learned_priors1, priors2], mode='concat', concat_axis=1)
learned_priors2 = AtrousConvolution2D(512, 5, 5, border_mode='same', activation='relu',
atrous_rate=(4, 4))(concateneted)
# Final Convolutional Layer
outs = Convolution2D(1, 1, 1, border_mode='same', activation='relu')(learned_priors2)
outs_up = Lambda(upsampling, upsampling_shape)(outs)
return [outs_up, outs_up, outs_up]
def get_correct_unit(inp, num_correct, num_units, dil_f, hidden_dim=64):
w_reg = L1_reg
# comment next line for deepcorrect
#hidden_dim = num_correct
inp_bn = BatchNormalization(mode=0,
axis=1,
trainable=True,
input_shape=(num_correct, img_height,
img_width))(inp)
layer1_c = Convolution2D(hidden_dim,
1,
1,
activation='linear',
init='he_normal',
W_regularizer=l1(w_reg),
trainable=True,
input_shape=(num_correct, img_height,
img_width))(inp_bn)
layer1_bn = BatchNormalization(mode=0, axis=1, trainable=True)(layer1_c)
layer1_act = Activation('relu')(layer1_bn)
layer2_pd = ZeroPadding2D((dil_f[0], dil_f[0]))(layer1_act)
layer2_c = AtrousConvolution2D(hidden_dim,
3,
3,
activation='linear',
init='he_normal',
atrous_rate=(dil_f[0], dil_f[0]),
W_regularizer=l1(w_reg),
trainable=True)(layer2_pd)
layer2_bn = BatchNormalization(mode=0, axis=1, trainable=True)(layer2_c)
layer2_act = Activation('relu')(layer2_bn)
if num_units > 1:
for stg_id in range(num_units - 1):
layer2_act = ZeroPadding2D(
(dil_f[stg_id + 1], dil_f[stg_id + 1]))(layer2_act)
layer2_act = AtrousConvolution2D(hidden_dim,
3,
3,
activation='linear',
init='he_normal',
atrous_rate=(dil_f[stg_id + 1],
dil_f[stg_id + 1]),
W_regularizer=l1(w_reg),
trainable=True)(layer2_act)
layer2_act = BatchNormalization(mode=0, axis=1,
trainable=True)(layer2_act)
layer2_act = Activation('relu')(layer2_act)
layer4_out = Convolution2D(num_correct,
1,
1,
activation='linear',
init='he_normal',
W_regularizer=l1(w_reg),
trainable=True)(layer2_act)
return layer4_out
def get_correct_unit_bottleneck(inp, inp_d, out_d, dil_f, num_units, stride=1):
set_trainable = True
w_reg = 0.00001
layer1_c = Convolution2D(inp_d,
1,
1,
activation='linear',
init='he_normal',
W_regularizer=l1(w_reg),
trainable=set_trainable)(inp)
layer1_bn = BatchNormalization(mode=0, axis=1,
trainable=set_trainable)(layer1_c)
layer1_act = Activation('relu')(layer1_bn)
layer2_pd = ZeroPadding2D((dil_f[0], dil_f[0]))(layer1_act)
layer2_c = AtrousConvolution2D(inp_d,
3,
3,
activation='linear',
init='he_normal',
atrous_rate=(dil_f[0], dil_f[0]),
W_regularizer=l1(w_reg),
trainable=True)(layer2_pd)
layer2_bn = BatchNormalization(mode=0, axis=1,
trainable=set_trainable)(layer2_c)
layer2_act = Activation('relu')(layer2_bn)
for unit_id in range(num_units - 1):
layer2_act = ZeroPadding2D(
(dil_f[unit_id + 1], dil_f[unit_id + 1]))(layer2_act)
layer2_act = AtrousConvolution2D(inp_d,
3,
3,
activation='linear',
init='he_normal',
atrous_rate=(dil_f[unit_id + 1],
dil_f[unit_id + 1]),
W_regularizer=l1(w_reg),
trainable=True)(layer2_act)
layer2_act = BatchNormalization(mode=0,
axis=1,
trainable=set_trainable)(layer2_act)
layer2_act = Activation('relu')(layer2_act)
unit_out = Convolution2D(out_d,
1,
1,
activation='linear',
init='he_normal',
W_regularizer=l1(w_reg),
trainable=set_trainable)(layer2_act)
return unit_out
def main(old_model_file, new_model_file):
if old_model_file.lower() != 'none':
old_model = load_model(old_model_file)
weights = [layer.get_weights() for layer in old_model.layers]
input_ = Input(shape=input_shape)
cv0 = Convolution2D(16, 3, 3, border_mode='same')(input_)
bn0 = BatchNormalization()(cv0)
act0 = Activation("relu")(bn0)
cv1 = AtrousConvolution2D(32, 3, 3, atrous_rate=(2, 2),
border_mode='same')(act0)
bn1 = BatchNormalization()(cv1)
act1 = Activation("relu")(bn1)
cv2 = AtrousConvolution2D(64, 3, 3, atrous_rate=(4, 4),
border_mode='same')(act1)
bn2 = BatchNormalization()(cv2)
act2 = Activation("relu")(bn2)
cv3 = AtrousConvolution2D(128,
3,
3,
atrous_rate=(8, 8),
border_mode='same')(act2)
bn3 = BatchNormalization()(cv3)
act3 = Activation("relu")(bn3)
output = AtrousConvolution2D(5,
3,
3,
atrous_rate=(16, 16),
border_mode='same',
activation='relu')(act3)
# output = Convolution2D(5,3,3,activation='relu',border_mode='same')(cv4)
opt = RMSprop(lr=0.0001)
# model = Model(input=input_, output=output)
model = Model(input=input_, output=output)
# for layer in model.layers[:23]:
# layer.trainable = False
model.compile(loss='mse', optimizer=opt, metrics=['accuracy'])
model.save(new_model_file)
def constructNet(input_dim=784,
n_hidden=1000,
n_out=1000,
nb_filter=50,
prob=0.5,
lr=0.0001):
nb_filters = 50
input_img = Input(shape=list(input_dim))
a = input_img
a1 = AtrousConvolution2D(nb_filters,
3,
3,
atrous_rate=(1, 1),
border_mode='same')(a)
b = AtrousConvolution2D(
nb_filters, 3, 3, atrous_rate=(1, 1), border_mode='same'
)(a
) #We only use the diagonal output from this, TODO: only filter diagonal
a2 = Lambda(GetDiag, output_shape=out_diag_shape)(b)
comb = merge([a1, a2], mode='sum')
comb = BatchNormalization()(comb)
a = Activation('relu')(comb)
l = 5
for i in range(1, l):
a1 = AtrousConvolution2D(nb_filters,
3,
3,
atrous_rate=(l, l),
border_mode='same')(a)
b = AtrousConvolution2D(
nb_filters, 3, 3, atrous_rate=(l, l), border_mode='same'
)(
a
) #We only use the diagonal output from this, TODO: only filter diagonal
a2 = Lambda(GetDiag, output_shape=out_diag_shape)(b)
comb = merge([a1, a2], mode='sum')
comb = BatchNormalization()(comb)
a = Activation('relu')(comb)
decoded = Convolution2D(1, 1, 1, activation='sigmoid',
border_mode='same')(a)
final = Flatten()(decoded)
model = Model(input_img, final)
model.summary()
model.compile(optimizer='adam', loss='binary_crossentropy')
return model
def __transition_up_block(ip, nb_filters, type='upsampling', output_shape=None, padding_param = 'same', weight_decay=1E-4):
''' SubpixelConvolutional Upscaling (factor = 2)
Args:
ip: keras tensor
nb_filters: number of layers
type: can be 'upsampling', 'subpixel', 'deconv', or 'atrous'. Determines type of upsampling performed
output_shape: required if type = 'deconv'. Output shape of tensor
weight_decay: weight decay factor
Returns: keras tensor, after applying upsampling operation.
'''
if type == 'upsampling':
x = UpSampling2D()(ip)
elif type == 'subpixel':
x = TimeDistributed(Conv2D(nb_filters, (3, 3), activation="relu", padding='same', kernel_regularizer=l2(weight_decay),
use_bias=False, kernel_initializer='he_uniform'))(ip)
x = SubPixelUpscaling(scale_factor=2)(x)
x = TimeDistributed(Conv2D(nb_filters, (3, 3), activation="relu", padding='same', kernel_regularizer=l2(weight_decay),
use_bias=False, kernel_initializer='he_uniform'))(x)
elif type == 'atrous':
# waiting on https://github.com/fchollet/keras/issues/4018
x = AtrousConvolution2D(nb_filters, (3, 3), activation="relu", kernel_regularizer=l2(weight_decay),
use_bias=False, atrous_rate=(2, 2), kernel_initializer='he_uniform')(ip)
else:
#x = Conv2DTranspose(nb_filters, (3, 3), activation='relu', padding=padding_param,
# strides=(2, 2), kernel_initializer='he_uniform')(ip)
x = TimeDistributed(Conv2DTranspose(nb_filters, (3, 3), activation='relu', padding=padding_param,
strides=(2, 2), kernel_initializer='he_uniform'))(ip)
return x
def __transition_up_block(ip, nb_filters, type='upsampling', output_shape=None, weight_decay=1E-4):
''' SubpixelConvolutional Upscaling (factor = 2)
Args:
ip: keras tensor
nb_filters: number of layers
type: can be 'upsampling', 'subpixel', 'deconv', or 'atrous'. Determines type of upsampling performed
output_shape: required if type = 'deconv'. Output shape of tensor
weight_decay: weight decay factor
Returns: keras tensor, after applying upsampling operation.
'''
if type == 'upsampling':
x = UpSampling2D()(ip)
elif type == 'subpixel':
x = Convolution2D(nb_filters, 3, 3, activation="relu", border_mode='same', W_regularizer=l2(weight_decay),
bias=False, init='he_uniform')(ip)
x = SubPixelUpscaling(scale_factor=2)(x)
x = Convolution2D(nb_filters, 3, 3, activation="relu", border_mode='same', W_regularizer=l2(weight_decay),
bias=False, init='he_uniform')(x)
elif type == 'atrous':
# waiting on https://github.com/fchollet/keras/issues/4018
x = AtrousConvolution2D(nb_filters, 3, 3, activation="relu", W_regularizer=l2(weight_decay),
bias=False, atrous_rate=(2, 2), init='he_uniform')(ip)
else:
x = Deconvolution2D(nb_filters, 3, 3, output_shape, activation='relu', border_mode='same',
subsample=(2, 2), init='he_uniform')(ip)
return x
def build_generator():
g_input = Input(shape=g_input_shape)
g1 = g_build_conv(g_input, 64, 5, strides=1)
g2 = g_build_conv(g1, 128, 4, strides=2)
g3 = g_build_conv(g2, 256, 4, strides=2)
g4 = g_build_conv(g3, 512, 4, strides=1)
g5 = g_build_conv(g4, 512, 4, strides=1)
g6 = g_build_conv(g5, 512, 4, strides=1, dilation=2)
g7 = g_build_conv(g6, 512, 4, strides=1, dilation=4)
g8 = g_build_conv(g7, 512, 4, strides=1, dilation=8)
g9 = g_build_conv(g8, 512, 4, strides=1, dilation=16)
g10 = g_build_conv(g9, 512, 4, strides=1)
g11 = g_build_conv(g10, 512, 4, strides=1)
g12 = g_build_deconv(g11, 256, 4, strides=2)
g13 = g_build_deconv(g12, 128, 4, strides=2)
g14 = g_build_conv(g13, 128, 4, strides=1)
g15 = g_build_conv(g14, 64, 4, strides=1)
g_output = AtrousConvolution2D(3,
kernel_size=4,
strides=(1, 1),
activation='tanh',
padding='same',
atrous_rate=(1, 1))(g15)
return Model(g_input, g_output)
def conv_block_atrous(input_tensor, kernel_size, filters, stage, block, atrous_rate=(2, 2)):
nb_filter1, nb_filter2, nb_filter3 = filters
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Convolution2D(nb_filter1, 1, 1, name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = AtrousConvolution2D(nb_filter2, kernel_size, kernel_size, border_mode='same',
atrous_rate=atrous_rate, name=conv_name_base + '2b')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
shortcut = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '1')(input_tensor)
shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)
x = add([x, shortcut])
x = Activation('relu')(x)
return x
def gaussian_prior_match(tensor, fdm):
# Learned Prior (1)
priors1 = LearningPrior(64, nb_gaussian=nb_gaussian)(fdm[1])
concateneted = concatenate([tensor, priors1], axis=1)
learned_priors1 = AtrousConvolution2D(64, [5, 5],
padding='same',
activation='relu',
atrous_rate=(4, 4))(concateneted)
# Learned Prior (2)
priors2 = LearningPrior(64, nb_gaussian=nb_gaussian)(fdm[1])
concateneted = concatenate([learned_priors1, priors2], axis=1)
learned_priors2 = AtrousConvolution2D(64, [5, 5],
padding='same',
activation='relu',
atrous_rate=(4, 4))(concateneted)
return learned_priors2
def create_classifier(body, data, n_classes, l2_reg=0.):
# Include last layers
top = BatchNormalization(mode=0, axis=channel_idx, name="bn7")(body)
top = Activation('relu', name="relu7")(top)
top = AtrousConvolution2D(512,
3,
3,
'he_normal',
atrous_rate=(12, 12),
border_mode='same',
name="conv6a",
W_regularizer=l2(l2_reg))(top)
top = Activation('relu', name="conv6a_relu")(top)
name = "hyperplane_num_cls_%d_branch_%d" % (n_classes, 12)
def my_init(shape, name=None, dim_ordering='th'):
return initializations.normal(shape, scale=0.01, name=name)
top = AtrousConvolution2D(n_classes,
3,
3,
my_init,
atrous_rate=(12, 12),
border_mode='same',
name=name,
W_regularizer=l2(l2_reg))(top)
top = Deconvolution2D(n_classes,
16,
16,
top._keras_shape,
bilinear_init,
'linear',
border_mode='valid',
subsample=(8, 8),
bias=False,
name="upscaling_" + str(n_classes),
W_regularizer=l2(l2_reg))(top)
top = CropLayer2D(data, name='score')(top)
top = NdSoftmax()(top)
return top
def DAC(x):
#Branch 1
x1 = AtrousConvolution2D(256, 3, 3,atrous_rate=(1,1),border_mode='same')(x)
#Branch 2
x2_1 = AtrousConvolution2D(256, 3, 3,atrous_rate=(3,3),border_mode='same')(x)
x2_2 = AtrousConvolution2D(256, 1, 1,atrous_rate=(1,1),border_mode='same')(x2_1)
#Branch 3
x3_1 = AtrousConvolution2D(256, 3, 3,atrous_rate=(1,1),border_mode='same')(x)
x3_2 = AtrousConvolution2D(256, 3, 3,atrous_rate=(3,3),border_mode='same')(x3_1)
x3_3 = AtrousConvolution2D(256, 1, 1,atrous_rate=(1,1),border_mode='same')(x3_2)
#Branch 4
x4_1 = AtrousConvolution2D(256, 3, 3,atrous_rate=(1,1),border_mode='same')(x)
x4_2 = AtrousConvolution2D(256, 3, 3,atrous_rate=(3,3),border_mode='same')(x4_1)
x4_3 = AtrousConvolution2D(256, 3, 3,atrous_rate=(5,5),border_mode='same')(x4_2)
x4_4 = AtrousConvolution2D(256, 1, 1,atrous_rate=(1,1),border_mode='same')(x4_3)
x = add([x1,x2_2,x3_3,x4_4,x])
x = BatchNormalization()(x)
x = Dropout(0.2)(x)
return x
def atrous_block(x, block_idx, nb_filter, nb_conv):
# 1st conv
for i in range(nb_conv):
name = "block%s_conv2D_%s" % (block_idx, i)
x = AtrousConvolution2D(nb_filter, 3, 3, name=name,
border_mode="same")(x)
if i == nb_conv - 1:
x = BatchNormalization(mode=2, axis=1)(x)
x = Activation("relu")(x)
return x
def atrous(layer_input, filters, f_size=4, bn=True):
a_list = []
for rate in [2, 4, 8]:
a = AtrousConvolution2D(filters,
f_size,
atrous_rate=rate,
border_mode='same')(layer_input)
a_list.append(a)
a = Concatenate()(a_list)
a = LeakyReLU(alpha=0.2)(a)
if bn:
#a = BatchNormalization(momentum=0.8)(a)
a = InstanceNormalization()(a)
return a
def bottleneck(inp, output, internal_scale=4, use_relu=True, asymmetric=0, dilated=0, downsample=False, dropout_rate=0.1):
# main branch
internal = output / internal_scale
encoder = inp
## 1x1
input_stride = 2 if downsample else 1 # the first 1x1 projection is replaced with a 2x2 convolution when downsampling
encoder = Convolution2D(internal, input_stride, input_stride, border_mode='same', subsample=(input_stride, input_stride), bias=False)(encoder)
## Batch normalization + PReLU
encoder = BatchNormalization(momentum=0.1)(encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = PReLU(shared_axes=[1, 2])(encoder)
## conv
if not asymmetric and not dilated:
encoder = Convolution2D(nb_filter=internal, nb_row=3, nb_col=3, border_mode='same')(encoder)
elif asymmetric:
encoder = Convolution2D(nb_filter=internal, nb_row=1, nb_col=asymmetric, border_mode='same', bias=False)(encoder)
encoder = Convolution2D(nb_filter=internal, nb_row=asymmetric, nb_col=1, border_mode='same')(encoder)
elif dilated:
encoder = AtrousConvolution2D(nb_filter=internal, nb_row=3, nb_col=3, atrous_rate=(dilated, dilated), border_mode='same')(encoder)
else:
raise(Exception('You shouldn\'t be here'))
## Batch normalization + PReLU
encoder = BatchNormalization(momentum=0.1)(encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = PReLU(shared_axes=[1, 2])(encoder)
## 1x1
encoder = Convolution2D(nb_filter=output, nb_row=1, nb_col=1, border_mode='same', bias=False)(encoder)
## Batch normalization + Spatial dropout
encoder = BatchNormalization(momentum=0.1)(encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = SpatialDropout2D(dropout_rate)(encoder)
other = inp
# other branch
if downsample:
other = MaxPooling2D()(other)
other = Permute((1, 3, 2))(other)
pad_featmaps = output - inp.get_shape().as_list()[3]
other = ZeroPadding2D(padding=(0, 0, 0, pad_featmaps))(other)
other = Permute((1, 3, 2))(other)
encoder = merge([encoder, other], mode='sum')
encoder = PReLU(shared_axes=[1, 2])(encoder)
return encoder
def g_build_conv(layer_input,
filter_size,
kernel_size=4,
strides=2,
activation='leakyrelu',
dropout_rate=g_dropout,
norm='inst',
dilation=1):
c = AtrousConvolution2D(filter_size,
kernel_size=kernel_size,
strides=strides,
atrous_rate=(dilation, dilation),
padding='same')(layer_input)
if activation == 'leakyrelu':
c = Activation("relu")(c)
if dropout_rate:
c = Dropout(dropout_rate)(c)
if norm == 'inst':
c = InstanceNormalization()(c)
return c
def bn_relu_conv(inputs,
n_filters,
filter_size,
stride,
dropout,
use_bias,
dilation=1,
name=None,
l2_reg=0.):
first = BatchNormalization(mode=0, axis=channel_idx,
name="bn" + name)(inputs)
first = Activation('relu', name="res" + name + "_relu")(first)
if dropout > 0:
first = Dropout(dropout, name="res" + name + "_dropout")(first)
if dilation == 1:
second = Convolution2D(n_filters,
filter_size,
filter_size,
'he_normal',
subsample=(stride, stride),
bias=use_bias,
border_mode='same',
name="res" + name,
W_regularizer=l2(l2_reg))(first)
else:
second = AtrousConvolution2D(n_filters,
filter_size,
filter_size,
'he_normal',
subsample=(stride, stride),
atrous_rate=(dilation, dilation),
border_mode='same',
bias=use_bias,
name="res" + name,
W_regularizer=l2(l2_reg))(first)
return first, second
def build_residual_block(input_shape, n_feature_maps, kernel_sizes=None, n_skip=2, is_subsample=False, subsample=None, verbose=1):
'''
[1] Building block of layers for residual learning.
Code based on https://github.com/ndronen/modeling/blob/master/modeling/residual.py
, but modification of (perhaps) incorrect relu(f)+x thing and it's for conv layer
[2] MaxPooling is used instead of strided convolution to make it easier
to set size(output of short-cut) == size(output of conv-layers).
If you want to remove MaxPooling,
i) change (border_mode in Convolution2D in shortcut), 'same'-->'valid'
ii) uncomment ZeroPadding2D in conv layers.
(Then the following Conv2D is not the first layer of this container anymore,
so you can remove the input_shape in the line 101, the line with comment #'OPTION' )
[3] It can be used for both cases whether it subsamples or not.
[4] In the short-cut connection, I used 1x1 convolution to increase #channel.
It occurs when is_expand_channels == True
input_shape = (None, num_channel, height, width)
n_feature_maps : number of feature maps. In ResidualNet it increases whenever image is downsampled.
kernel_sizes : list or tuple, (3,3) or [3,3] for example
n_skip : number of layers to skip
is_subsample : If it is True, the layers subsamples by *subsample* to reduce the size.
subsample : tuple, (2,2) or (1,2) for example. Used only if is_subsample==True
'''
is_expand_channels = not (input_shape[0] == n_feature_maps)
kernel_row, kernel_col = kernel_sizes
if verbose:
# ***** VERBOSE_PART *****
print(' - New residual block with')
print(' input shape: ', input_shape)
print(' kernel size: ', kernel_sizes)
# is_expand_channels == True when num_channels increases.
# E.g. the very first residual block (e.g. 1->64, 3->128, 128->256, ...)
if is_expand_channels:
print(' - Input channels: %d ---> num feature maps on out: %d' % (input_shape[0], n_feature_maps) )
if is_subsample:
print(' - with subsample:', subsample)
# set input
x = Input(shape=(input_shape))
# ***** SHORTCUT PATH *****
if is_subsample: # subsample (+ channel expansion if needed)
shortcut_y = Convolution2D(n_feature_maps, kernel_sizes[0], kernel_sizes[1],
subsample=subsample,
border_mode='valid')(x)
else: # channel expansion only (e.g. the very first layer of the whole networks)
if is_expand_channels:
shortcut_y = Convolution2D(n_feature_maps, 1, 1, border_mode='valid')(x)
else:
# if no subsample and no channel expension, there's nothing to add on the shortcut.
shortcut_y = x
# ***** CONVOLUTION_PATH *****
conv_y = x
#for i in range(n_skip):
# conv_y = BatchNormalization(axis=1, mode=2)(conv_y)
# conv_y = Activation('relu')(conv_y)
# if i==0 and is_subsample: # [Subsample at layer 0 if needed]
# conv_y = Convolution2D(n_feature_maps, kernel_row, kernel_col,
# subsample=subsample,
# border_mode='valid')(conv_y)
# else:
# conv_y = Convolution2D(n_feature_maps, kernel_row, kernel_col, border_mode='same')(conv_y)
conv_y = AtrousConvolution2D(n_feature_maps, kernel_row, kernel_col, atrous_rate=(2, 2), border_mode='same')(conv_y)
conv_y = Convolution2D(n_feature_maps, 1, 1)(x)
conv_y = (conv_y)
# output
y = merge([shortcut_y, conv_y], mode='sum')
block = Model(input=x, output=y)
if verbose:
print(' -- model was built.')
return block
def dcn_vgg(input_tensor=None):
input_shape = (3, None, None)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# conv_1
x = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='block1_conv1')(img_input)
x = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='block1_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# conv_2
x = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='block2_conv1')(x)
x = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='block2_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# conv_3
x = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='block3_conv1')(x)
x = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='block3_conv2')(x)
x = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='block3_conv3')(x)
x = MaxPooling2D((2, 2),
strides=(2, 2),
name='block3_pool',
border_mode='same')(x)
# conv_4
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block4_conv1')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block4_conv2')(x)
x = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block4_conv3')(x)
x = MaxPooling2D((2, 2),
strides=(1, 1),
name='block4_pool',
border_mode='same')(x)
# conv_5
x = AtrousConvolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block5_conv1',
atrous_rate=(2, 2))(x)
x = AtrousConvolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block5_conv2',
atrous_rate=(2, 2))(x)
x = AtrousConvolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='block5_conv3',
atrous_rate=(2, 2))(x)
# Create model
model = Model(img_input, x)
# Load weights
weights_path = 'models/vgg16_weights_th_dim_ordering_th_kernels_notop.h5'
model.load_weights(weights_path)
return model
def main(old_model_file, new_model_file):
if old_model_file.lower() != 'none':
old_model = load_model(old_model_file)
weights = [layer.get_weights() for layer in old_model.layers]
input_ = Input(shape=input_shape)
input_prev = Input(shape=input_alt_shape)
merged_input = merge([input_, input_prev], mode='concat')
#Branch 1
cv0 = Convolution2D(32, 3, 3, border_mode='same')(merged_input)
bn0 = BatchNormalization()(cv0)
act0 = Activation("relu")(bn0)
cv1 = AtrousConvolution2D(32, 3, 3, atrous_rate=(2, 2),
border_mode='same')(act0)
bn1 = BatchNormalization()(cv1)
act1 = Activation("relu")(bn1)
cv2 = AtrousConvolution2D(32, 3, 3, atrous_rate=(4, 4),
border_mode='same')(act1)
bn2 = BatchNormalization()(cv2)
act2 = Activation("relu")(bn2)
cv3 = AtrousConvolution2D(32, 3, 3, atrous_rate=(8, 8),
border_mode='same')(act2)
bn3 = BatchNormalization()(cv3)
act3 = Activation("relu")(bn3)
#Branch 2
cv0_2 = Convolution2D(32, 3, 3, border_mode='same',
activation='relu')(merged_input)
cv1_2 = AtrousConvolution2D(32,
3,
3,
atrous_rate=(2, 2),
border_mode='same',
activation='relu')(cv0_2)
cv2_2 = AtrousConvolution2D(32,
3,
3,
atrous_rate=(4, 4),
border_mode='same',
activation='relu')(cv1_2)
cv3_2 = AtrousConvolution2D(32,
3,
3,
atrous_rate=(8, 8),
border_mode='same',
activation='relu')(cv2_2)
#Branch 3
cv0_11 = Convolution2D(4, 1, 1, activation='relu',
border_mode='same')(merged_input)
cv0_33 = Convolution2D(8, 3, 3, activation='relu',
border_mode='same')(merged_input)
cv0_77 = Convolution2D(4, 9, 9, activation='relu',
border_mode='same')(merged_input)
merged_0 = merge([cv0_11, cv0_33, cv0_77], mode='concat')
cv1_11 = Convolution2D(4, 1, 1, activation='relu',
border_mode='same')(merged_0)
cv1_33 = Convolution2D(8, 3, 3, activation='relu',
border_mode='same')(merged_0)
cv1_77 = Convolution2D(4, 9, 9, activation='relu',
border_mode='same')(merged_0)
merged_1 = merge([cv1_11, cv1_33, cv1_77], mode='concat')
cv2_3 = Convolution2D(32, 5, 5, activation='relu',
border_mode='same')(merged_1)
cv3_3 = Convolution2D(32, 5, 5, activation='relu',
border_mode='same')(cv2_3)
cv4_3 = Convolution2D(5, 3, 3, border_mode='same')(cv3_3)
merged_final = merge([act3, cv3_2, cv4_3], mode='concat')
output = AtrousConvolution2D(5,
3,
3,
atrous_rate=(16, 16),
border_mode='same',
activation='relu')(merged_final)
# Branch
opt = RMSprop(lr=0.0001)
# model = Model(input=input_, output=output)
model = Model(input=[input_, input_prev], output=output)
# for layer in model.layers[:23]:
# layer.trainable = False
model.compile(loss='mse', optimizer=opt, metrics=['accuracy'])
model.save(new_model_file)
def build_dilation(img_shape=(3, None, None),
nclasses=11,
l2_reg=0.,
init='glorot_uniform',
path_weights=None,
load_pretrained=False,
freeze_layers_from=None,
vgg_weights=True):
# Build network
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
# CONTRACTING PATH
# Input layer
inputs = Input(img_shape)
# padded = ZeroPadding2D(padding=(100, 100), name='pad100')(inputs)
# Block1
conv1_1 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='same',
name='block1_conv1',
W_regularizer=l2(l2_reg))(inputs)
conv1_2 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='same',
name='block1_conv2',
W_regularizer=l2(l2_reg))(conv1_1)
pool1 = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(conv1_2)
# Block2
conv2_1 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='same',
name='block2_conv1',
W_regularizer=l2(l2_reg))(pool1)
conv2_2 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='same',
name='block2_conv2',
W_regularizer=l2(l2_reg))(conv2_1)
pool2 = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(conv2_2)
# Block3
conv3_1 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='block3_conv1',
W_regularizer=l2(l2_reg))(pool2)
conv3_2 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='block3_conv2',
W_regularizer=l2(l2_reg))(conv3_1)
conv3_3 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='block3_conv3',
W_regularizer=l2(l2_reg))(conv3_2)
pool3 = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(conv3_3)
# Block4
conv4_1 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='block4_conv1',
W_regularizer=l2(l2_reg))(pool3)
conv4_2 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='block4_conv2',
W_regularizer=l2(l2_reg))(conv4_1)
conv4_3 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='block4_conv3',
W_regularizer=l2(l2_reg))(conv4_2)
vgg_base_model = Model(input=inputs, output=conv4_3)
vgg_base_in = vgg_base_model.output
# Block5
conv5_bn = BatchNormalization(axis=bn_axis, name='block5_bn')(vgg_base_in)
conv5_1 = AtrousConvolution2D(512,
3,
3,
atrous_rate=(2, 2),
name='atrous_conv_5_1',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv5_bn)
conv5_1_relu = Activation('relu')(conv5_1)
conv5_bn_2 = BatchNormalization(axis=bn_axis,
name='block5_bn2')(conv5_1_relu)
conv5_2 = AtrousConvolution2D(512,
3,
3,
atrous_rate=(2, 2),
name='atrous_conv_5_2',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv5_bn_2)
conv5_2_relu = Activation('relu')(conv5_2)
conv5_bn3 = BatchNormalization(axis=bn_axis,
name='block5_bn3')(conv5_2_relu)
conv5_3 = AtrousConvolution2D(512,
3,
3,
atrous_rate=(2, 2),
name='atrous_conv_5_3',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv5_bn3)
conv5_3_relu = Activation('relu')(conv5_3)
# Block6
conv6_bn = BatchNormalization(axis=bn_axis, name='block6_bn')(conv5_3_relu)
conv6 = AtrousConvolution2D(1024,
7,
7,
atrous_rate=(4, 4),
name='atrous_conv_6',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv6_bn)
conv6_relu = Activation('relu')(conv6)
conv6_relu = Dropout(0.5)(conv6_relu)
# Block7
conv7_bn = BatchNormalization(axis=bn_axis, name='block7_bn')(conv6_relu)
conv7 = AtrousConvolution2D(4096,
1,
1,
atrous_rate=(1, 1),
name='atrous_conv_7',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv7_bn)
conv7_relu = Activation('relu')(conv7)
conv7_relu = Dropout(0.5)(conv7_relu)
# Final block
convf_bn = BatchNormalization(axis=bn_axis, name='blockf_bn')(conv7_relu)
x = AtrousConvolution2D(nclasses,
1,
1,
atrous_rate=(1, 1),
name='final_block',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(convf_bn)
# Appending context block
upsampling = 8
context_out = context_block(x, [1, 1, 2, 4, 8, 16, 1],
nclasses,
init=identity_init)
deconv_out = Deconvolution2D(
nclasses,
upsampling,
upsampling,
init=bilinear_init,
subsample=(upsampling, upsampling),
input_shape=context_out._keras_shape)(context_out)
# Softmax
prob = NdSoftmax()(deconv_out)
# Complete model
model = Model(input=vgg_base_model.input, output=prob)
# Load pretrained weights VGG part of the model
if vgg_weights == True:
if K.image_dim_ordering() == 'th':
weights_path = get_file(
'vgg19_weights_th_dim_ordering_th_kernels_notop.h5',
TH_WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
else:
weights_path = get_file(
'vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5',
TF_WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
print('Loaded pre-trained VGG weights')
if path_weights:
print('Loaded pre-trained weights for the full network')
model.load_weights(weights_path, by_name=True)
# load_matcovnet(model, path_weights, n_classes=nclasses)
# Freeze some layers
if freeze_layers_from is not None:
freeze_layers(model, freeze_layers_from)
return model
def base_model(input_shape, output_shape):
routings = 3
input_x = Input(input_shape)
_routings = ROUTING
_digitcap_num = NUM_CLASSES
_digitcap_dim = DIGIT_CAP_DIM
_digitcap_dimm = DIGIT_CAP_DIM2
nb_filters = 64
######################### ENCODER LAYER ###################################
## First Convolution Block, consist of 4 Convs and 1 PrimaryCaps with short skips
#
x = enhance()(input_x)
#input_x = improved_clahe()(input_x)
#print('clahe', input_x)
conv1 = AtrousConvolution2D(32,
1,
1,
atrous_rate=(2, 2),
border_mode='same',
kernel_initializer='he_normal')(x)
act0 = Activation('tanh')(conv1)
bn0 = layers.BatchNormalization(epsilon=1e-06,
mode=0,
momentum=0.9,
weights=None)(act0)
mx0 = MaxPool2D(2, strides=2, padding='same')(bn0)
conv2 = AtrousConvolution2D(32,
1,
1,
atrous_rate=(4, 4),
border_mode='same',
kernel_initializer='he_normal')(x)
act1 = Activation('tanh')(conv2)
bn1 = layers.BatchNormalization(epsilon=1e-06,
mode=0,
momentum=0.9,
weights=None)(act1)
mx1 = MaxPool2D(2, strides=2, padding='same')(bn1)
conv3 = AtrousConvolution2D(32,
1,
1,
atrous_rate=(6, 6),
border_mode='same',
kernel_initializer='he_normal')(x)
act2 = Activation('tanh')(conv3)
bn2 = layers.BatchNormalization(epsilon=1e-06,
mode=0,
momentum=0.9,
weights=None)(act2)
mx2 = MaxPool2D(2, strides=2, padding='same')(bn2)
con1 = layers.concatenate([mx0, mx1, mx2], axis=1)
conv4 = Conv2D(64,
3,
strides=2,
padding='valid',
activation='tanh',
kernel_initializer='he_normal',
name='lane_1_Conv2')(con1)
bn3 = layers.BatchNormalization(epsilon=1e-06,
mode=0,
momentum=0.9,
weights=None)(conv4)
mx3 = MaxPool2D(2, strides=2, padding='same')(bn3)
conv5 = Conv2D(64,
3,
strides=2,
padding='valid',
activation='tanh',
kernel_initializer='he_normal',
name='lane_2_Conv2')(con1)
bn4 = layers.BatchNormalization(epsilon=1e-06,
mode=0,
momentum=0.9,
weights=None)(conv5)
mx4 = MaxPool2D(2, strides=2, padding='same')(bn4)
conv6 = Conv2D(64,
3,
strides=2,
padding='valid',
activation='tanh',
kernel_initializer='he_normal',
name='lane_3_Conv2')(con1)
bn5 = layers.BatchNormalization(epsilon=1e-06,
mode=0,
momentum=0.9,
weights=None)(conv6)
mx5 = MaxPool2D(2, strides=2, padding='same')(bn5)
#con2 = layers.concatenate([bn3, bn4], axis=1)
#con3 = layers.concatenate([con2, bn5], axis=1)
con2 = layers.concatenate([mx3, mx4], axis=1)
con3 = layers.concatenate([con2, mx5], axis=1)
conv7 = Conv2D(128,
1,
strides=1,
padding='valid',
activation='tanh',
kernel_initializer='he_normal',
name='lane_1_Conv3')(con2)
conv8 = Conv2D(128,
1,
strides=1,
padding='valid',
activation='tanh',
kernel_initializer='he_normal',
name='lane_2_Conv3')(con2)
#conv9 = AtrousConvolution2D(128, 3, 3, atrous_rate=(1,1), border_mode='same')(con3)
#act4 = Activation('relu')(conv9)
#bn5 = layers.BatchNormalization(epsilon=1e-06, mode=0, momentum=0.9, weights=None)(act4)
#mx5 = MaxPool2D(2, strides = 2, padding='same')(bn5)
conv9 = Conv2D(128,
1,
strides=1,
padding='valid',
activation='tanh',
kernel_initializer='he_normal',
name='lane_3_Conv3')(con3)
con4 = layers.concatenate([conv7, conv8], axis=1)
#print('con4:', con4)
#con5 = layers.concatenate([con4, mx5], axis=1)
#print('con4', con5)
######################### DECODER LAYER ###################################
conv10 = Conv2D(128,
1,
strides=1,
padding='valid',
activation='tanh',
kernel_initializer='he_normal',
name='Conv4')(con4)
bn3 = layers.BatchNormalization(epsilon=1e-06,
mode=0,
momentum=0.9,
weights=None)(conv10)
con6 = layers.concatenate([conv9, bn3], axis=1)
#primarycaps1 = PrimaryCap(conv10, dim_capsule=8, n_channels=8, kernel_size=3,strides=2,padding='valid')
## Define the Primary capsule (PrimaryCaps)
primarycaps = PrimaryCap(con6,
dim_capsule=8,
n_channels=32,
kernel_size=3,
strides=2,
padding='valid')
#primarycaps = PrimaryCap(bn7, dim_capsule_attr=1, num_capsule=32, kernel_size=3, strides=2, padding='valid')
#print('Primary_Caps',primarycaps)
#con6 = layers.concatenate([primarycaps1, primarycaps], axis=1)
#pc2 = PrimaryCap(output, dim_capsule=8, n_channels=16, kernel_size=7,strides=2,padding='valid')
#output = concatenate([pc1, pc2])
#attn_caps = CAN(num_capsule=NUM_CLASSES, dim_capsule_attr=1, routings=routings, num_instance=n_instance, num_part=n_part,
#name='digitcaps')(primarycaps)
#print('attn_caps', attn_caps)
x = CapsuleLayer(num_capsule=_digitcap_num,
dim_capsule=_digitcap_dim,
routings=_routings,
name='digitcaps')(primarycaps)
#x1 = CapsuleLayer(num_capsule = 16, dim_capsule = _digitcap_dim,routings = _routings,name='digitcaps')(primarycaps)
#x = layers.concatenate([x0,x1], axis=1)
#x = layers.concatenate([x, attn_caps])
#print('concat_x',x)
#####################################################
digitcaps = x
print("digitcaps", digitcaps.shape)
x = CapsLength(name='capsnet')(digitcaps)
y_pred = x
print('y_pred', y_pred)
#y_pred=svmclf.predict(digitcaps)
# For 'reconstruction' and also as a 'regulerizer',
# we get y label as an input as well
y_label = Input(shape=output_shape)
print("y_label", y_label.shape)
true_digitcap = Mask()([digitcaps, y_label])
maxlen_digitcap = Mask()(digitcaps)
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(
layers.Dense(512,
input_dim=_digitcap_dim * _digitcap_num,
init='he_normal')) #(digitcaps) init='uniform'
decoder.add(
BatchNormalization(epsilon=1e-06, mode=0, momentum=0.9, weights=None))
decoder.add(Activation('tanh'))
decoder.add(layers.Dropout(0.4))
#decoder.add(layers.Batch_Normalization())
decoder.add(layers.Dense(1024, init='he_normal'))
decoder.add(
BatchNormalization(epsilon=1e-06, mode=0, momentum=0.9, weights=None))
decoder.add(Activation('tanh'))
decoder.add(layers.Dropout(0.4))
#decoder.add(layers.BatchNormalization())
decoder.add(layers.Dense(np.prod(input_shape), init='he_normal'))
decoder.add(
BatchNormalization(epsilon=1e-06, mode=0, momentum=0.9, weights=None))
decoder.add(Activation('sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
#train_model = models.Model([input_x], [out_caps, decoder(out_pose)]) ## New line added
#eval_model = models.Model([input_x], [out_caps,decoder(out_pose)])
# Models for training and evaluation (prediction)
train_model = models.Model([input_x, y_label],
[y_pred, decoder(true_digitcap)])
eval_model = models.Model(input_x, [y_pred, decoder(maxlen_digitcap)])
pc_model = models.Model(input_x, primarycaps)
digitcaps_model = models.Model(input_x, digitcaps)
# summary
train_model.summary()
'''
noise = Input(shape=(output_shape, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
#output = Conv2D(n_classes, 1)(output)
'''
#output = GlobalAvgPool2D()(output)
#output = Activation('softmax')(output)
#model = Model(input, output)
return train_model, eval_model, digitcaps_model, decoder