def identity_block(input_tensor, kernel_size, filters, stage, block):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = layers.Conv2D(filters1, (1, 1),
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2a')(input_tensor)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2a')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters2,
kernel_size,
use_bias=False,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2b')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2b')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters3, (1, 1),
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2c')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = layers.Activation('relu')(x)
return x
def identity_building_block(input_tensor,
kernel_size,
filters,
stage,
block,
training=None):
"""The identity block is the block that has no conv layer at shortcut.
Arguments:
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: current block label, used for generating layer names
training: Only used if training keras model with Estimator. In other
scenarios it is handled automatically.
Returns:
Output tensor for the block.
"""
filters1, filters2 = filters
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = layers.Conv2D(filters1,
kernel_size,
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2a')(input_tensor)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2a')(x,
training=training)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters2,
kernel_size,
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2b')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2b')(x,
training=training)
x = layers.add([x, input_tensor])
x = layers.Activation('relu')(x)
return x
def residual_block(inputs, filters):
x = conv2d_unit(inputs, filters, (1, 1))
x = conv2d_unit(x, 2 * filters, (3, 3))
x = add([inputs, x])
x = Activation('linear')(x)
return x
def _xception_block(inputs,
depth_list,
prefix,
skip_connection_type,
stride,
rate=1,
depth_activation=False,
return_skip=False):
""" Basic building block of modified Xception network
Args:
inputs: input tensor
depth_list: number of filters in each SepConv layer. len(depth_list) == 3
prefix: prefix before name
skip_connection_type: one of {'conv','sum','none'}
stride: stride at last depthwise conv
rate: atrous rate for depthwise convolution
depth_activation: flag to use activation between depthwise & pointwise convs
return_skip: flag to return additional tensor after 2 SepConvs for decoder
"""
residual = inputs
for i in range(3):
residual = SepConv_BN(
residual, #一种新的conv网络
depth_list[i],
prefix + '_separable_conv{}'.format(i + 1),
stride=stride if i == 2 else 1,
rate=rate,
depth_activation=depth_activation)
if i == 1:
skip = residual
if skip_connection_type == 'conv':
shortcut = _conv2d_same(inputs,
depth_list[-1],
prefix + '_shortcut',
kernel_size=1,
stride=stride)
shortcut = BatchNormalization(name=prefix + '_shortcut_BN')(shortcut)
outputs = layers.add([residual, shortcut])
elif skip_connection_type == 'sum':
outputs = layers.add([residual, inputs])
elif skip_connection_type == 'none':
outputs = residual
if return_skip:
return outputs, skip
else:
return outputs
def build_model(fragment_length, nb_filters, nb_output_bins, dilation_depth, nb_stacks, use_skip_connections,
learn_all_outputs, _log, desired_sample_rate, use_bias, res_l2, final_l2):
def residual_block(x):
original_x = x
# TODO: initalization, regularization?
# Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet.
tanh_out = CausalAtrousConvolution1D(nb_filters, 2, dilation_rate=2 ** i, padding='valid', causal=True,
use_bias=use_bias,
name='dilated_conv_%d_tanh_s%d' % (2 ** i, s), activation='tanh',
kernel_regularizer=l2(res_l2))(x)
sigm_out = CausalAtrousConvolution1D(nb_filters, 2, dilation_rate=2 ** i, padding='valid', causal=True,
use_bias=use_bias,
name='dilated_conv_%d_sigm_s%d' % (2 ** i, s), activation='sigmoid',
kernel_regularizer=l2(res_l2))(x)
x = layers.Multiply(name='gated_activation_%d_s%d' % (i, s))([tanh_out, sigm_out])
res_x = layers.Convolution1D(nb_filters, 1, padding='same', use_bias=use_bias,
kernel_regularizer=l2(res_l2))(x)
skip_x = layers.Convolution1D(nb_filters, 1, padding='same', use_bias=use_bias,
kernel_regularizer=l2(res_l2))(x)
res_x = layers.add([original_x, res_x])
return res_x, skip_x
input = Input(shape=(fragment_length, nb_output_bins), name='input_part')
out = input
skip_connections = []
out = CausalAtrousConvolution1D(nb_filters, 2,
dilation_rate=1,
padding='valid',
causal=True,
name='initial_causal_conv'
)(out)
for s in range(nb_stacks):
for i in range(0, dilation_depth + 1):
out, skip_out = residual_block(out)
skip_connections.append(skip_out)
if use_skip_connections:
out = layers.add(skip_connections)
out = layers.Activation('relu')(out)
out = layers.Convolution1D(nb_output_bins, 1, padding='same',
kernel_regularizer=l2(final_l2))(out)
out = layers.Activation('relu')(out)
out = layers.Convolution1D(nb_output_bins, 1, padding='same')(out)
if not learn_all_outputs:
raise DeprecationWarning('Learning on just all outputs is wasteful, now learning only inside receptive field.')
out = layers.Lambda(lambda x: x[:, -1, :], output_shape=(out._keras_shape[-1],))(
out) # Based on gif in deepmind blog: take last output?
out = layers.Activation('softmax', name="output_softmax")(out)
model = Model(input, out)
receptive_field, receptive_field_ms = compute_receptive_field()
_log.info('Receptive Field: %d (%dms)' % (receptive_field, int(receptive_field_ms)))
return model
def conv_block(input_tensor, kernel_size, filters, stride, stage, block, strides=(2, 2), trainable=True):
"""A block that has a conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: filters
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
Note that from stage 3, the first conv layer at main path is with strides=(2,2)
And the shortcut should have strides=(2,2) as well
"""
bn_axis = 3
conv_name_1 = 'conv' + str(stage) + '_' + str(block) + '_3x3_1'
bn_name_1 = 'conv' + str(stage) + '_' + str(block) + '_3x3_1/bn'
x = Conv2D(filters, (3, 3),
strides=(stride, stride),
padding='same',
kernel_initializer=tf.initializers.glorot_normal(),
use_bias=False,
kernel_regularizer=l2(WEIGHT_DECAY),
trainable=trainable,
name=conv_name_1)(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_1)(x)
x = Activation('relu')(x)
conv_name_2 = 'conv' + str(stage) + '_' + str(block) + '_3x3_2'
bn_name_2 = 'conv' + str(stage) + '_' + str(block) + '_3x3_2/bn'
x = Conv2D(filters, kernel_size,
strides=(1, 1),
padding='same',
kernel_initializer=tf.initializers.glorot_normal(),
use_bias=False,
kernel_regularizer=l2(WEIGHT_DECAY),
trainable=trainable,
name=conv_name_2)(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_2)(x)
conv_name_3 = 'conv' + str(stage) + '_' + str(block) + '_3x3_proj'
bn_name_3 = 'conv' + str(stage) + '_' + str(block) + '_3x3_proj/bn'
shortcut = Conv2D(filters, (1, 1),
strides=(stride, stride),
padding='same',
kernel_initializer=tf.initializers.glorot_normal(),
use_bias=False,
kernel_regularizer=l2(WEIGHT_DECAY),
trainable=trainable,
name=conv_name_3)(input_tensor)
shortcut = BatchNormalization(axis=bn_axis, name=bn_name_3)(shortcut)
x = layers.add([x, shortcut])
x = Activation('relu')(x)
return x
def identity_block(input_tensor, kernel_size, filters, stage, block):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = layers.Conv2D(filters1, (1, 1), use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2a')(input_tensor)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2a')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters2, kernel_size,
padding='same', use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2b')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2b')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters3, (1, 1), use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2c')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = layers.Activation('relu')(x)
return x
def conv_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2, 2)):
"""A block that has a conv layer at shortcut.
Arguments:
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
strides: Strides for the first conv layer in the block.
Returns:
Output tensor for the block.
Note that from stage 3,
the first conv layer at main path is with strides=(2, 2)
And the shortcut should have strides=(2, 2) as well
"""
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2D(filters1, (1, 1), strides=strides,
name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv2D(filters2,
kernel_size,
padding='same',
name=conv_name_base + '2b')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
shortcut = Conv2D(filters3, (1, 1),
strides=strides,
name=conv_name_base + '1')(input_tensor)
shortcut = BatchNormalization(axis=bn_axis,
name=bn_name_base + '1')(shortcut)
x = layers.add([x, shortcut])
x = Activation('relu')(x)
return x
def call(self, inputs):
if not self.pre_act:
x = self.batch1(inputs)
x = self.ac1(x)
x = self.conv1(x)
x = self.batch2(x)
x = self.ac2(x)
x = self.drop_1(x)
x = self.conv2(x)
x = self.batch3(x)
x = self.ac3(x)
x = self.drop_2(x)
pre_x = self.downsample(inputs)
squeeze = self.gavgpool(x)
squeeze = self.dnn_1(squeeze)
squeeze = self.ac4(squeeze)
squeeze = self.dnn_2(squeeze)
squeeze = tf.nn.sigmoid(squeeze)
squeeze = tf.reshape(squeeze, [-1, 1, self.out_dim])
x = x * squeeze
x = layers.add([x, pre_x])
else:
x = self.conv1(inputs)
x = self.batch1(x)
x = self.ac1(x)
x = self.drop_1(x)
x = self.conv2(x)
x = self.batch2(x)
x = self.ac2(x)
x = self.drop_2(x)
pre_x = self.downsample(inputs)
x = self.batch3(x)
squeeze = self.gavgpool(x)
squeeze = self.dnn_1(squeeze)
squeeze = self.ac4(squeeze)
squeeze = self.dnn_2(squeeze)
squeeze = tf.nn.sigmoid(squeeze)
squeeze = tf.reshape(squeeze, [-1, 1, self.out_dim])
x = x * squeeze
x = layers.add([x, pre_x])
x = self.ac3(x)
return x
def get_linear_logit(linear_term, dense_input_, l2_reg):
if len(linear_term) > 1:
linear_term = add(linear_term)
elif len(linear_term) == 1:
linear_term = linear_term[0]
else:
linear_term = None
dense_input = list(dense_input_.values())
if len(dense_input) > 0:
dense_input__ = dense_input[0] if len(
dense_input) == 1 else Concatenate()(dense_input)
linear_dense_logit = Dense(
1, activation=None, use_bias=False, kernel_regularizer=l2(l2_reg))(dense_input__)
if linear_term is not None:
linear_term = add([linear_dense_logit, linear_term])
else:
linear_term = linear_dense_logit
return linear_term
def call(self, inputs, training=False):
x = self._conv2d_1(inputs)
x = self._bn_1(x, training=training)
x = self.activation_1(x)
x = self._conv2d_2(x)
x = self._bn_2(x, training=training)
x = self._activation_2(x)
x = self._conv2d_3(x)
x = self._bn_3(x, training=training)
x = layers.add([x, inputs])
return self._activation_e3(x)
def _identity_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(1, 1)):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
con_name_base = "res" + str(stage) + block + "_branch"
bn_name_base = 'bn' + str(stage) + block + '_branch'
filters1, filters2, filters3 = filters
# 创建第一个conv2d-BN-relu小模块，卷积核大小为(1, 1)
x = _conv_bn(input_tensor, filters1, (1, 1), strides, con_name_base + "2a",
bn_name_base + "2a")
x = layers.Activation("relu")(x)
# 创建第二个conv2d-BN-relu小模块，卷积核大小为(3, 3)
x = _conv_bn(x,
filters2,
kernel_size,
strides,
con_name_base + "2b",
bn_name_base + "2b",
padding="same")
x = layers.Activation("relu")(x)
# 创建第3个conv2d-BN-relu小模块，卷积核大小为(1, 1)
x = _conv_bn(x, filters3, (1, 1), strides, con_name_base + "2c",
bn_name_base + "2c")
#add
x = layers.add([x, input_tensor])
x = layers.Activation("relu")(x)
return x
def get_linear_logit(linear_emb_list, dense_input_dict, l2_reg):
if len(linear_emb_list) > 1:
linear_term = add(linear_emb_list)
elif len(linear_emb_list) == 1:
linear_term = linear_emb_list[0]
else:
linear_term = None
dense_input = list(dense_input_dict.values())
if len(dense_input) > 0:
dense_input__ = dense_input[0] if len(
dense_input) == 1 else Concatenate()(dense_input)
linear_dense_logit = Dense(
1, activation=None, use_bias=False, kernel_regularizer=l2(l2_reg))(dense_input__)
if linear_term is not None:
linear_term = add([linear_dense_logit, linear_term])
else:
linear_term = linear_dense_logit
return linear_term
def decoder(x_4, x_2, x_1):
x_4 = Conv2D(filters=12,
kernel_size=1,
padding='same',
kernel_regularizer=regularizers.l2(weight_decay))(x_4)
dec_4 = Lambda(upscale,
arguments={
'method': interpolation,
'up_factor': 8
})(x_4)
if stride == 4:
return dec_4
x_2 = Conv2D(filters=12,
kernel_size=1,
padding='same',
kernel_regularizer=regularizers.l2(weight_decay))(x_2)
up_42 = Lambda(upscale, arguments={'method': interpolation})(x_4)
add_2 = add([up_42, x_2])
dec_2 = Lambda(upscale,
arguments={
'method': interpolation,
'up_factor': 4
})(add_2)
if stride == 2:
return dec_2
x_1 = Conv2D(filters=12,
kernel_size=1,
padding='same',
kernel_regularizer=regularizers.l2(weight_decay))(x_1)
up_21 = Lambda(upscale, arguments={'method': interpolation})(add_2)
add_1 = add([up_21, x_1])
dec_1 = Lambda(upscale,
arguments={
'method': interpolation,
'up_factor': 2
})(add_1)
return dec_1
def WideDeep():
# embedding_size=8
hidden_size = (128, 128)
l2_reg_linear = 1e-5
l2_reg_embedding = 1e-5
l2_reg_deep = 0
init_std = 0.0001
seed = 1024
keep_prob = 1
activation = 'relu'
final_activation = 'relu'
wide_features = pd.read_csv('data/path_matrix.txt',
sep=' ',
header=None,
nrows=1)
deep_features = pd.read_csv('data/sns_dense.csv',
sep=',',
header=0,
nrows=2)
wide_input = Input(shape=(wide_features.shape[1], ),
name='wide_' + str(wide_features.shape[1]))
wide_term = Dense(1, use_bias=False, activation=None)(wide_input)
deep_input = deep_features.iloc[:, :-1]
deep_feats = {
feat: Input(shape=(1, ), name=feat + '_' + str(i))
for i, feat in enumerate(deep_input)
}
deep_list = [v for v in deep_feats.values()]
deep_input = Concatenate()(deep_list)
deep_input = Flatten()(deep_input)
# hidden_size, activation='relu', l2_reg=0, keep_prob=1, use_bn=False, seed=1024
deep_out = MLP(hidden_size=hidden_size,
activation=activation,
l2_reg=l2_reg_deep,
keep_prob=keep_prob,
use_bn=False,
seed=seed)(deep_input)
deep_logit = Dense(1, use_bias=False, activation=None)(deep_out)
final_logit = add([deep_logit, wide_term])
output = PredictionLayer(final_activation)(final_logit)
deep_list.append(wide_input)
model = Model(inputs=deep_list, outputs=output)
model.summary()
keras.utils.plot_model(model, to_file='image/widedeep_model.png')
return model
def real_distances(L, num_blocks, intermediate_n_channels, input_n_channels):
print('')
print('Model params:')
print('L', L)
print('num_blocks', num_blocks)
print('intermediate_n_channels', intermediate_n_channels)
print('input_n_channels', input_n_channels)
print('')
dropout_value = 0.2
my_input = Input(shape=(L, L, input_n_channels))
tower = BatchNormalization()(my_input)
tower = Activation('elu')(tower)
tower = Convolution2D(intermediate_n_channels, 1, padding='same')(tower)
flag_1D = False
d_rate = 1
for i in range(num_blocks):
block = BatchNormalization()(tower)
block = Activation('elu')(block)
if flag_1D:
block = Convolution2D(intermediate_n_channels,
kernel_size=(1, 5),
padding='same')(block)
else:
block = Convolution2D(intermediate_n_channels,
kernel_size=(3, 3),
padding='same')(block)
block = Dropout(dropout_value)(block)
block = Activation('elu')(block)
if flag_1D:
block = Convolution2D(intermediate_n_channels,
kernel_size=(1, 5),
dilation_rate=(d_rate, d_rate),
padding='same')(block)
flag_1D = False
else:
block = Convolution2D(intermediate_n_channels,
kernel_size=(3, 3),
dilation_rate=(d_rate, d_rate),
padding='same')(block)
flag_1D = True
tower = add([block, tower])
if d_rate == 1:
d_rate = 2
elif d_rate == 2:
d_rate = 4
else:
d_rate = 1
tower = BatchNormalization()(tower)
tower = Activation('relu')(tower)
tower = Convolution2D(1, 3, padding='same')(tower)
tower = Activation('relu')(tower)
model = Model(my_input, tower)
return model
def conv_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2, 2),
use_bias=True,
train_bn=True):
"""conv_block is the block that has a conv layer at shortcut
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the nb_filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
use_bias: Boolean. To use or not use a bias in conv layers.
train_bn: Boolean. Train or freeze Batch Norm layers
Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
And the shortcut should have subsample=(2,2) as well
"""
nb_filter1, nb_filter2, nb_filter3 = filters
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2D(nb_filter1, (1, 1),
strides=strides,
name=conv_name_base + '2a',
use_bias=use_bias)(input_tensor)
x = BatchNormalization(name=bn_name_base + '2a')(x, training=train_bn)
x = Activation('relu')(x)
x = Conv2D(nb_filter2, (kernel_size, kernel_size),
padding='same',
name=conv_name_base + '2b',
use_bias=use_bias)(x)
x = BatchNormalization(name=bn_name_base + '2b')(x, training=train_bn)
x = Activation('relu')(x)
x = Conv2D(nb_filter3, (1, 1),
name=conv_name_base + '2c',
use_bias=use_bias)(x)
x = BatchNormalization(name=bn_name_base + '2c')(x, training=train_bn)
shortcut = Conv2D(nb_filter3, (1, 1),
strides=strides,
name=conv_name_base + '1',
use_bias=use_bias)(input_tensor)
shortcut = BatchNormalization(name=bn_name_base + '1')(shortcut,
training=train_bn)
x = add([x, shortcut])
x = Activation('relu', name='res' + str(stage) + block + '_out')(x)
return x
def Unit(x, filters, pool=False):
res = x
if pool: # makes res & out the same shape
x = MaxPooling2D(pool_size=(2, 2))(x)
res = Conv2D(filters=filters,
kernel_size=[1, 1],
strides=(2, 2),
padding="same")(res)
out = Conv2D(filters, (1, 1), padding='same', activation='relu')(x)
out = Conv2D(filters, (1, 1), padding='same', activation='relu')(out)
out = add([res, out]) # res & out must be same shape
return out
def identity_block_5d(input_tensor, kernel_size, filters, stage, block,
l2_reg):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
bn_axis = -1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = layers.TimeDistributed(layers.Conv2D(
filters1, (1, 1),
kernel_regularizer=regularizers.l2(l2_reg),
bias_regularizer=regularizers.l2(l2_reg),
kernel_initializer='he_normal'),
name=conv_name_base + '2a')(input_tensor)
x = layers.TimeDistributed(BatchNorm(axis=-1), name=bn_name_base + '2a')(x)
x = layers.Activation('relu')(x)
x = layers.TimeDistributed(layers.Conv2D(
filters2,
kernel_size,
padding='same',
kernel_regularizer=regularizers.l2(l2_reg),
bias_regularizer=regularizers.l2(l2_reg),
kernel_initializer='he_normal'),
name=conv_name_base + '2b')(x)
x = layers.TimeDistributed(BatchNorm(axis=bn_axis),
name=bn_name_base + '2b')(x)
x = layers.Activation('relu')(x)
x = layers.TimeDistributed(layers.Conv2D(
filters3, (1, 1),
kernel_regularizer=regularizers.l2(l2_reg),
bias_regularizer=regularizers.l2(l2_reg),
kernel_initializer='he_normal'),
name=conv_name_base + '2c')(x)
x = layers.TimeDistributed(BatchNorm(axis=bn_axis),
name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = layers.Activation('relu')(x)
return x
def identity_block(input_tensor,
kernel_size,
filters,
stage,
block,
path,
non_degenerate_temporal_conv=False):
"""The identity block is the block that has no conv layer at shortcut.
Arguments:
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
Returns:
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 4
else:
bn_axis = 1
conv_name_base = str(path) + 'res' + str(stage) + block + '_branch'
bn_name_base = str(path) + 'bn' + str(stage) + block + '_branch'
if non_degenerate_temporal_conv == True:
x = Conv3D(filters1, (3, 1, 1),
padding='same',
name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
else:
x = Conv3D(filters1, (1, 1, 1),
name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv3D(filters2,
kernel_size,
padding='same',
name=conv_name_base + '2b')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv3D(filters3, (1, 1, 1), name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = Activation('relu')(x)
return x
def residual_block(x, out_filters, increase=False, name=None):
strides = 1
if increase:
strides = 2
o1 = BatchNormalization(momentum=0.9, epsilon=1e-5)(x)
if name is not None:
o1 = Activation('relu', name=name)(o1)
else:
o1 = Activation('relu')(o1)
conv1 = Conv2D(out_filters,
kernel_size=3,
padding='SAME',
strides=strides,
kernel_regularizer=regularizers.l2(weight_decay),
kernel_initializer='he_normal')(o1)
conv1 = BatchNormalization(momentum=0.9, epsilon=1e-5)(conv1)
conv1 = Activation('relu')(conv1)
conv2 = Conv2D(out_filters,
kernel_size=3,
padding='SAME',
strides=1,
kernel_regularizer=regularizers.l2(weight_decay),
kernel_initializer='he_normal')(conv1)
if increase:
proj = Conv2D(out_filters,
kernel_size=1,
padding='SAME',
strides=strides,
kernel_regularizer=regularizers.l2(weight_decay),
kernel_initializer='he_normal')(o1)
proj = add([conv2, proj])
else:
proj = add([x, conv2])
return proj
def call(self, inputs, training=False):
x = self._conv2d_1(inputs)
x = self._bn_1(x, training=training)
x = self._activation_1(x)
x = self._conv2d_2(x)
x = self._bn_2(x, training=training)
x = self._activation_2(x)
x = self._conv2d_3(x)
x = self._bn_3(x, training=training)
shortcut = self._shortcut(inputs)
shortcut = self._bn_4(shortcut, training=training)
x = layers.add([x, shortcut])
return self._activation_4(x)
def call(self, inputs, training=None):
out = self.conv1(inputs)
out = self.bn1(out)
out = self.relu(out)
#将两个卷积层组合
out = self.conv2(out)
out = self.bn2(out)
#定义短接层
identity = self.downsample(inputs)
#用Add将卷积层与短接层相加
output = layers.add([out, identity])
output = tf.nn.relu(output)
return output
def identity_block(input_tensor,
kernel_size,
filters,
stage,
block,
dilation=1,
conv_trainable=True):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = klayers.Conv2D(filters1, (1, 1),
kernel_initializer='he_uniform',
trainable=conv_trainable,
name=conv_name_base + '2a')(input_tensor)
x = klayers.BatchNormalizationV2(name=bn_name_base + '2a')(x)
x = klayers.Activation('relu')(x)
x = klayers.Conv2D(filters2,
kernel_size,
dilation_rate=dilation,
padding='same',
kernel_initializer='he_uniform',
trainable=conv_trainable,
name=conv_name_base + '2b')(x)
x = klayers.BatchNormalizationV2(name=bn_name_base + '2b')(x)
x = klayers.Activation('relu')(x)
x = klayers.Conv2D(filters3, (1, 1),
kernel_initializer='he_uniform',
trainable=conv_trainable,
name=conv_name_base + '2c')(x)
x = klayers.BatchNormalizationV2(name=bn_name_base + '2c')(x)
x = klayers.add([x, input_tensor])
x = klayers.Activation('relu')(x)
return x
def _resnet_bottleneck_block(input, filters, k=1, strides=(1, 1)):
''' Adds a pre-activation resnet block with bottleneck layers
Args:
input: input tensor
filters: number of output filters
k: width factor
strides: strides of the convolution layer
Returns: a keras tensor
'''
init = input
channel_axis = 1 if K.image_data_format() == "channels_first" else -1
bottleneck_expand = 4
x = BatchNormalization(axis=channel_axis)(input)
x = Activation('relu')(x)
if strides != (1, 1) or K.int_shape(
init)[channel_axis] != bottleneck_expand * filters * k:
init = Conv2D(bottleneck_expand * filters * k, (1, 1),
padding='same',
kernel_initializer='he_normal',
use_bias=False,
strides=strides)(x)
x = Conv2D(filters * k, (1, 1),
padding='same',
kernel_initializer='he_normal',
use_bias=False)(x)
x = BatchNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
x = Conv2D(filters * k, (3, 3),
padding='same',
kernel_initializer='he_normal',
use_bias=False,
strides=strides)(x)
x = BatchNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
x = Conv2D(bottleneck_expand * filters * k, (1, 1),
padding='same',
kernel_initializer='he_normal',
use_bias=False)(x)
# squeeze and excite block
x = squeeze_excite_block(x)
m = add([x, init])
return m
def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2,
2)):
"""A block that has a conv layer at shortcut.
Arguments:
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
strides: Strides for the first conv layer in the block.
Returns:
Output tensor for the block.
Note that from stage 3,
the first conv layer at main path is with strides=(2, 2)
And the shortcut should have strides=(2, 2) as well
"""
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2D(
filters1, (1, 1), strides=strides, name=conv_name_base + '2a')(
input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv2D(
filters2, kernel_size, padding='same', name=conv_name_base + '2b')(
x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
shortcut = Conv2D(
filters3, (1, 1), strides=strides, name=conv_name_base + '1')(
input_tensor)
shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)
x = layers.add([x, shortcut])
x = Activation('relu')(x)
return x
def identity_block_v2(input_tensor, kernel_size, filters, stage, block, trainable=True):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filterss of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
bn_axis = 3
conv_name_1 = 'conv' + str(stage) + '_' + str(block) + '_1x1_reduce'
bn_name_1 = 'conv' + str(stage) + '_' + str(block) + '_1x1_reduce/bn'
x = Conv2D(filters1, (1, 1),
kernel_initializer=tf.initializers.glorot_normal(),
use_bias=False,
kernel_regularizer=l2(WEIGHT_DECAY),
trainable=trainable,
name=conv_name_1)(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_1)(x)
x = Activation('relu')(x)
conv_name_2 = 'conv' + str(stage) + '_' + str(block) + '_3x3'
bn_name_2 = 'conv' + str(stage) + '_' + str(block) + '_3x3/bn'
x = Conv2D(filters2, kernel_size,
padding='same',
kernel_initializer=tf.initializers.glorot_normal(),
use_bias=False,
kernel_regularizer=l2(WEIGHT_DECAY),
trainable=trainable,
name=conv_name_2)(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_2)(x)
x = Activation('relu')(x)
conv_name_3 = 'conv' + str(stage) + '_' + str(block) + '_1x1_increase'
bn_name_3 = 'conv' + str(stage) + '_' + str(block) + '_1x1_increase/bn'
x = Conv2D(filters3, (1, 1),
kernel_initializer=tf.initializers.glorot_normal(),
use_bias=False,
kernel_regularizer=l2(WEIGHT_DECAY),
trainable=trainable,
name=conv_name_3)(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_3)(x)
x = layers.add([x, input_tensor])
x = Activation('relu')(x)
return x
def identity_block(input_tensor, kernel_size, filters, stage, block):
"""The identity block is the block that has no conv layer at shortcut.
Args:
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: integer, filters of the bottleneck layer.
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
Returns:
Output tensor for the block.
"""
if backend.image_data_format() == 'channels_last':
bn_axis = -1
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = layers.Conv2D(filters=filters,
kernel_size=1,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2a')(input_tensor)
x = layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters=filters,
kernel_size=kernel_size,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2b')(x)
x = layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters=4 * filters,
kernel_size=1,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2c')(x)
x = layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = layers.Activation('relu')(x)
return x
def xception_block(inputs,
depth_list,
prefix,
skip_type,
stride,
rate=1,
depth_activation=False,
return_skip=False):
residual_inputs = inputs
for i in range(3):
residual_inputs = separable_conv_with_batch_normalization(
residual_inputs,
depth_list[i],
prefix + 'xception_separable_conv{}'.format(i + 1),
stride=stride if i == 2 else 1,
rate=rate,
depth_activation=depth_activation,
epsilon=1e-3)
if i == 1:
skip = residual_inputs
if skip_type == 'conv':
shortcut = shortcut_conv(inputs,
depth_list[-1],
prefix + '_shortcut',
kernel_size=1,
stride=stride)
shortcut = BatchNormalization(
name=prefix + '_shortcut_batch_normalization')(shortcut)
outputs = layers.add([residual_inputs, shortcut])
elif skip_type == 'sum':
outputs = layers.add([residual_inputs, inputs])
elif skip_type == 'none':
outputs = residual_inputs
if return_skip:
return outputs, skip
else:
return outputs
def channel_spatial_squeeze_excite(input_tensor):
""" Create a spatial squeeze-excite block
Args:
input_tensor: input Keras tensor
Returns: a Keras tensor
References
- [Squeeze and Excitation Networks](https://arxiv.org/abs/1709.01507)
- [Concurrent Spatial and Channel Squeeze & Excitation in Fully Convolutional Networks](https://arxiv.org/abs/1803.02579)
"""
cse = squeeze_excite_block(input_tensor)
sse = spatial_squeeze_excite_block(input_tensor)
x = add([cse, sse])
return x
def residual_connection_model():
x = Input(shape=(12, 4, 256))
# padding 填充和之前size一样
# 128 是通道数量，3是卷积核尺寸
# 进行卷积后 通道数128将成为第三个维度，之前两个维度由于补全不变
y = layers.Conv2D(128, 3, activation='relu', padding='same')(x)
# 使用1x1卷积核使得尺寸可以相加
# strides 是卷积步长
residual = layers.Conv2D(128, 1, strides=1, padding='same')(x)
y = layers.add([y, residual])
prediction = layers.Dense(1)(y)
model = Model(x, prediction)
return model
def __call__(self, input_tensor):
x = self.conv1(input_tensor)
x = self.bn1(x)
x = self.relu(x)
x = self.conv2(x)
x = self.bn2(x)
x = self.relu(x)
x = self.conv3(x)
x = self.bn3(x)
x = layers.add([x, input_tensor])
x = self.relu(x)
return x
def conv_block_1d(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2)):
"""A block that has a conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
Note that from stage 3, the first conv layer at main path is with strides=(2,2)
And the shortcut should have strides=(2,2) as well
"""
filters1, filters2, filters3 = filters
conv_name_base = 'res_1d' + str(stage) + block + '_branch'
bn_name_base = 'bn_1d' + str(stage) + block + '_branch'
x = Conv1D(filters1, (1), strides=strides,
name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv1D(filters2,
kernel_size,
padding='same',
name=conv_name_base + '2b')(x)
x = BatchNormalization(name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv1D(filters3, (1), name=conv_name_base + '2c')(x)
x = BatchNormalization(name=bn_name_base + '2c')(x)
shortcut = Conv1D(filters3, (1),
strides=strides,
name=conv_name_base + '1')(input_tensor)
shortcut = BatchNormalization(name=bn_name_base + '1')(shortcut)
x = layers.add([x, shortcut])
x = Activation('relu')(x)
return x
def resnet_first_block_first_module(input, channel_depth):
residual_input = input
stride = 1
residual_input = Conv2D(channel_depth, kernel_size=1, strides=1, padding="same")(residual_input)
residual_input = BatchNormalization()(residual_input)
input = Conv2D(int(channel_depth/4), kernel_size=1, strides=stride, padding="same")(input)
input = BatchNormalization()(input)
input = Activation("relu")(input)
input = Conv2D(int(channel_depth / 4), kernel_size=3, strides=stride, padding="same")(input)
input = BatchNormalization()(input)
input = Activation("relu")(input)
input = Conv2D(channel_depth, kernel_size=1, strides=stride, padding="same")(input)
input = BatchNormalization()(input)
input = add([input, residual_input])
input = Activation("relu")(input)
return input
def identity_block(input_tensor, kernel_size, filters, stage, block):
"""The identity block is the block that has no conv layer at shortcut.
Arguments:
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
Returns:
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2D(filters1, (1, 1), name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv2D(
filters2, kernel_size, padding='same', name=conv_name_base + '2b')(
x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = Activation('relu')(x)
return x
def resnet_module(input, channel_depth, strided_pool=False ):
residual_input = input
stride = 1
if(strided_pool):
stride = 2
residual_input = Conv2D(channel_depth, kernel_size=1, strides=stride, padding="same")(residual_input)
residual_input = BatchNormalization()(residual_input)
input = Conv2D(int(channel_depth/4), kernel_size=1, strides=stride, padding="same")(input)
input = BatchNormalization()(input)
input = Activation("relu")(input)
input = Conv2D(int(channel_depth / 4), kernel_size=3, strides=1, padding="same")(input)
input = BatchNormalization()(input)
input = Activation("relu")(input)
input = Conv2D(channel_depth, kernel_size=1, strides=1, padding="same")(input)
input = BatchNormalization()(input)
input = add([input, residual_input])
input = Activation("relu")(input)
return input
def _reduction_a_cell(ip, p, filters, block_id=None):
"""Adds a Reduction cell for NASNet-A (Fig. 4 in the paper).
Arguments:
ip: Input tensor `x`
p: Input tensor `p`
filters: Number of output filters
block_id: String block_id
Returns:
A Keras tensor
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
with K.name_scope('reduction_A_block_%s' % block_id):
p = _adjust_block(p, ip, filters, block_id)
h = Activation('relu')(ip)
h = Conv2D(
filters, (1, 1),
strides=(1, 1),
padding='same',
name='reduction_conv_1_%s' % block_id,
use_bias=False,
kernel_initializer='he_normal')(
h)
h = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='reduction_bn_1_%s' % block_id)(
h)
with K.name_scope('block_1'):
x1_1 = _separable_conv_block(
h,
filters, (5, 5),
strides=(2, 2),
block_id='reduction_left1_%s' % block_id)
x1_2 = _separable_conv_block(
p,
filters, (7, 7),
strides=(2, 2),
block_id='reduction_1_%s' % block_id)
x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % block_id)
with K.name_scope('block_2'):
x2_1 = MaxPooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_left2_%s' % block_id)(
h)
x2_2 = _separable_conv_block(
p,
filters, (7, 7),
strides=(2, 2),
block_id='reduction_right2_%s' % block_id)
x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % block_id)
with K.name_scope('block_3'):
x3_1 = AveragePooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_left3_%s' % block_id)(
h)
x3_2 = _separable_conv_block(
p,
filters, (5, 5),
strides=(2, 2),
block_id='reduction_right3_%s' % block_id)
x3 = add([x3_1, x3_2], name='reduction_add3_%s' % block_id)
with K.name_scope('block_4'):
x4 = AveragePooling2D(
(3, 3),
strides=(1, 1),
padding='same',
name='reduction_left4_%s' % block_id)(
x1)
x4 = add([x2, x4])
with K.name_scope('block_5'):
x5_1 = _separable_conv_block(
x1, filters, (3, 3), block_id='reduction_left4_%s' % block_id)
x5_2 = MaxPooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_right5_%s' % block_id)(
h)
x5 = add([x5_1, x5_2], name='reduction_add4_%s' % block_id)
x = concatenate(
[x2, x3, x4, x5],
axis=channel_dim,
name='reduction_concat_%s' % block_id)
return x, ip
def WDL(deep_feature_dim_dict, wide_feature_dim_dict, embedding_size=8, hidden_size=(128, 128), l2_reg_linear=1e-5, l2_reg_embedding=1e-5, l2_reg_deep=0, init_std=0.0001, seed=1024, keep_prob=1, activation='relu', final_activation='sigmoid',):
"""Instantiates the Wide&Deep Learning architecture.
:param deep_feature_dim_dict: dict,to indicate sparse field and dense field in deep part like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
:param wide_feature_dim_dict: dict,to indicate sparse field and dense field in wide part like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
:param embedding_size: positive integer,sparse feature embedding_size
:param hidden_size: list,list of positive integer or empty list, the layer number and units in each layer of deep net
:param l2_reg_linear: float. L2 regularizer strength applied to wide part
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param l2_reg_deep: float. L2 regularizer strength applied to deep net
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param keep_prob: float in (0,1]. keep_prob used in deep net
:param activation: Activation function to use in deep net
:param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'``
:return: A Keras model instance.
"""
if not isinstance(deep_feature_dim_dict,
dict) or "sparse" not in deep_feature_dim_dict or "dense" not in deep_feature_dim_dict:
raise ValueError(
"feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}")
sparse_input, dense_input, = create_input_dict(
deep_feature_dim_dict)
bias_sparse_input, bias_dense_input = create_input_dict(
wide_feature_dim_dict, 'bias')
sparse_embedding = create_embedding_dict(
deep_feature_dim_dict, embedding_size, init_std, seed, l2_reg_embedding)
wide_linear_embedding = create_embedding_dict(
wide_feature_dim_dict, 1, init_std, seed, l2_reg_linear, 'linear')
embed_list = get_embedding_vec_list(sparse_embedding, sparse_input)
deep_input = Concatenate()(embed_list) if len(
embed_list) > 1 else embed_list[0]
deep_input = Flatten()(deep_input)
if len(dense_input) > 0:
deep_input = Concatenate()([deep_input]+list(dense_input.values()))
deep_out = MLP(hidden_size, activation, l2_reg_deep, keep_prob,
False, seed)(deep_input)
deep_logit = Dense(1, use_bias=False, activation=None)(deep_out)
final_logit = deep_logit
if len(wide_feature_dim_dict['dense']) + len(wide_feature_dim_dict['sparse']) > 0:
if len(wide_feature_dim_dict['sparse']) > 0:
bias_embed_list = get_embedding_vec_list(
wide_linear_embedding, bias_sparse_input)
linear_term = add(bias_embed_list) if len(
bias_embed_list) > 1 else bias_embed_list[0]
final_logit = add([final_logit, linear_term])
if len(wide_feature_dim_dict['dense']) > 0:
wide_dense_term = Dense(1, use_bias=False, activation=None)(Concatenate()(
list(bias_dense_input.values())) if len(bias_dense_input) > 1 else list(bias_dense_input.values())[0])
final_logit = add([final_logit, wide_dense_term])
output = PredictionLayer(final_activation)(final_logit)
inputs_list = get_inputs_list(
[sparse_input, dense_input, bias_sparse_input, bias_dense_input])
model = Model(inputs=inputs_list, outputs=output)
return model
def conv_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2, 2)):
"""A block that has a conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
strides: Strides for the second conv layer in the block.
# Returns
Output tensor for the block.
Note that from stage 3,
the second conv layer at main path is with strides=(2, 2)
And the shortcut should have strides=(2, 2) as well
"""
filters1, filters2, filters3 = filters
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = layers.Conv2D(filters1, (1, 1), use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2a')(input_tensor)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2a')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters2, kernel_size, strides=strides, padding='same',
use_bias=False, kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2b')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2b')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters3, (1, 1), use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2c')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2c')(x)
shortcut = layers.Conv2D(filters3, (1, 1), strides=strides, use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '1')(input_tensor)
shortcut = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '1')(shortcut)
x = layers.add([x, shortcut])
x = layers.Activation('relu')(x)
return x
def MLR(region_feature_dim_dict, base_feature_dim_dict={"sparse": {}, "dense": []}, region_num=4,
l2_reg_linear=1e-5,
init_std=0.0001, seed=1024, final_activation='sigmoid',
bias_feature_dim_dict={"sparse": {}, "dense": []}):
"""Instantiates the Mixed Logistic Regression/Piece-wise Linear Model.
:param region_feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
:param base_feature_dim_dict: dict or None,to indicate sparse field and dense field of base learner.if None, it is same as region_feature_dim_dict
:param region_num: integer > 1,indicate the piece number
:param l2_reg_linear: float. L2 regularizer strength applied to weight
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'``
:param bias_feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
:return: A Keras model instance.
"""
if region_num <= 1:
raise ValueError("region_num must > 1")
if not isinstance(region_feature_dim_dict,
dict) or "sparse" not in region_feature_dim_dict or "dense" not in region_feature_dim_dict:
raise ValueError(
"feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}")
same_flag = False
if base_feature_dim_dict == {"sparse": {}, "dense": []}:
base_feature_dim_dict = region_feature_dim_dict
same_flag = True
region_sparse_input, region_dense_input, base_sparse_input, base_dense_input, bias_sparse_input, bias_dense_input = get_input(
region_feature_dim_dict, base_feature_dim_dict, bias_feature_dim_dict, same_flag)
region_embeddings, base_embeddings, bias_embedding = get_embedding(
region_num, region_feature_dim_dict, base_feature_dim_dict, bias_feature_dim_dict, init_std, seed, l2_reg_linear)
if same_flag:
base_dense_input_ = region_dense_input
base_sparse_input_ = region_sparse_input
else:
base_dense_input_ = base_dense_input
base_sparse_input_ = base_sparse_input
region_dense_feature_num = len(region_feature_dim_dict['dense'])
region_sparse_feature_num = len(region_feature_dim_dict['sparse'])
base_dense_feature_num = len(base_feature_dim_dict['dense'])
base_sparse_feature_num = len(base_feature_dim_dict['sparse'])
bias_dense_feature_num = len(bias_feature_dim_dict['dense'])
bias_sparse_feature_num = len(bias_feature_dim_dict['sparse'])
if region_dense_feature_num > 1:
region_dense_logits_ = [Dense(1, )(Concatenate()(region_dense_input)) for _ in
range(region_num)]
elif region_dense_feature_num == 1:
region_dense_logits_ = [Dense(1, )(region_dense_input[0]) for _ in
range(region_num)]
if base_dense_feature_num > 1:
base_dense_logits = [Dense(1, )(Concatenate()(base_dense_input_))for _ in
range(region_num)]
elif base_dense_feature_num == 1:
base_dense_logits = [Dense(1, )(base_dense_input_[0])for _ in
range(region_num)]
if region_dense_feature_num > 0 and region_sparse_feature_num == 0:
region_logits = Concatenate()(region_dense_logits_)
elif region_dense_feature_num == 0 and region_sparse_feature_num > 0:
region_sparse_logits = [
add([region_embeddings[j][i](region_sparse_input[i])
for i in range(region_sparse_feature_num)])
if region_sparse_feature_num > 1 else region_embeddings[j][0](region_sparse_input[0])
for j in range(region_num)]
region_logits = Concatenate()(region_sparse_logits)
else:
region_sparse_logits = [
add([region_embeddings[j][i](region_sparse_input[i])
for i in range(region_sparse_feature_num)])
for j in range(region_num)]
region_logits = Concatenate()(
[add([region_sparse_logits[i], region_dense_logits_[i]]) for i in range(region_num)])
if base_dense_feature_num > 0 and base_sparse_feature_num == 0:
base_logits = base_dense_logits
elif base_dense_feature_num == 0 and base_sparse_feature_num > 0:
base_sparse_logits = [add(
[base_embeddings[j][i](base_sparse_input_[i]) for i in range(base_sparse_feature_num)]) if base_sparse_feature_num > 1 else base_embeddings[j][0](base_sparse_input_[0])
for j in range(region_num)]
base_logits = base_sparse_logits
else:
base_sparse_logits = [add(
[base_embeddings[j][i](base_sparse_input_[i]) for i in range(base_sparse_feature_num)]) if base_sparse_feature_num > 1 else base_embeddings[j][0](base_sparse_input_[0])
for j in range(region_num)]
base_logits = [add([base_sparse_logits[i], base_dense_logits[i]])
for i in range(region_num)]
# Dense(self.region_num, activation='softmax')(final_logit)
region_weights = Activation("softmax")(region_logits)
learner_score = Concatenate()(
[Activation(final_activation, name='learner' + str(i))(base_logits[i]) for i in range(region_num)])
final_logit = dot([region_weights, learner_score], axes=-1)
if bias_dense_feature_num + bias_sparse_feature_num > 0:
if bias_dense_feature_num > 1:
bias_dense_logits = Dense(1,)(Concatenate()(bias_dense_input))
elif bias_dense_feature_num == 1:
bias_dense_logits = Dense(1,)(bias_dense_input[0])
else:
pass
if bias_sparse_feature_num > 1:
bias_cate_logits = add([bias_embedding[i](bias_sparse_input[i])
for i, feat in enumerate(bias_feature_dim_dict['sparse'])])
elif bias_sparse_feature_num == 1:
bias_cate_logits = bias_embedding[0](bias_sparse_input[0])
else:
pass
if bias_dense_feature_num > 0 and bias_sparse_feature_num > 0:
bias_logits = add([bias_dense_logits, bias_cate_logits])
elif bias_dense_feature_num > 0:
bias_logits = bias_dense_logits
else:
bias_logits = bias_cate_logits
bias_prob = Activation('sigmoid')(bias_logits)
final_logit = dot([final_logit, bias_prob], axes=-1)
output = Reshape([1])(final_logit)
model = Model(inputs=region_sparse_input + region_dense_input+base_sparse_input +
base_dense_input+bias_sparse_input+bias_dense_input, outputs=output)
return model
def Xception(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Xception architecture.
Optionally loads weights pre-trained
on ImageNet. This model is available for TensorFlow only,
and can only be used with inputs following the TensorFlow
data format `(width, height, channels)`.
You should set `image_data_format='channels_last'` in your Keras config
located at ~/.keras/keras.json.
Note that the default input image size for this model is 299x299.
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)`.
It should have exactly 3 inputs channels,
and width and height should be no smaller than 71.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
if K.image_data_format() != 'channels_last':
logging.warning(
'The Xception model is only available for the '
'input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" (channels, width, height). '
'You should set `image_data_format="channels_last"` in your Keras '
'config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
K.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=299,
min_size=71,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = Conv2D(
32, (3, 3), strides=(2, 2), use_bias=False, name='block1_conv1')(
img_input)
x = BatchNormalization(name='block1_conv1_bn')(x)
x = Activation('relu', name='block1_conv1_act')(x)
x = Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x)
x = BatchNormalization(name='block1_conv2_bn')(x)
x = Activation('relu', name='block1_conv2_act')(x)
residual = Conv2D(
128, (1, 1), strides=(2, 2), padding='same', use_bias=False)(
x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(
128, (3, 3), padding='same', use_bias=False, name='block2_sepconv1')(
x)
x = BatchNormalization(name='block2_sepconv1_bn')(x)
x = Activation('relu', name='block2_sepconv2_act')(x)
x = SeparableConv2D(
128, (3, 3), padding='same', use_bias=False, name='block2_sepconv2')(
x)
x = BatchNormalization(name='block2_sepconv2_bn')(x)
x = MaxPooling2D(
(3, 3), strides=(2, 2), padding='same', name='block2_pool')(
x)
x = layers.add([x, residual])
residual = Conv2D(
256, (1, 1), strides=(2, 2), padding='same', use_bias=False)(
x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block3_sepconv1_act')(x)
x = SeparableConv2D(
256, (3, 3), padding='same', use_bias=False, name='block3_sepconv1')(
x)
x = BatchNormalization(name='block3_sepconv1_bn')(x)
x = Activation('relu', name='block3_sepconv2_act')(x)
x = SeparableConv2D(
256, (3, 3), padding='same', use_bias=False, name='block3_sepconv2')(
x)
x = BatchNormalization(name='block3_sepconv2_bn')(x)
x = MaxPooling2D(
(3, 3), strides=(2, 2), padding='same', name='block3_pool')(
x)
x = layers.add([x, residual])
residual = Conv2D(
728, (1, 1), strides=(2, 2), padding='same', use_bias=False)(
x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block4_sepconv1_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name='block4_sepconv1')(
x)
x = BatchNormalization(name='block4_sepconv1_bn')(x)
x = Activation('relu', name='block4_sepconv2_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name='block4_sepconv2')(
x)
x = BatchNormalization(name='block4_sepconv2_bn')(x)
x = MaxPooling2D(
(3, 3), strides=(2, 2), padding='same', name='block4_pool')(
x)
x = layers.add([x, residual])
for i in range(8):
residual = x
prefix = 'block' + str(i + 5)
x = Activation('relu', name=prefix + '_sepconv1_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv1')(
x)
x = BatchNormalization(name=prefix + '_sepconv1_bn')(x)
x = Activation('relu', name=prefix + '_sepconv2_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv2')(
x)
x = BatchNormalization(name=prefix + '_sepconv2_bn')(x)
x = Activation('relu', name=prefix + '_sepconv3_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv3')(
x)
x = BatchNormalization(name=prefix + '_sepconv3_bn')(x)
x = layers.add([x, residual])
residual = Conv2D(
1024, (1, 1), strides=(2, 2), padding='same', use_bias=False)(
x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block13_sepconv1_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name='block13_sepconv1')(
x)
x = BatchNormalization(name='block13_sepconv1_bn')(x)
x = Activation('relu', name='block13_sepconv2_act')(x)
x = SeparableConv2D(
1024, (3, 3), padding='same', use_bias=False, name='block13_sepconv2')(
x)
x = BatchNormalization(name='block13_sepconv2_bn')(x)
x = MaxPooling2D(
(3, 3), strides=(2, 2), padding='same', name='block13_pool')(
x)
x = layers.add([x, residual])
x = SeparableConv2D(
1536, (3, 3), padding='same', use_bias=False, name='block14_sepconv1')(
x)
x = BatchNormalization(name='block14_sepconv1_bn')(x)
x = Activation('relu', name='block14_sepconv1_act')(x)
x = SeparableConv2D(
2048, (3, 3), padding='same', use_bias=False, name='block14_sepconv2')(
x)
x = BatchNormalization(name='block14_sepconv2_bn')(x)
x = Activation('relu', name='block14_sepconv2_act')(x)
if include_top:
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='xception')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'xception_weights_tf_dim_ordering_tf_kernels.h5',
TF_WEIGHTS_PATH,
cache_subdir='models',
file_hash='0a58e3b7378bc2990ea3b43d5981f1f6')
else:
weights_path = get_file(
'xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
TF_WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='b0042744bf5b25fce3cb969f33bebb97')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
if old_data_format:
K.set_image_data_format(old_data_format)
return model
