def identity_block(input_tensor, kernel_size, filters, stage, block): """The identity block is the block that has no conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names # Returns Output tensor for the block. """ filters1, filters2, filters3 = filters if backend.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = layers.Conv2D(filters1, (1, 1), use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2a')(input_tensor) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2a')(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters2, kernel_size, use_bias=False, padding='same', kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2b')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2b')(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters3, (1, 1), use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2c')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2c')(x) x = layers.add([x, input_tensor]) x = layers.Activation('relu')(x) return x
def identity_building_block(input_tensor, kernel_size, filters, stage, block, training=None): """The identity block is the block that has no conv layer at shortcut. Arguments: input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: current block label, used for generating layer names training: Only used if training keras model with Estimator. In other scenarios it is handled automatically. Returns: Output tensor for the block. """ filters1, filters2 = filters if backend.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = layers.Conv2D(filters1, kernel_size, padding='same', use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2a')(input_tensor) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2a')(x, training=training) x = layers.Activation('relu')(x) x = layers.Conv2D(filters2, kernel_size, padding='same', use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2b')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2b')(x, training=training) x = layers.add([x, input_tensor]) x = layers.Activation('relu')(x) return x
def residual_block(inputs, filters): x = conv2d_unit(inputs, filters, (1, 1)) x = conv2d_unit(x, 2 * filters, (3, 3)) x = add([inputs, x]) x = Activation('linear')(x) return x
def _xception_block(inputs, depth_list, prefix, skip_connection_type, stride, rate=1, depth_activation=False, return_skip=False): """ Basic building block of modified Xception network Args: inputs: input tensor depth_list: number of filters in each SepConv layer. len(depth_list) == 3 prefix: prefix before name skip_connection_type: one of {'conv','sum','none'} stride: stride at last depthwise conv rate: atrous rate for depthwise convolution depth_activation: flag to use activation between depthwise & pointwise convs return_skip: flag to return additional tensor after 2 SepConvs for decoder """ residual = inputs for i in range(3): residual = SepConv_BN( residual, #一种新的conv网络 depth_list[i], prefix + '_separable_conv{}'.format(i + 1), stride=stride if i == 2 else 1, rate=rate, depth_activation=depth_activation) if i == 1: skip = residual if skip_connection_type == 'conv': shortcut = _conv2d_same(inputs, depth_list[-1], prefix + '_shortcut', kernel_size=1, stride=stride) shortcut = BatchNormalization(name=prefix + '_shortcut_BN')(shortcut) outputs = layers.add([residual, shortcut]) elif skip_connection_type == 'sum': outputs = layers.add([residual, inputs]) elif skip_connection_type == 'none': outputs = residual if return_skip: return outputs, skip else: return outputs
def build_model(fragment_length, nb_filters, nb_output_bins, dilation_depth, nb_stacks, use_skip_connections, learn_all_outputs, _log, desired_sample_rate, use_bias, res_l2, final_l2): def residual_block(x): original_x = x # TODO: initalization, regularization? # Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet. tanh_out = CausalAtrousConvolution1D(nb_filters, 2, dilation_rate=2 ** i, padding='valid', causal=True, use_bias=use_bias, name='dilated_conv_%d_tanh_s%d' % (2 ** i, s), activation='tanh', kernel_regularizer=l2(res_l2))(x) sigm_out = CausalAtrousConvolution1D(nb_filters, 2, dilation_rate=2 ** i, padding='valid', causal=True, use_bias=use_bias, name='dilated_conv_%d_sigm_s%d' % (2 ** i, s), activation='sigmoid', kernel_regularizer=l2(res_l2))(x) x = layers.Multiply(name='gated_activation_%d_s%d' % (i, s))([tanh_out, sigm_out]) res_x = layers.Convolution1D(nb_filters, 1, padding='same', use_bias=use_bias, kernel_regularizer=l2(res_l2))(x) skip_x = layers.Convolution1D(nb_filters, 1, padding='same', use_bias=use_bias, kernel_regularizer=l2(res_l2))(x) res_x = layers.add([original_x, res_x]) return res_x, skip_x input = Input(shape=(fragment_length, nb_output_bins), name='input_part') out = input skip_connections = [] out = CausalAtrousConvolution1D(nb_filters, 2, dilation_rate=1, padding='valid', causal=True, name='initial_causal_conv' )(out) for s in range(nb_stacks): for i in range(0, dilation_depth + 1): out, skip_out = residual_block(out) skip_connections.append(skip_out) if use_skip_connections: out = layers.add(skip_connections) out = layers.Activation('relu')(out) out = layers.Convolution1D(nb_output_bins, 1, padding='same', kernel_regularizer=l2(final_l2))(out) out = layers.Activation('relu')(out) out = layers.Convolution1D(nb_output_bins, 1, padding='same')(out) if not learn_all_outputs: raise DeprecationWarning('Learning on just all outputs is wasteful, now learning only inside receptive field.') out = layers.Lambda(lambda x: x[:, -1, :], output_shape=(out._keras_shape[-1],))( out) # Based on gif in deepmind blog: take last output? out = layers.Activation('softmax', name="output_softmax")(out) model = Model(input, out) receptive_field, receptive_field_ms = compute_receptive_field() _log.info('Receptive Field: %d (%dms)' % (receptive_field, int(receptive_field_ms))) return model
def conv_block(input_tensor, kernel_size, filters, stride, stage, block, strides=(2, 2), trainable=True): """A block that has a conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: filters stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names # Returns Output tensor for the block. Note that from stage 3, the first conv layer at main path is with strides=(2,2) And the shortcut should have strides=(2,2) as well """ bn_axis = 3 conv_name_1 = 'conv' + str(stage) + '_' + str(block) + '_3x3_1' bn_name_1 = 'conv' + str(stage) + '_' + str(block) + '_3x3_1/bn' x = Conv2D(filters, (3, 3), strides=(stride, stride), padding='same', kernel_initializer=tf.initializers.glorot_normal(), use_bias=False, kernel_regularizer=l2(WEIGHT_DECAY), trainable=trainable, name=conv_name_1)(input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_1)(x) x = Activation('relu')(x) conv_name_2 = 'conv' + str(stage) + '_' + str(block) + '_3x3_2' bn_name_2 = 'conv' + str(stage) + '_' + str(block) + '_3x3_2/bn' x = Conv2D(filters, kernel_size, strides=(1, 1), padding='same', kernel_initializer=tf.initializers.glorot_normal(), use_bias=False, kernel_regularizer=l2(WEIGHT_DECAY), trainable=trainable, name=conv_name_2)(x) x = BatchNormalization(axis=bn_axis, name=bn_name_2)(x) conv_name_3 = 'conv' + str(stage) + '_' + str(block) + '_3x3_proj' bn_name_3 = 'conv' + str(stage) + '_' + str(block) + '_3x3_proj/bn' shortcut = Conv2D(filters, (1, 1), strides=(stride, stride), padding='same', kernel_initializer=tf.initializers.glorot_normal(), use_bias=False, kernel_regularizer=l2(WEIGHT_DECAY), trainable=trainable, name=conv_name_3)(input_tensor) shortcut = BatchNormalization(axis=bn_axis, name=bn_name_3)(shortcut) x = layers.add([x, shortcut]) x = Activation('relu')(x) return x
def identity_block(input_tensor, kernel_size, filters, stage, block): """The identity block is the block that has no conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names # Returns Output tensor for the block. """ filters1, filters2, filters3 = filters if backend.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = layers.Conv2D(filters1, (1, 1), use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2a')(input_tensor) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2a')(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters2, kernel_size, padding='same', use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2b')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2b')(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters3, (1, 1), use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2c')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2c')(x) x = layers.add([x, input_tensor]) x = layers.Activation('relu')(x) return x
def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)): """A block that has a conv layer at shortcut. Arguments: input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names strides: Strides for the first conv layer in the block. Returns: Output tensor for the block. Note that from stage 3, the first conv layer at main path is with strides=(2, 2) And the shortcut should have strides=(2, 2) as well """ filters1, filters2, filters3 = filters if K.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = Conv2D(filters1, (1, 1), strides=strides, name=conv_name_base + '2a')(input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) x = Activation('relu')(x) x = Conv2D(filters2, kernel_size, padding='same', name=conv_name_base + '2b')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) x = Activation('relu')(x) x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) shortcut = Conv2D(filters3, (1, 1), strides=strides, name=conv_name_base + '1')(input_tensor) shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut) x = layers.add([x, shortcut]) x = Activation('relu')(x) return x
def call(self, inputs): if not self.pre_act: x = self.batch1(inputs) x = self.ac1(x) x = self.conv1(x) x = self.batch2(x) x = self.ac2(x) x = self.drop_1(x) x = self.conv2(x) x = self.batch3(x) x = self.ac3(x) x = self.drop_2(x) pre_x = self.downsample(inputs) squeeze = self.gavgpool(x) squeeze = self.dnn_1(squeeze) squeeze = self.ac4(squeeze) squeeze = self.dnn_2(squeeze) squeeze = tf.nn.sigmoid(squeeze) squeeze = tf.reshape(squeeze, [-1, 1, self.out_dim]) x = x * squeeze x = layers.add([x, pre_x]) else: x = self.conv1(inputs) x = self.batch1(x) x = self.ac1(x) x = self.drop_1(x) x = self.conv2(x) x = self.batch2(x) x = self.ac2(x) x = self.drop_2(x) pre_x = self.downsample(inputs) x = self.batch3(x) squeeze = self.gavgpool(x) squeeze = self.dnn_1(squeeze) squeeze = self.ac4(squeeze) squeeze = self.dnn_2(squeeze) squeeze = tf.nn.sigmoid(squeeze) squeeze = tf.reshape(squeeze, [-1, 1, self.out_dim]) x = x * squeeze x = layers.add([x, pre_x]) x = self.ac3(x) return x
def get_linear_logit(linear_term, dense_input_, l2_reg): if len(linear_term) > 1: linear_term = add(linear_term) elif len(linear_term) == 1: linear_term = linear_term[0] else: linear_term = None dense_input = list(dense_input_.values()) if len(dense_input) > 0: dense_input__ = dense_input[0] if len( dense_input) == 1 else Concatenate()(dense_input) linear_dense_logit = Dense( 1, activation=None, use_bias=False, kernel_regularizer=l2(l2_reg))(dense_input__) if linear_term is not None: linear_term = add([linear_dense_logit, linear_term]) else: linear_term = linear_dense_logit return linear_term
def call(self, inputs, training=False): x = self._conv2d_1(inputs) x = self._bn_1(x, training=training) x = self.activation_1(x) x = self._conv2d_2(x) x = self._bn_2(x, training=training) x = self._activation_2(x) x = self._conv2d_3(x) x = self._bn_3(x, training=training) x = layers.add([x, inputs]) return self._activation_e3(x)
def _identity_block(input_tensor, kernel_size, filters, stage, block, strides=(1, 1)): """The identity block is the block that has no conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names # Returns Output tensor for the block. """ con_name_base = "res" + str(stage) + block + "_branch" bn_name_base = 'bn' + str(stage) + block + '_branch' filters1, filters2, filters3 = filters # 创建第一个conv2d-BN-relu小模块,卷积核大小为(1, 1) x = _conv_bn(input_tensor, filters1, (1, 1), strides, con_name_base + "2a", bn_name_base + "2a") x = layers.Activation("relu")(x) # 创建第二个conv2d-BN-relu小模块,卷积核大小为(3, 3) x = _conv_bn(x, filters2, kernel_size, strides, con_name_base + "2b", bn_name_base + "2b", padding="same") x = layers.Activation("relu")(x) # 创建第3个conv2d-BN-relu小模块,卷积核大小为(1, 1) x = _conv_bn(x, filters3, (1, 1), strides, con_name_base + "2c", bn_name_base + "2c") #add x = layers.add([x, input_tensor]) x = layers.Activation("relu")(x) return x
def get_linear_logit(linear_emb_list, dense_input_dict, l2_reg): if len(linear_emb_list) > 1: linear_term = add(linear_emb_list) elif len(linear_emb_list) == 1: linear_term = linear_emb_list[0] else: linear_term = None dense_input = list(dense_input_dict.values()) if len(dense_input) > 0: dense_input__ = dense_input[0] if len( dense_input) == 1 else Concatenate()(dense_input) linear_dense_logit = Dense( 1, activation=None, use_bias=False, kernel_regularizer=l2(l2_reg))(dense_input__) if linear_term is not None: linear_term = add([linear_dense_logit, linear_term]) else: linear_term = linear_dense_logit return linear_term
def decoder(x_4, x_2, x_1): x_4 = Conv2D(filters=12, kernel_size=1, padding='same', kernel_regularizer=regularizers.l2(weight_decay))(x_4) dec_4 = Lambda(upscale, arguments={ 'method': interpolation, 'up_factor': 8 })(x_4) if stride == 4: return dec_4 x_2 = Conv2D(filters=12, kernel_size=1, padding='same', kernel_regularizer=regularizers.l2(weight_decay))(x_2) up_42 = Lambda(upscale, arguments={'method': interpolation})(x_4) add_2 = add([up_42, x_2]) dec_2 = Lambda(upscale, arguments={ 'method': interpolation, 'up_factor': 4 })(add_2) if stride == 2: return dec_2 x_1 = Conv2D(filters=12, kernel_size=1, padding='same', kernel_regularizer=regularizers.l2(weight_decay))(x_1) up_21 = Lambda(upscale, arguments={'method': interpolation})(add_2) add_1 = add([up_21, x_1]) dec_1 = Lambda(upscale, arguments={ 'method': interpolation, 'up_factor': 2 })(add_1) return dec_1
def WideDeep(): # embedding_size=8 hidden_size = (128, 128) l2_reg_linear = 1e-5 l2_reg_embedding = 1e-5 l2_reg_deep = 0 init_std = 0.0001 seed = 1024 keep_prob = 1 activation = 'relu' final_activation = 'relu' wide_features = pd.read_csv('data/path_matrix.txt', sep=' ', header=None, nrows=1) deep_features = pd.read_csv('data/sns_dense.csv', sep=',', header=0, nrows=2) wide_input = Input(shape=(wide_features.shape[1], ), name='wide_' + str(wide_features.shape[1])) wide_term = Dense(1, use_bias=False, activation=None)(wide_input) deep_input = deep_features.iloc[:, :-1] deep_feats = { feat: Input(shape=(1, ), name=feat + '_' + str(i)) for i, feat in enumerate(deep_input) } deep_list = [v for v in deep_feats.values()] deep_input = Concatenate()(deep_list) deep_input = Flatten()(deep_input) # hidden_size, activation='relu', l2_reg=0, keep_prob=1, use_bn=False, seed=1024 deep_out = MLP(hidden_size=hidden_size, activation=activation, l2_reg=l2_reg_deep, keep_prob=keep_prob, use_bn=False, seed=seed)(deep_input) deep_logit = Dense(1, use_bias=False, activation=None)(deep_out) final_logit = add([deep_logit, wide_term]) output = PredictionLayer(final_activation)(final_logit) deep_list.append(wide_input) model = Model(inputs=deep_list, outputs=output) model.summary() keras.utils.plot_model(model, to_file='image/widedeep_model.png') return model
def real_distances(L, num_blocks, intermediate_n_channels, input_n_channels): print('') print('Model params:') print('L', L) print('num_blocks', num_blocks) print('intermediate_n_channels', intermediate_n_channels) print('input_n_channels', input_n_channels) print('') dropout_value = 0.2 my_input = Input(shape=(L, L, input_n_channels)) tower = BatchNormalization()(my_input) tower = Activation('elu')(tower) tower = Convolution2D(intermediate_n_channels, 1, padding='same')(tower) flag_1D = False d_rate = 1 for i in range(num_blocks): block = BatchNormalization()(tower) block = Activation('elu')(block) if flag_1D: block = Convolution2D(intermediate_n_channels, kernel_size=(1, 5), padding='same')(block) else: block = Convolution2D(intermediate_n_channels, kernel_size=(3, 3), padding='same')(block) block = Dropout(dropout_value)(block) block = Activation('elu')(block) if flag_1D: block = Convolution2D(intermediate_n_channels, kernel_size=(1, 5), dilation_rate=(d_rate, d_rate), padding='same')(block) flag_1D = False else: block = Convolution2D(intermediate_n_channels, kernel_size=(3, 3), dilation_rate=(d_rate, d_rate), padding='same')(block) flag_1D = True tower = add([block, tower]) if d_rate == 1: d_rate = 2 elif d_rate == 2: d_rate = 4 else: d_rate = 1 tower = BatchNormalization()(tower) tower = Activation('relu')(tower) tower = Convolution2D(1, 3, padding='same')(tower) tower = Activation('relu')(tower) model = Model(my_input, tower) return model
def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2), use_bias=True, train_bn=True): """conv_block is the block that has a conv layer at shortcut # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the nb_filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names use_bias: Boolean. To use or not use a bias in conv layers. train_bn: Boolean. Train or freeze Batch Norm layers Note that from stage 3, the first conv layer at main path is with subsample=(2,2) And the shortcut should have subsample=(2,2) as well """ nb_filter1, nb_filter2, nb_filter3 = filters conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = Conv2D(nb_filter1, (1, 1), strides=strides, name=conv_name_base + '2a', use_bias=use_bias)(input_tensor) x = BatchNormalization(name=bn_name_base + '2a')(x, training=train_bn) x = Activation('relu')(x) x = Conv2D(nb_filter2, (kernel_size, kernel_size), padding='same', name=conv_name_base + '2b', use_bias=use_bias)(x) x = BatchNormalization(name=bn_name_base + '2b')(x, training=train_bn) x = Activation('relu')(x) x = Conv2D(nb_filter3, (1, 1), name=conv_name_base + '2c', use_bias=use_bias)(x) x = BatchNormalization(name=bn_name_base + '2c')(x, training=train_bn) shortcut = Conv2D(nb_filter3, (1, 1), strides=strides, name=conv_name_base + '1', use_bias=use_bias)(input_tensor) shortcut = BatchNormalization(name=bn_name_base + '1')(shortcut, training=train_bn) x = add([x, shortcut]) x = Activation('relu', name='res' + str(stage) + block + '_out')(x) return x
def Unit(x, filters, pool=False): res = x if pool: # makes res & out the same shape x = MaxPooling2D(pool_size=(2, 2))(x) res = Conv2D(filters=filters, kernel_size=[1, 1], strides=(2, 2), padding="same")(res) out = Conv2D(filters, (1, 1), padding='same', activation='relu')(x) out = Conv2D(filters, (1, 1), padding='same', activation='relu')(out) out = add([res, out]) # res & out must be same shape return out
def identity_block_5d(input_tensor, kernel_size, filters, stage, block, l2_reg): """The identity block is the block that has no conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names # Returns Output tensor for the block. """ filters1, filters2, filters3 = filters bn_axis = -1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = layers.TimeDistributed(layers.Conv2D( filters1, (1, 1), kernel_regularizer=regularizers.l2(l2_reg), bias_regularizer=regularizers.l2(l2_reg), kernel_initializer='he_normal'), name=conv_name_base + '2a')(input_tensor) x = layers.TimeDistributed(BatchNorm(axis=-1), name=bn_name_base + '2a')(x) x = layers.Activation('relu')(x) x = layers.TimeDistributed(layers.Conv2D( filters2, kernel_size, padding='same', kernel_regularizer=regularizers.l2(l2_reg), bias_regularizer=regularizers.l2(l2_reg), kernel_initializer='he_normal'), name=conv_name_base + '2b')(x) x = layers.TimeDistributed(BatchNorm(axis=bn_axis), name=bn_name_base + '2b')(x) x = layers.Activation('relu')(x) x = layers.TimeDistributed(layers.Conv2D( filters3, (1, 1), kernel_regularizer=regularizers.l2(l2_reg), bias_regularizer=regularizers.l2(l2_reg), kernel_initializer='he_normal'), name=conv_name_base + '2c')(x) x = layers.TimeDistributed(BatchNorm(axis=bn_axis), name=bn_name_base + '2c')(x) x = layers.add([x, input_tensor]) x = layers.Activation('relu')(x) return x
def identity_block(input_tensor, kernel_size, filters, stage, block, path, non_degenerate_temporal_conv=False): """The identity block is the block that has no conv layer at shortcut. Arguments: input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names Returns: Output tensor for the block. """ filters1, filters2, filters3 = filters if K.image_data_format() == 'channels_last': bn_axis = 4 else: bn_axis = 1 conv_name_base = str(path) + 'res' + str(stage) + block + '_branch' bn_name_base = str(path) + 'bn' + str(stage) + block + '_branch' if non_degenerate_temporal_conv == True: x = Conv3D(filters1, (3, 1, 1), padding='same', name=conv_name_base + '2a')(input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) x = Activation('relu')(x) else: x = Conv3D(filters1, (1, 1, 1), name=conv_name_base + '2a')(input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) x = Activation('relu')(x) x = Conv3D(filters2, kernel_size, padding='same', name=conv_name_base + '2b')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) x = Activation('relu')(x) x = Conv3D(filters3, (1, 1, 1), name=conv_name_base + '2c')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) x = layers.add([x, input_tensor]) x = Activation('relu')(x) return x
def residual_block(x, out_filters, increase=False, name=None): strides = 1 if increase: strides = 2 o1 = BatchNormalization(momentum=0.9, epsilon=1e-5)(x) if name is not None: o1 = Activation('relu', name=name)(o1) else: o1 = Activation('relu')(o1) conv1 = Conv2D(out_filters, kernel_size=3, padding='SAME', strides=strides, kernel_regularizer=regularizers.l2(weight_decay), kernel_initializer='he_normal')(o1) conv1 = BatchNormalization(momentum=0.9, epsilon=1e-5)(conv1) conv1 = Activation('relu')(conv1) conv2 = Conv2D(out_filters, kernel_size=3, padding='SAME', strides=1, kernel_regularizer=regularizers.l2(weight_decay), kernel_initializer='he_normal')(conv1) if increase: proj = Conv2D(out_filters, kernel_size=1, padding='SAME', strides=strides, kernel_regularizer=regularizers.l2(weight_decay), kernel_initializer='he_normal')(o1) proj = add([conv2, proj]) else: proj = add([x, conv2]) return proj
def call(self, inputs, training=False): x = self._conv2d_1(inputs) x = self._bn_1(x, training=training) x = self._activation_1(x) x = self._conv2d_2(x) x = self._bn_2(x, training=training) x = self._activation_2(x) x = self._conv2d_3(x) x = self._bn_3(x, training=training) shortcut = self._shortcut(inputs) shortcut = self._bn_4(shortcut, training=training) x = layers.add([x, shortcut]) return self._activation_4(x)
def call(self, inputs, training=None): out = self.conv1(inputs) out = self.bn1(out) out = self.relu(out) #将两个卷积层组合 out = self.conv2(out) out = self.bn2(out) #定义短接层 identity = self.downsample(inputs) #用Add将卷积层与短接层相加 output = layers.add([out, identity]) output = tf.nn.relu(output) return output
def identity_block(input_tensor, kernel_size, filters, stage, block, dilation=1, conv_trainable=True): """The identity block is the block that has no conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names # Returns Output tensor for the block. """ filters1, filters2, filters3 = filters conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = klayers.Conv2D(filters1, (1, 1), kernel_initializer='he_uniform', trainable=conv_trainable, name=conv_name_base + '2a')(input_tensor) x = klayers.BatchNormalizationV2(name=bn_name_base + '2a')(x) x = klayers.Activation('relu')(x) x = klayers.Conv2D(filters2, kernel_size, dilation_rate=dilation, padding='same', kernel_initializer='he_uniform', trainable=conv_trainable, name=conv_name_base + '2b')(x) x = klayers.BatchNormalizationV2(name=bn_name_base + '2b')(x) x = klayers.Activation('relu')(x) x = klayers.Conv2D(filters3, (1, 1), kernel_initializer='he_uniform', trainable=conv_trainable, name=conv_name_base + '2c')(x) x = klayers.BatchNormalizationV2(name=bn_name_base + '2c')(x) x = klayers.add([x, input_tensor]) x = klayers.Activation('relu')(x) return x
def _resnet_bottleneck_block(input, filters, k=1, strides=(1, 1)): ''' Adds a pre-activation resnet block with bottleneck layers Args: input: input tensor filters: number of output filters k: width factor strides: strides of the convolution layer Returns: a keras tensor ''' init = input channel_axis = 1 if K.image_data_format() == "channels_first" else -1 bottleneck_expand = 4 x = BatchNormalization(axis=channel_axis)(input) x = Activation('relu')(x) if strides != (1, 1) or K.int_shape( init)[channel_axis] != bottleneck_expand * filters * k: init = Conv2D(bottleneck_expand * filters * k, (1, 1), padding='same', kernel_initializer='he_normal', use_bias=False, strides=strides)(x) x = Conv2D(filters * k, (1, 1), padding='same', kernel_initializer='he_normal', use_bias=False)(x) x = BatchNormalization(axis=channel_axis)(x) x = Activation('relu')(x) x = Conv2D(filters * k, (3, 3), padding='same', kernel_initializer='he_normal', use_bias=False, strides=strides)(x) x = BatchNormalization(axis=channel_axis)(x) x = Activation('relu')(x) x = Conv2D(bottleneck_expand * filters * k, (1, 1), padding='same', kernel_initializer='he_normal', use_bias=False)(x) # squeeze and excite block x = squeeze_excite_block(x) m = add([x, init]) return m
def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)): """A block that has a conv layer at shortcut. Arguments: input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names strides: Strides for the first conv layer in the block. Returns: Output tensor for the block. Note that from stage 3, the first conv layer at main path is with strides=(2, 2) And the shortcut should have strides=(2, 2) as well """ filters1, filters2, filters3 = filters if K.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = Conv2D( filters1, (1, 1), strides=strides, name=conv_name_base + '2a')( input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) x = Activation('relu')(x) x = Conv2D( filters2, kernel_size, padding='same', name=conv_name_base + '2b')( x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) x = Activation('relu')(x) x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) shortcut = Conv2D( filters3, (1, 1), strides=strides, name=conv_name_base + '1')( input_tensor) shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut) x = layers.add([x, shortcut]) x = Activation('relu')(x) return x
def identity_block_v2(input_tensor, kernel_size, filters, stage, block, trainable=True): """The identity block is the block that has no conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filterss of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names # Returns Output tensor for the block. """ filters1, filters2, filters3 = filters bn_axis = 3 conv_name_1 = 'conv' + str(stage) + '_' + str(block) + '_1x1_reduce' bn_name_1 = 'conv' + str(stage) + '_' + str(block) + '_1x1_reduce/bn' x = Conv2D(filters1, (1, 1), kernel_initializer=tf.initializers.glorot_normal(), use_bias=False, kernel_regularizer=l2(WEIGHT_DECAY), trainable=trainable, name=conv_name_1)(input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_1)(x) x = Activation('relu')(x) conv_name_2 = 'conv' + str(stage) + '_' + str(block) + '_3x3' bn_name_2 = 'conv' + str(stage) + '_' + str(block) + '_3x3/bn' x = Conv2D(filters2, kernel_size, padding='same', kernel_initializer=tf.initializers.glorot_normal(), use_bias=False, kernel_regularizer=l2(WEIGHT_DECAY), trainable=trainable, name=conv_name_2)(x) x = BatchNormalization(axis=bn_axis, name=bn_name_2)(x) x = Activation('relu')(x) conv_name_3 = 'conv' + str(stage) + '_' + str(block) + '_1x1_increase' bn_name_3 = 'conv' + str(stage) + '_' + str(block) + '_1x1_increase/bn' x = Conv2D(filters3, (1, 1), kernel_initializer=tf.initializers.glorot_normal(), use_bias=False, kernel_regularizer=l2(WEIGHT_DECAY), trainable=trainable, name=conv_name_3)(x) x = BatchNormalization(axis=bn_axis, name=bn_name_3)(x) x = layers.add([x, input_tensor]) x = Activation('relu')(x) return x
def identity_block(input_tensor, kernel_size, filters, stage, block): """The identity block is the block that has no conv layer at shortcut. Args: input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: integer, filters of the bottleneck layer. stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names Returns: Output tensor for the block. """ if backend.image_data_format() == 'channels_last': bn_axis = -1 else: bn_axis = 1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = layers.Conv2D(filters=filters, kernel_size=1, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2a')(input_tensor) x = layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters=filters, kernel_size=kernel_size, padding='same', kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2b')(x) x = layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters=4 * filters, kernel_size=1, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2c')(x) x = layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) x = layers.add([x, input_tensor]) x = layers.Activation('relu')(x) return x
def xception_block(inputs, depth_list, prefix, skip_type, stride, rate=1, depth_activation=False, return_skip=False): residual_inputs = inputs for i in range(3): residual_inputs = separable_conv_with_batch_normalization( residual_inputs, depth_list[i], prefix + 'xception_separable_conv{}'.format(i + 1), stride=stride if i == 2 else 1, rate=rate, depth_activation=depth_activation, epsilon=1e-3) if i == 1: skip = residual_inputs if skip_type == 'conv': shortcut = shortcut_conv(inputs, depth_list[-1], prefix + '_shortcut', kernel_size=1, stride=stride) shortcut = BatchNormalization( name=prefix + '_shortcut_batch_normalization')(shortcut) outputs = layers.add([residual_inputs, shortcut]) elif skip_type == 'sum': outputs = layers.add([residual_inputs, inputs]) elif skip_type == 'none': outputs = residual_inputs if return_skip: return outputs, skip else: return outputs
def channel_spatial_squeeze_excite(input_tensor): """ Create a spatial squeeze-excite block Args: input_tensor: input Keras tensor Returns: a Keras tensor References - [Squeeze and Excitation Networks](https://arxiv.org/abs/1709.01507) - [Concurrent Spatial and Channel Squeeze & Excitation in Fully Convolutional Networks](https://arxiv.org/abs/1803.02579) """ cse = squeeze_excite_block(input_tensor) sse = spatial_squeeze_excite_block(input_tensor) x = add([cse, sse]) return x
def residual_connection_model(): x = Input(shape=(12, 4, 256)) # padding 填充和之前size一样 # 128 是通道数量,3是卷积核尺寸 # 进行卷积后 通道数128将成为第三个维度,之前两个维度由于补全不变 y = layers.Conv2D(128, 3, activation='relu', padding='same')(x) # 使用1x1卷积核使得尺寸可以相加 # strides 是卷积步长 residual = layers.Conv2D(128, 1, strides=1, padding='same')(x) y = layers.add([y, residual]) prediction = layers.Dense(1)(y) model = Model(x, prediction) return model
def __call__(self, input_tensor): x = self.conv1(input_tensor) x = self.bn1(x) x = self.relu(x) x = self.conv2(x) x = self.bn2(x) x = self.relu(x) x = self.conv3(x) x = self.bn3(x) x = layers.add([x, input_tensor]) x = self.relu(x) return x
def conv_block_1d(input_tensor, kernel_size, filters, stage, block, strides=(2)): """A block that has a conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names # Returns Output tensor for the block. Note that from stage 3, the first conv layer at main path is with strides=(2,2) And the shortcut should have strides=(2,2) as well """ filters1, filters2, filters3 = filters conv_name_base = 'res_1d' + str(stage) + block + '_branch' bn_name_base = 'bn_1d' + str(stage) + block + '_branch' x = Conv1D(filters1, (1), strides=strides, name=conv_name_base + '2a')(input_tensor) x = BatchNormalization(name=bn_name_base + '2a')(x) x = Activation('relu')(x) x = Conv1D(filters2, kernel_size, padding='same', name=conv_name_base + '2b')(x) x = BatchNormalization(name=bn_name_base + '2b')(x) x = Activation('relu')(x) x = Conv1D(filters3, (1), name=conv_name_base + '2c')(x) x = BatchNormalization(name=bn_name_base + '2c')(x) shortcut = Conv1D(filters3, (1), strides=strides, name=conv_name_base + '1')(input_tensor) shortcut = BatchNormalization(name=bn_name_base + '1')(shortcut) x = layers.add([x, shortcut]) x = Activation('relu')(x) return x
def resnet_first_block_first_module(input, channel_depth): residual_input = input stride = 1 residual_input = Conv2D(channel_depth, kernel_size=1, strides=1, padding="same")(residual_input) residual_input = BatchNormalization()(residual_input) input = Conv2D(int(channel_depth/4), kernel_size=1, strides=stride, padding="same")(input) input = BatchNormalization()(input) input = Activation("relu")(input) input = Conv2D(int(channel_depth / 4), kernel_size=3, strides=stride, padding="same")(input) input = BatchNormalization()(input) input = Activation("relu")(input) input = Conv2D(channel_depth, kernel_size=1, strides=stride, padding="same")(input) input = BatchNormalization()(input) input = add([input, residual_input]) input = Activation("relu")(input) return input
def identity_block(input_tensor, kernel_size, filters, stage, block): """The identity block is the block that has no conv layer at shortcut. Arguments: input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names Returns: Output tensor for the block. """ filters1, filters2, filters3 = filters if K.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = Conv2D(filters1, (1, 1), name=conv_name_base + '2a')(input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) x = Activation('relu')(x) x = Conv2D( filters2, kernel_size, padding='same', name=conv_name_base + '2b')( x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) x = Activation('relu')(x) x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) x = layers.add([x, input_tensor]) x = Activation('relu')(x) return x
def resnet_module(input, channel_depth, strided_pool=False ): residual_input = input stride = 1 if(strided_pool): stride = 2 residual_input = Conv2D(channel_depth, kernel_size=1, strides=stride, padding="same")(residual_input) residual_input = BatchNormalization()(residual_input) input = Conv2D(int(channel_depth/4), kernel_size=1, strides=stride, padding="same")(input) input = BatchNormalization()(input) input = Activation("relu")(input) input = Conv2D(int(channel_depth / 4), kernel_size=3, strides=1, padding="same")(input) input = BatchNormalization()(input) input = Activation("relu")(input) input = Conv2D(channel_depth, kernel_size=1, strides=1, padding="same")(input) input = BatchNormalization()(input) input = add([input, residual_input]) input = Activation("relu")(input) return input
def _reduction_a_cell(ip, p, filters, block_id=None): """Adds a Reduction cell for NASNet-A (Fig. 4 in the paper). Arguments: ip: Input tensor `x` p: Input tensor `p` filters: Number of output filters block_id: String block_id Returns: A Keras tensor """ channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 with K.name_scope('reduction_A_block_%s' % block_id): p = _adjust_block(p, ip, filters, block_id) h = Activation('relu')(ip) h = Conv2D( filters, (1, 1), strides=(1, 1), padding='same', name='reduction_conv_1_%s' % block_id, use_bias=False, kernel_initializer='he_normal')( h) h = BatchNormalization( axis=channel_dim, momentum=0.9997, epsilon=1e-3, name='reduction_bn_1_%s' % block_id)( h) with K.name_scope('block_1'): x1_1 = _separable_conv_block( h, filters, (5, 5), strides=(2, 2), block_id='reduction_left1_%s' % block_id) x1_2 = _separable_conv_block( p, filters, (7, 7), strides=(2, 2), block_id='reduction_1_%s' % block_id) x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % block_id) with K.name_scope('block_2'): x2_1 = MaxPooling2D( (3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % block_id)( h) x2_2 = _separable_conv_block( p, filters, (7, 7), strides=(2, 2), block_id='reduction_right2_%s' % block_id) x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % block_id) with K.name_scope('block_3'): x3_1 = AveragePooling2D( (3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % block_id)( h) x3_2 = _separable_conv_block( p, filters, (5, 5), strides=(2, 2), block_id='reduction_right3_%s' % block_id) x3 = add([x3_1, x3_2], name='reduction_add3_%s' % block_id) with K.name_scope('block_4'): x4 = AveragePooling2D( (3, 3), strides=(1, 1), padding='same', name='reduction_left4_%s' % block_id)( x1) x4 = add([x2, x4]) with K.name_scope('block_5'): x5_1 = _separable_conv_block( x1, filters, (3, 3), block_id='reduction_left4_%s' % block_id) x5_2 = MaxPooling2D( (3, 3), strides=(2, 2), padding='same', name='reduction_right5_%s' % block_id)( h) x5 = add([x5_1, x5_2], name='reduction_add4_%s' % block_id) x = concatenate( [x2, x3, x4, x5], axis=channel_dim, name='reduction_concat_%s' % block_id) return x, ip
def WDL(deep_feature_dim_dict, wide_feature_dim_dict, embedding_size=8, hidden_size=(128, 128), l2_reg_linear=1e-5, l2_reg_embedding=1e-5, l2_reg_deep=0, init_std=0.0001, seed=1024, keep_prob=1, activation='relu', final_activation='sigmoid',): """Instantiates the Wide&Deep Learning architecture. :param deep_feature_dim_dict: dict,to indicate sparse field and dense field in deep part like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']} :param wide_feature_dim_dict: dict,to indicate sparse field and dense field in wide part like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']} :param embedding_size: positive integer,sparse feature embedding_size :param hidden_size: list,list of positive integer or empty list, the layer number and units in each layer of deep net :param l2_reg_linear: float. L2 regularizer strength applied to wide part :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector :param l2_reg_deep: float. L2 regularizer strength applied to deep net :param init_std: float,to use as the initialize std of embedding vector :param seed: integer ,to use as random seed. :param keep_prob: float in (0,1]. keep_prob used in deep net :param activation: Activation function to use in deep net :param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'`` :return: A Keras model instance. """ if not isinstance(deep_feature_dim_dict, dict) or "sparse" not in deep_feature_dim_dict or "dense" not in deep_feature_dim_dict: raise ValueError( "feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}") sparse_input, dense_input, = create_input_dict( deep_feature_dim_dict) bias_sparse_input, bias_dense_input = create_input_dict( wide_feature_dim_dict, 'bias') sparse_embedding = create_embedding_dict( deep_feature_dim_dict, embedding_size, init_std, seed, l2_reg_embedding) wide_linear_embedding = create_embedding_dict( wide_feature_dim_dict, 1, init_std, seed, l2_reg_linear, 'linear') embed_list = get_embedding_vec_list(sparse_embedding, sparse_input) deep_input = Concatenate()(embed_list) if len( embed_list) > 1 else embed_list[0] deep_input = Flatten()(deep_input) if len(dense_input) > 0: deep_input = Concatenate()([deep_input]+list(dense_input.values())) deep_out = MLP(hidden_size, activation, l2_reg_deep, keep_prob, False, seed)(deep_input) deep_logit = Dense(1, use_bias=False, activation=None)(deep_out) final_logit = deep_logit if len(wide_feature_dim_dict['dense']) + len(wide_feature_dim_dict['sparse']) > 0: if len(wide_feature_dim_dict['sparse']) > 0: bias_embed_list = get_embedding_vec_list( wide_linear_embedding, bias_sparse_input) linear_term = add(bias_embed_list) if len( bias_embed_list) > 1 else bias_embed_list[0] final_logit = add([final_logit, linear_term]) if len(wide_feature_dim_dict['dense']) > 0: wide_dense_term = Dense(1, use_bias=False, activation=None)(Concatenate()( list(bias_dense_input.values())) if len(bias_dense_input) > 1 else list(bias_dense_input.values())[0]) final_logit = add([final_logit, wide_dense_term]) output = PredictionLayer(final_activation)(final_logit) inputs_list = get_inputs_list( [sparse_input, dense_input, bias_sparse_input, bias_dense_input]) model = Model(inputs=inputs_list, outputs=output) return model
def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)): """A block that has a conv layer at shortcut. # Arguments input_tensor: input tensor kernel_size: default 3, the kernel size of middle conv layer at main path filters: list of integers, the filters of 3 conv layer at main path stage: integer, current stage label, used for generating layer names block: 'a','b'..., current block label, used for generating layer names strides: Strides for the second conv layer in the block. # Returns Output tensor for the block. Note that from stage 3, the second conv layer at main path is with strides=(2, 2) And the shortcut should have strides=(2, 2) as well """ filters1, filters2, filters3 = filters if backend.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' x = layers.Conv2D(filters1, (1, 1), use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2a')(input_tensor) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2a')(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters2, kernel_size, strides=strides, padding='same', use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2b')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2b')(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters3, (1, 1), use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '2c')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '2c')(x) shortcut = layers.Conv2D(filters3, (1, 1), strides=strides, use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name=conv_name_base + '1')(input_tensor) shortcut = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name=bn_name_base + '1')(shortcut) x = layers.add([x, shortcut]) x = layers.Activation('relu')(x) return x
def MLR(region_feature_dim_dict, base_feature_dim_dict={"sparse": {}, "dense": []}, region_num=4, l2_reg_linear=1e-5, init_std=0.0001, seed=1024, final_activation='sigmoid', bias_feature_dim_dict={"sparse": {}, "dense": []}): """Instantiates the Mixed Logistic Regression/Piece-wise Linear Model. :param region_feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']} :param base_feature_dim_dict: dict or None,to indicate sparse field and dense field of base learner.if None, it is same as region_feature_dim_dict :param region_num: integer > 1,indicate the piece number :param l2_reg_linear: float. L2 regularizer strength applied to weight :param init_std: float,to use as the initialize std of embedding vector :param seed: integer ,to use as random seed. :param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'`` :param bias_feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']} :return: A Keras model instance. """ if region_num <= 1: raise ValueError("region_num must > 1") if not isinstance(region_feature_dim_dict, dict) or "sparse" not in region_feature_dim_dict or "dense" not in region_feature_dim_dict: raise ValueError( "feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}") same_flag = False if base_feature_dim_dict == {"sparse": {}, "dense": []}: base_feature_dim_dict = region_feature_dim_dict same_flag = True region_sparse_input, region_dense_input, base_sparse_input, base_dense_input, bias_sparse_input, bias_dense_input = get_input( region_feature_dim_dict, base_feature_dim_dict, bias_feature_dim_dict, same_flag) region_embeddings, base_embeddings, bias_embedding = get_embedding( region_num, region_feature_dim_dict, base_feature_dim_dict, bias_feature_dim_dict, init_std, seed, l2_reg_linear) if same_flag: base_dense_input_ = region_dense_input base_sparse_input_ = region_sparse_input else: base_dense_input_ = base_dense_input base_sparse_input_ = base_sparse_input region_dense_feature_num = len(region_feature_dim_dict['dense']) region_sparse_feature_num = len(region_feature_dim_dict['sparse']) base_dense_feature_num = len(base_feature_dim_dict['dense']) base_sparse_feature_num = len(base_feature_dim_dict['sparse']) bias_dense_feature_num = len(bias_feature_dim_dict['dense']) bias_sparse_feature_num = len(bias_feature_dim_dict['sparse']) if region_dense_feature_num > 1: region_dense_logits_ = [Dense(1, )(Concatenate()(region_dense_input)) for _ in range(region_num)] elif region_dense_feature_num == 1: region_dense_logits_ = [Dense(1, )(region_dense_input[0]) for _ in range(region_num)] if base_dense_feature_num > 1: base_dense_logits = [Dense(1, )(Concatenate()(base_dense_input_))for _ in range(region_num)] elif base_dense_feature_num == 1: base_dense_logits = [Dense(1, )(base_dense_input_[0])for _ in range(region_num)] if region_dense_feature_num > 0 and region_sparse_feature_num == 0: region_logits = Concatenate()(region_dense_logits_) elif region_dense_feature_num == 0 and region_sparse_feature_num > 0: region_sparse_logits = [ add([region_embeddings[j][i](region_sparse_input[i]) for i in range(region_sparse_feature_num)]) if region_sparse_feature_num > 1 else region_embeddings[j][0](region_sparse_input[0]) for j in range(region_num)] region_logits = Concatenate()(region_sparse_logits) else: region_sparse_logits = [ add([region_embeddings[j][i](region_sparse_input[i]) for i in range(region_sparse_feature_num)]) for j in range(region_num)] region_logits = Concatenate()( [add([region_sparse_logits[i], region_dense_logits_[i]]) for i in range(region_num)]) if base_dense_feature_num > 0 and base_sparse_feature_num == 0: base_logits = base_dense_logits elif base_dense_feature_num == 0 and base_sparse_feature_num > 0: base_sparse_logits = [add( [base_embeddings[j][i](base_sparse_input_[i]) for i in range(base_sparse_feature_num)]) if base_sparse_feature_num > 1 else base_embeddings[j][0](base_sparse_input_[0]) for j in range(region_num)] base_logits = base_sparse_logits else: base_sparse_logits = [add( [base_embeddings[j][i](base_sparse_input_[i]) for i in range(base_sparse_feature_num)]) if base_sparse_feature_num > 1 else base_embeddings[j][0](base_sparse_input_[0]) for j in range(region_num)] base_logits = [add([base_sparse_logits[i], base_dense_logits[i]]) for i in range(region_num)] # Dense(self.region_num, activation='softmax')(final_logit) region_weights = Activation("softmax")(region_logits) learner_score = Concatenate()( [Activation(final_activation, name='learner' + str(i))(base_logits[i]) for i in range(region_num)]) final_logit = dot([region_weights, learner_score], axes=-1) if bias_dense_feature_num + bias_sparse_feature_num > 0: if bias_dense_feature_num > 1: bias_dense_logits = Dense(1,)(Concatenate()(bias_dense_input)) elif bias_dense_feature_num == 1: bias_dense_logits = Dense(1,)(bias_dense_input[0]) else: pass if bias_sparse_feature_num > 1: bias_cate_logits = add([bias_embedding[i](bias_sparse_input[i]) for i, feat in enumerate(bias_feature_dim_dict['sparse'])]) elif bias_sparse_feature_num == 1: bias_cate_logits = bias_embedding[0](bias_sparse_input[0]) else: pass if bias_dense_feature_num > 0 and bias_sparse_feature_num > 0: bias_logits = add([bias_dense_logits, bias_cate_logits]) elif bias_dense_feature_num > 0: bias_logits = bias_dense_logits else: bias_logits = bias_cate_logits bias_prob = Activation('sigmoid')(bias_logits) final_logit = dot([final_logit, bias_prob], axes=-1) output = Reshape([1])(final_logit) model = Model(inputs=region_sparse_input + region_dense_input+base_sparse_input + base_dense_input+bias_sparse_input+bias_dense_input, outputs=output) return model
def Xception(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000): """Instantiates the Xception architecture. Optionally loads weights pre-trained on ImageNet. This model is available for TensorFlow only, and can only be used with inputs following the TensorFlow data format `(width, height, channels)`. You should set `image_data_format='channels_last'` in your Keras config located at ~/.keras/keras.json. Note that the default input image size for this model is 299x299. Arguments: include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization), 'imagenet' (pre-training on ImageNet), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(299, 299, 3)`. It should have exactly 3 inputs channels, and width and height should be no smaller than 71. E.g. `(150, 150, 3)` would be one valid value. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. Returns: A Keras model instance. Raises: ValueError: in case of invalid argument for `weights`, or invalid input shape. RuntimeError: If attempting to run this model with a backend that does not support separable convolutions. """ if not (weights in {'imagenet', None} or os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `imagenet` ' '(pre-training on ImageNet), ' 'or the path to the weights file to be loaded.') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as imagenet with `include_top`' ' as true, `classes` should be 1000') if K.image_data_format() != 'channels_last': logging.warning( 'The Xception model is only available for the ' 'input data format "channels_last" ' '(width, height, channels). ' 'However your settings specify the default ' 'data format "channels_first" (channels, width, height). ' 'You should set `image_data_format="channels_last"` in your Keras ' 'config located at ~/.keras/keras.json. ' 'The model being returned right now will expect inputs ' 'to follow the "channels_last" data format.') K.set_image_data_format('channels_last') old_data_format = 'channels_first' else: old_data_format = None # Determine proper input shape input_shape = _obtain_input_shape( input_shape, default_size=299, min_size=71, data_format=K.image_data_format(), require_flatten=False, weights=weights) if input_tensor is None: img_input = Input(shape=input_shape) else: if not K.is_keras_tensor(input_tensor): img_input = Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor x = Conv2D( 32, (3, 3), strides=(2, 2), use_bias=False, name='block1_conv1')( img_input) x = BatchNormalization(name='block1_conv1_bn')(x) x = Activation('relu', name='block1_conv1_act')(x) x = Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x) x = BatchNormalization(name='block1_conv2_bn')(x) x = Activation('relu', name='block1_conv2_act')(x) residual = Conv2D( 128, (1, 1), strides=(2, 2), padding='same', use_bias=False)( x) residual = BatchNormalization()(residual) x = SeparableConv2D( 128, (3, 3), padding='same', use_bias=False, name='block2_sepconv1')( x) x = BatchNormalization(name='block2_sepconv1_bn')(x) x = Activation('relu', name='block2_sepconv2_act')(x) x = SeparableConv2D( 128, (3, 3), padding='same', use_bias=False, name='block2_sepconv2')( x) x = BatchNormalization(name='block2_sepconv2_bn')(x) x = MaxPooling2D( (3, 3), strides=(2, 2), padding='same', name='block2_pool')( x) x = layers.add([x, residual]) residual = Conv2D( 256, (1, 1), strides=(2, 2), padding='same', use_bias=False)( x) residual = BatchNormalization()(residual) x = Activation('relu', name='block3_sepconv1_act')(x) x = SeparableConv2D( 256, (3, 3), padding='same', use_bias=False, name='block3_sepconv1')( x) x = BatchNormalization(name='block3_sepconv1_bn')(x) x = Activation('relu', name='block3_sepconv2_act')(x) x = SeparableConv2D( 256, (3, 3), padding='same', use_bias=False, name='block3_sepconv2')( x) x = BatchNormalization(name='block3_sepconv2_bn')(x) x = MaxPooling2D( (3, 3), strides=(2, 2), padding='same', name='block3_pool')( x) x = layers.add([x, residual]) residual = Conv2D( 728, (1, 1), strides=(2, 2), padding='same', use_bias=False)( x) residual = BatchNormalization()(residual) x = Activation('relu', name='block4_sepconv1_act')(x) x = SeparableConv2D( 728, (3, 3), padding='same', use_bias=False, name='block4_sepconv1')( x) x = BatchNormalization(name='block4_sepconv1_bn')(x) x = Activation('relu', name='block4_sepconv2_act')(x) x = SeparableConv2D( 728, (3, 3), padding='same', use_bias=False, name='block4_sepconv2')( x) x = BatchNormalization(name='block4_sepconv2_bn')(x) x = MaxPooling2D( (3, 3), strides=(2, 2), padding='same', name='block4_pool')( x) x = layers.add([x, residual]) for i in range(8): residual = x prefix = 'block' + str(i + 5) x = Activation('relu', name=prefix + '_sepconv1_act')(x) x = SeparableConv2D( 728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv1')( x) x = BatchNormalization(name=prefix + '_sepconv1_bn')(x) x = Activation('relu', name=prefix + '_sepconv2_act')(x) x = SeparableConv2D( 728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv2')( x) x = BatchNormalization(name=prefix + '_sepconv2_bn')(x) x = Activation('relu', name=prefix + '_sepconv3_act')(x) x = SeparableConv2D( 728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv3')( x) x = BatchNormalization(name=prefix + '_sepconv3_bn')(x) x = layers.add([x, residual]) residual = Conv2D( 1024, (1, 1), strides=(2, 2), padding='same', use_bias=False)( x) residual = BatchNormalization()(residual) x = Activation('relu', name='block13_sepconv1_act')(x) x = SeparableConv2D( 728, (3, 3), padding='same', use_bias=False, name='block13_sepconv1')( x) x = BatchNormalization(name='block13_sepconv1_bn')(x) x = Activation('relu', name='block13_sepconv2_act')(x) x = SeparableConv2D( 1024, (3, 3), padding='same', use_bias=False, name='block13_sepconv2')( x) x = BatchNormalization(name='block13_sepconv2_bn')(x) x = MaxPooling2D( (3, 3), strides=(2, 2), padding='same', name='block13_pool')( x) x = layers.add([x, residual]) x = SeparableConv2D( 1536, (3, 3), padding='same', use_bias=False, name='block14_sepconv1')( x) x = BatchNormalization(name='block14_sepconv1_bn')(x) x = Activation('relu', name='block14_sepconv1_act')(x) x = SeparableConv2D( 2048, (3, 3), padding='same', use_bias=False, name='block14_sepconv2')( x) x = BatchNormalization(name='block14_sepconv2_bn')(x) x = Activation('relu', name='block14_sepconv2_act')(x) if include_top: x = GlobalAveragePooling2D(name='avg_pool')(x) x = Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='xception') # load weights if weights == 'imagenet': if include_top: weights_path = get_file( 'xception_weights_tf_dim_ordering_tf_kernels.h5', TF_WEIGHTS_PATH, cache_subdir='models', file_hash='0a58e3b7378bc2990ea3b43d5981f1f6') else: weights_path = get_file( 'xception_weights_tf_dim_ordering_tf_kernels_notop.h5', TF_WEIGHTS_PATH_NO_TOP, cache_subdir='models', file_hash='b0042744bf5b25fce3cb969f33bebb97') model.load_weights(weights_path) elif weights is not None: model.load_weights(weights) if old_data_format: K.set_image_data_format(old_data_format) return model