def MT_CAN(nb_filters1,
nb_filters2,
input_shape,
kernel_size=(3, 3),
dropout_rate1=0.25,
dropout_rate2=0.5,
pool_size=(2, 2),
nb_dense=128):
diff_input = Input(shape=input_shape)
rawf_input = Input(shape=input_shape)
d1 = Conv2D(nb_filters1, kernel_size, padding='same',
activation='tanh')(diff_input)
d2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(d1)
r1 = Conv2D(nb_filters1, kernel_size, padding='same',
activation='tanh')(rawf_input)
r2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(r1)
g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2)
g1 = Attention_mask()(g1)
gated1 = multiply([d2, g1])
d3 = AveragePooling2D(pool_size)(gated1)
d4 = Dropout(dropout_rate1)(d3)
r3 = AveragePooling2D(pool_size)(r2)
r4 = Dropout(dropout_rate1)(r3)
d5 = Conv2D(nb_filters2, kernel_size, padding='same',
activation='tanh')(d4)
d6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(d5)
r5 = Conv2D(nb_filters2, kernel_size, padding='same',
activation='tanh')(r4)
r6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(r5)
g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6)
g2 = Attention_mask()(g2)
gated2 = multiply([d6, g2])
d7 = AveragePooling2D(pool_size)(gated2)
d8 = Dropout(dropout_rate1)(d7)
d9 = Flatten()(d8)
d10_y = Dense(nb_dense, activation='tanh')(d9)
d11_y = Dropout(dropout_rate2)(d10_y)
out_y = Dense(1, name='output_1')(d11_y)
d10_r = Dense(nb_dense, activation='tanh')(d9)
d11_r = Dropout(dropout_rate2)(d10_r)
out_r = Dense(1, name='output_2')(d11_r)
model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r])
return model
def call(self, input):
v1=input[0]
v2=input[1]
sketch_v1 = self.get_sketch_matrix(self.h_s[0], self.h_s[1],v1,self.d)
sketch_v2 = self.get_sketch_matrix(self.h_s[2], self.h_s[3],v2,self.d)
fft_1, fft_2 = tf.fft(sketch_v1), tf.fft(sketch_v2)
fft_product = multiply([fft_1, fft_2])
inv_fft = tf.ifft(fft_product)
sgn_sqrt = multiply([tf.real(tf.sign(inv_fft)), tf.sqrt(tf.abs(inv_fft))])
l2_norm = tf.keras.backend.l2_normalize(sgn_sqrt)
return l2_norm
def line_loss(y_true, y_pred):
'''
y_true = np.vstack([k_weight, odw]).T
'''
r1 = layers.multiply([y_true[:, 1], y_pred])
r2 = K.sigmoid(r1)
r3 = K.log(r2)
r4 = layers.multiply([y_true[:, 0], r3])
result = -K.mean(r4)
return result
def get_model_attention(self, embedding_matrix, vocab_size, h_s_img_text_1, h_s_img_text_2, hidden_units_LSTM=1024, question_len=15, img_feat=2048, embed_dim=300, ):
input_image = Input(shape=(img_feat,))
# Language Model - 2 LSTMs
input_lang = Input(shape=(question_len,))
output_embedding = Embedding(vocab_size, embedding_matrix.shape[1], input_length=question_len,
weights=[embedding_matrix], trainable=False)(input_lang)
print("Embedding done")
output_lstm_1 = LSTM(units=hidden_units_LSTM, return_sequences=True, unroll=True, name='lstm1')(output_embedding)
print("LSTM 1 done")
output_lstm_2 = LSTM(units=hidden_units_LSTM, return_sequences=False, unroll=True, name='lstm2')(output_lstm_1)
print("LSTM 2")
concatenated_lang_features = Concatenate()([output_lstm_1[:,-1,:], output_lstm_2])
#mcb_1 = self.get_mcb_layer(v1=input_image, v2=concatenated_lang_features, h_s=h_s_img_text_1, d=128, n1=2048, n2=2048)
mcb_1 = MyMCBLayer(h_s=h_s_img_text_1, d=128, n1=2048, n2=2048)([input_image, concatenated_lang_features])
# conv_1 = Conv1D(filters=1, kernel_size=32, activation='relu', padding='same')(mcb_1)
# conv_2 = Conv1D(filters=1, kernel_size=32, activation='softmax', padding='same')(conv_1)
weights = Dense(img_feat, activation="softmax")(mcb_1)
weighted_sum = multiply([weights, input_image])
#mcb_2 = self.get_mcb_layer(v1=weighted_sum, v2=concatenated_lang_features, h_s=h_s_img_text_2, d=128,n1=2048, n2=2048)
mcb_2 = MyMCBLayer(h_s=h_s_img_text_2, d=128,n1=2048, n2=2048)([weighted_sum, concatenated_lang_features])
final_fc = Dense(Constants.NUM_CLASSES, activation="softmax")(mcb_2)
model = Model(inputs=[input_image, input_lang], outputs=final_fc)
model.compile(loss="categorical_crossentropy", optimizer="rmsprop" , metrics=['accuracy'] )
model.summary()
return model
def squeeze_excite_block(input, ratio=16):
''' Create a channel-wise squeeze-excite block
Args:
input: input tensor
filters: number of output filters
Returns: a keras tensor
References
- [Squeeze and Excitation Networks](https://arxiv.org/abs/1709.01507)
'''
init = input
channel_axis = 1 if K.image_data_format() == "channels_first" else -1
filters = K.int_shape(init)[channel_axis]
se_shape = (1, 1, filters)
se = GlobalAveragePooling2D()(init)
se = Reshape(se_shape)(se)
se = Dense(filters // ratio,
activation='relu',
kernel_initializer='he_normal',
use_bias=False)(se)
se = Dense(filters,
activation='sigmoid',
kernel_initializer='he_normal',
use_bias=False)(se)
if K.image_data_format() == 'channels_first':
se = Permute((3, 1, 2))(se)
x = multiply([init, se])
return x
def build_generator(self, config_gener):
m = float(config_gener['momentum'])
a = float(config_gener['leaky_relu_alpha'])
model = Sequential()
model.add(
Dense(int(config_gener['dense1']), input_dim=self.embeddings_dim))
model.add(LeakyReLU(alpha=a))
model.add(BatchNormalization(momentum=m))
model.add(Dense(int(config_gener['dense2'])))
model.add(LeakyReLU(alpha=a))
model.add(BatchNormalization(momentum=m))
model.add(Dense(int(config_gener['dense3'])))
model.add(LeakyReLU(alpha=a))
model.add(BatchNormalization(momentum=m))
model.add(
Dense(np.prod(self.img_shape),
activation=config_gener['activation']))
model.add(Reshape(self.img_shape))
noise = Input(shape=(self.embeddings_dim, ))
label = Input(shape=(1, ), dtype='int32')
label_embedding = Flatten()(Embedding(
self.embeddings_vocab,
self.embeddings_dim,
weights=[self.word_embeddings.vectors],
trainable=False)(label))
model_input = multiply([noise, label_embedding])
img = model(model_input)
return Model([noise, label], img)
def attention_block_oktay(g, x, nr_of_convolutions):
"""
Following the original paper and implementation at https://github.com/ozan-oktay/Attention-Gated-Networks
"""
g1 = Convolution3D(nr_of_convolutions,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(g)
g1 = BatchNormalization()(g1)
x1 = MaxPooling3D([2, 2, 2])(x)
x1 = Convolution3D(nr_of_convolutions,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(x1)
x1 = BatchNormalization()(x1)
psi = Concatenate()([g1, x1])
psi = Activation(activation='relu')(psi)
psi = Convolution3D(1,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(psi)
psi = BatchNormalization()(psi)
psi = Activation(activation='sigmoid')(psi)
return multiply([x, psi])
def create_model(self):
K.clear_session()
input0 = Input(shape=(self.c['sentencepad'], self.c['wordvectdim']))
input1 = Input(shape=(self.c['sentencepad'], self.c['wordvectdim']))
Convolt_Layer = []
MaxPool_Layer = []
Flatten_Layer = []
for kernel_size, filters in self.c['cnnfilters'].items():
Convolt_Layer.append(
Convolution1D(filters=filters,
kernel_size=kernel_size,
padding='valid',
activation=self.c['cnnactivate'],
kernel_initializer=self.c['cnninitial']))
MaxPool_Layer.append(
MaxPooling1D(pool_size=int(self.c['sentencepad'] -
kernel_size + 1)))
Flatten_Layer.append(Flatten())
Convolted_tensor0 = []
Convolted_tensor1 = []
for channel in range(len(self.c['cnnfilters'])):
Convolted_tensor0.append(Convolt_Layer[channel](input0))
Convolted_tensor1.append(Convolt_Layer[channel](input1))
MaxPooled_tensor0 = []
MaxPooled_tensor1 = []
for channel in range(len(self.c['cnnfilters'])):
MaxPooled_tensor0.append(MaxPool_Layer[channel](
Convolted_tensor0[channel]))
MaxPooled_tensor1.append(MaxPool_Layer[channel](
Convolted_tensor1[channel]))
Flattened_tensor0 = []
Flattened_tensor1 = []
for channel in range(len(self.c['cnnfilters'])):
Flattened_tensor0.append(Flatten_Layer[channel](
MaxPooled_tensor0[channel]))
Flattened_tensor1.append(Flatten_Layer[channel](
MaxPooled_tensor1[channel]))
if len(self.c['cnnfilters']) > 1:
Flattened_tensor0 = concatenate(Flattened_tensor0)
Flattened_tensor1 = concatenate(Flattened_tensor1)
else:
Flattened_tensor0 = Flattened_tensor0[0]
Flattened_tensor1 = Flattened_tensor1[0]
absDifference = Lambda(lambda X: K.abs(X[0] - X[1]))(
[Flattened_tensor0, Flattened_tensor1])
mulDifference = multiply([Flattened_tensor0, Flattened_tensor1])
allDifference = concatenate([absDifference, mulDifference])
for ilayer, densedimension in enumerate(self.c['densedimension']):
allDifference = Dense(
units=int(densedimension),
activation=self.c['denseactivate'],
kernel_initializer=self.c['denseinitial'])(allDifference)
output = Dense(
name='output',
units=self.c['num_classes'],
activation='softmax',
kernel_initializer=self.c['denseinitial'])(allDifference)
self.model = Model(inputs=[input0, input1], outputs=output)
self.model.compile(loss='mean_squared_error',
optimizer=self.c['optimizer'])
def build_generator(channels, num_classes, latent_dim):
model = Sequential()
model.add(Dense(128 * 7 * 7, activation="relu", input_dim=latent_dim))
model.add(Reshape((7, 7, 128)))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
#7x7x128
model.add(Conv2D(128, kernel_size=3, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(UpSampling2D())
#14x14x128
model.add(Conv2D(64, kernel_size=3, padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
#28x28x64
model.add(Conv2D(channels, kernel_size=3, padding='same'))
model.add(Activation("tanh"))
#28x28x3
model.summary()
noise = Input(shape=(latent_dim, ))
label = Input(shape=(1, ), dtype='int32')
label_embedding = Flatten()(Embedding(num_classes, latent_dim)(label))
model_input = multiply([noise, label_embedding])
img = model(model_input)
return Model([noise, label], img)
def attention_block(g, x, nr_of_convolutions):
"""
Taken from https://github.com/LeeJunHyun/Image_Segmentation
"""
g1 = Convolution3D(nr_of_convolutions,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(g)
g1 = BatchNormalization()(g1)
x1 = Convolution3D(nr_of_convolutions,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(x)
x1 = BatchNormalization()(x1)
psi = Concatenate()([g1, x1])
psi = Activation(activation='relu')(psi)
psi = Convolution3D(1,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(psi)
psi = BatchNormalization()(psi)
psi = Activation(activation='sigmoid')(psi)
return multiply([x, psi])
def build_discriminator(img_shape, num_classes, optimizer):
model = Sequential()
model.add(Dense(512, input_dim=np.prod(img_shape)))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.4))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.4))
model.add(Dense(1, activation='sigmoid'))
model.summary()
img = Input(shape=img_shape)
label = Input(shape=(1, ), dtype='int32')
label_embedding = Flatten()(Embedding(num_classes,
np.prod(img_shape))(label))
flat_img = Flatten()(img)
model_input = multiply([flat_img, label_embedding])
validity = model(model_input)
# Compilar discriminator
Model2 = Model([img, label], validity)
Model2.compile(loss=['binary_crossentropy'],
optimizer=optimizer,
metrics=['accuracy'])
return Model2
def build_generator(img_shape, num_classes, latent_dim):
model = Sequential()
model.add(Dense(256, input_dim=latent_dim))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(1024))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(np.prod(img_shape), activation='tanh'))
model.add(Reshape(img_shape))
model.summary()
noise = Input(shape=(latent_dim, ))
label = Input(shape=(1, ), dtype='int32')
label_embedding = Flatten()(Embedding(num_classes, latent_dim)(label))
model_input = multiply([noise, label_embedding])
img = model(model_input)
return Model([noise, label], img)
def CAN_3D(n_frame,
nb_filters1,
nb_filters2,
input_shape,
kernel_size=(3, 3, 3),
dropout_rate1=0.25,
dropout_rate2=0.5,
pool_size=(2, 2, 2),
nb_dense=128):
diff_input = Input(shape=input_shape)
rawf_input = Input(shape=input_shape)
d1 = Conv3D(nb_filters1, kernel_size, padding='same',
activation='tanh')(diff_input)
d2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(d1)
# Appearance Branch
r1 = Conv3D(nb_filters1, kernel_size, padding='same',
activation='tanh')(rawf_input)
r2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(r1)
g1 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r2)
g1 = Attention_mask()(g1)
gated1 = multiply([d2, g1])
d3 = AveragePooling3D(pool_size)(gated1)
d4 = Dropout(dropout_rate1)(d3)
d5 = Conv3D(nb_filters2, kernel_size, padding='same',
activation='tanh')(d4)
d6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(d5)
r3 = AveragePooling3D(pool_size)(r2)
r4 = Dropout(dropout_rate1)(r3)
r5 = Conv3D(nb_filters2, kernel_size, padding='same',
activation='tanh')(r4)
r6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(r5)
g2 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r6)
g2 = Attention_mask()(g2)
gated2 = multiply([d6, g2])
d7 = AveragePooling3D(pool_size)(gated2)
d8 = Dropout(dropout_rate1)(d7)
d9 = Flatten()(d8)
d10 = Dense(nb_dense, activation='tanh')(d9)
d11 = Dropout(dropout_rate2)(d10)
out = Dense(n_frame)(d11)
model = Model(inputs=[diff_input, rawf_input], outputs=out)
return model
def attention_3d_block(inputs, seq_len=21):
# input_dim = int(inputs.shape[2])
a = Permute((2, 1))(inputs)
a = Dense(seq_len, activation='softmax')(a)
a_probs = Permute((2, 1), name='attention_vec')(a)
# output_attention_mul = merge([inputs, a_probs], name='attention_mul', mode='mul')
output_attention_mul = multiply([inputs, a_probs], name='attention_mul')
return output_attention_mul
def conv_attention_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2, 2)):
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2D(filters1, (1, 1), strides=strides,
name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
y1 = x
x = Activation('relu')(x)
x = Conv2D(filters2,
kernel_size,
padding='same',
name=conv_name_base + '2b')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
y2 = x
x = Activation('relu')(x)
x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
y3 = x
shortcut = Conv2D(filters3, (1, 1),
strides=strides,
name=conv_name_base + '1')(input_tensor)
shortcut = BatchNormalization(axis=bn_axis,
name=bn_name_base + '1')(shortcut)
y4 = shortcut
x = layers.add([x, shortcut])
#将y1,y2,y3,y4的滤波器个数统一
y1 = Conv2D(filters3, (1, 1), name=conv_name_base + '2d')(y1)
y2 = Conv2D(filters3, (1, 1), name=conv_name_base + '2e')(y2)
#concat y1,y2,y3,y4
y = layers.concatenate([y1, y2, y3, y4], axis=bn_axis)
y = Conv2D((filters3 * 4), (1, 1), name=conv_name_base + '2')(y)
y = BatchNormalization(axis=bn_axis, name=bn_name_base + '2')(y)
t = Lambda(lambda y: tf.split(y, 4, axis=3))(y)
y = layers.average([t[0], t[1], t[2], t[3]])
y = layers.multiply([x, y])
x = layers.add([x, y])
x = Activation('relu')(x)
return x
def make_generator_model(input_tensor=None,
input_shape=(noise_dim,)):
"""
Returns:
tf.keras.Model
"""
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = layers.Dense(7 * 7 * 256,
activation=tf.nn.relu,
use_bias=False,
name='fc1')(img_input)
x = layers.Reshape(target_shape=(7, 7, 128), name='reshape1')(x)
x = layers.BatchNormalization(momentum=0.8, name='bn1')(x)
x = layers.UpSampling2D(name='upsampling1')(x)
x = layers.Conv2D(128, (3, 3),
activation=tf.nn.relu,
padding="same",
use_bias=False,
name='conv1')(x)
x = layers.BatchNormalization(momentum=0.8, name='bn2')(x)
x = layers.UpSampling2D(name='upsampling2')(x)
x = layers.Conv2D(64, (3, 3),
activation=tf.nn.relu,
padding="same",
use_bias=False,
name='conv2')(x)
x = layers.BatchNormalization(momentum=0.8, name='bn3')(x)
x = layers.Conv2D(1, (3, 3),
activation=tf.nn.tanh,
use_bias=False,
name='conv3')(x)
noise = layers.Input(shape=(noise_dim,))
label = layers.Input(shape=(1,), dtype='int32')
label_embedding = layers.Flatten()(layers.Embedding(num_classes, 100)(label))
x = layers.multiply([noise, label_embedding])(x)
return models.Model([noise, label], x)
def get_sketch_matrix(self, h, s,v,d,n=2048):
batch_size=v.get_shape().as_list()[0]
if batch_size==None:
batch_size=1
print("Batch size None")
#y=[0.0 for i in range(d)]
#y=[[0 for i in range(d)] for j in range(batch_size)]
y=np.zeros(shape=(batch_size,d))
for i in range(batch_size):
for j in range(n):
if batch_size==1:
y[i,h[j]]=0.0
else:
y[i,h[j]]=tf.add(y[i,h[j]],multiply(s[j],tf.keras.backend.get_value(v[i,j])))
return y
def swish(x, name="swish"):
"""
Swish activation function from 'Searching for Activation Functions,' https://arxiv.org/abs/1710.05941.
Parameters:
----------
x : keras.backend tensor/variable/symbol
Input tensor/variable/symbol.
name : str, default 'swish'
Block name.
Returns
-------
keras.backend tensor/variable/symbol
Resulted tensor/variable/symbol.
"""
w = nn.Activation("sigmoid", name=name + "/sigmoid")(x)
x = nn.multiply([x, w], name=name + "/mul")
return x
def f(input):
filters = input.shape[CHANNEL_AXIS]
se_shape = (1, 1, filters)
se = GlobalAveragePooling2D()(input)
se = Reshape(se_shape)(se)
se = Dense(filters // ratio,
activation='relu',
kernel_initializer='he_normal',
use_bias=False)(se)
se = Dense(filters,
activation='sigmoid',
kernel_initializer='he_normal',
use_bias=False)(se)
x = multiply([input, se])
return x
def interest_evolution(concat_behavior, deep_input_item, user_behavior_length, gru_type="GRU", use_neg=False,
neg_concat_behavior=None, att_hidden_size=(64, 16), att_activation='sigmoid',
att_weight_normalization=False, ):
if gru_type not in ["GRU", "AIGRU", "AGRU", "AUGRU"]:
raise ValueError("gru_type error ")
aux_loss_1 = None
embedding_size = None
rnn_outputs = DynamicGRU(embedding_size, return_sequence=True,
name="gru1")([concat_behavior, user_behavior_length])
print(concat_behavior)
print(neg_concat_behavior)
if gru_type == "AUGRU" and use_neg:
aux_loss_1 = auxiliary_loss(rnn_outputs[:, :-1, :], concat_behavior[:, 1:, :],
neg_concat_behavior[:, 1:, :],
tf.subtract(user_behavior_length, 1), stag="gru") # [:, 1:]
if gru_type == "GRU":
rnn_outputs2 = DynamicGRU(embedding_size, return_sequence=True,
name="gru2")([rnn_outputs, user_behavior_length])
# attention_score = AttentionSequencePoolingLayer(hidden_size=att_hidden_size, activation=att_activation, weight_normalization=att_weight_normalization, return_score=True)([
# deep_input_item, rnn_outputs2, user_behavior_length])
# outputs = Lambda(lambda x: tf.matmul(x[0], x[1]))(
# [attention_score, rnn_outputs2])
# hist = outputs
hist = AttentionSequencePoolingLayer(att_hidden_units=att_hidden_size, att_activation=att_activation,
weight_normalization=att_weight_normalization, return_score=False)([
deep_input_item, rnn_outputs2, user_behavior_length])
else: # AIGRU AGRU AUGRU
scores = AttentionSequencePoolingLayer(att_hidden_units=att_hidden_size, att_activation=att_activation,
weight_normalization=att_weight_normalization, return_score=True)([
deep_input_item, rnn_outputs, user_behavior_length])
if gru_type == "AIGRU":
hist = multiply([rnn_outputs, Permute([2, 1])(scores)])
final_state2 = DynamicGRU(embedding_size, gru_type="GRU", return_sequence=False, name='gru2')(
[hist, user_behavior_length])
else: # AGRU AUGRU
final_state2 = DynamicGRU(embedding_size, gru_type=gru_type, return_sequence=False,
name='gru2')([rnn_outputs, user_behavior_length, Permute([2, 1])(scores)])
hist = final_state2
return hist, aux_loss_1
def spatial_squeeze_excite_block(input):
''' Create a spatial squeeze-excite block
Args:
input: input tensor
Returns: a keras tensor
References
- [Concurrent Spatial and Channel Squeeze & Excitation in Fully Convolutional Networks](https://arxiv.org/abs/1803.02579)
'''
se = Conv2D(1, (1, 1),
activation='sigmoid',
use_bias=False,
kernel_initializer='he_normal')(input)
x = multiply([input, se])
return x
def call(self, inputs, **kwargs):
g, x = inputs#128:64
theta_x = self.theta_x(x)#64*64*12
phi_g = self.phi_g(g)#64*64*12
upsample_g = self.deconv_g(phi_g)
add_xg = layers.add([upsample_g, theta_x])
act_xg = self.act_xg(add_xg)
psi = self.conv(act_xg)
sigmoid_psi = self.act_sigmoid(psi)
upsample_psi = self.upsample(sigmoid_psi)
shape_x = x.get_shape().as_list()[-1]
repeat_psi = tf.tile(upsample_psi, multiples=tf.constant([1,1,1,shape_x]))
y = layers.multiply([repeat_psi, x])
res = self.conv_end(y)
res_bn = self.bn(res)
return res_bn
def channel_attention_block(input, filters, kernel_size, padding, reduction_ratio, weight_decay=5e-4):
x = Conv3D(filters=filters, kernel_size=kernel_size, padding=padding, use_bias=False,
kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(input)
x = BatchNormalization(axis=-1)(x)
x = Activation('relu')(x)
squeeze = GlobalAveragePooling3D()(x)
excitation = Reshape((1, 1, 1, filters))(squeeze)
excitation = Conv3D(filters=filters // reduction_ratio, kernel_size=1,
use_bias=False, kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(excitation)
excitation = Activation('relu')(excitation)
excitation = Conv3D(filters=filters, kernel_size=1,
use_bias=False, kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(excitation)
excitation = Activation('sigmoid')(excitation)
scale = multiply([x, excitation])
x_add = add([scale, input])
return x_add
def senet_se_block(input_tensor, stage, block, compress_rate=16, bias=False):
conv1_down_name = 'conv' + str(stage) + "_" + str(
block) + "_1x1_down"
conv1_up_name = 'conv' + str(stage) + "_" + str(
block) + "_1x1_up"
num_channels = int(input_tensor.shape[-1])
bottle_neck = int(num_channels // compress_rate)
se = GlobalAveragePooling2D()(input_tensor)
se = Reshape((1, 1, num_channels))(se)
se = Conv2D(bottle_neck, (1, 1), use_bias=bias,
name=conv1_down_name)(se)
se = Activation('relu')(se)
se = Conv2D(num_channels, (1, 1), use_bias=bias,
name=conv1_up_name)(se)
se = Activation('sigmoid')(se)
x = input_tensor
x = multiply([x, se])
return x
def se_block(x, channels, reduction=16, activation="relu", name="se_block"):
"""
Squeeze-and-Excitation block from 'Squeeze-and-Excitation Networks,' https://arxiv.org/abs/1709.01507.
Parameters:
----------
x : keras.backend tensor/variable/symbol
Input tensor/variable/symbol.
channels : int
Number of channels.
reduction : int, default 16
Squeeze reduction value.
activation : function or str, default 'relu'
Activation function or name of activation function.
name : str, default 'se_block'
Block name.
Returns
-------
keras.backend tensor/variable/symbol
Resulted tensor/variable/symbol.
"""
assert (len(x.shape) == 4)
mid_cannels = channels // reduction
pool_size = x.shape[2:4] if is_channels_first() else x.shape[1:3]
w = nn.AvgPool2D(pool_size=pool_size, name=name + "/pool")(x)
w = conv1x1(x=w,
in_channels=channels,
out_channels=mid_cannels,
use_bias=True,
name=name + "/conv1")
w = get_activation_layer(x=w, activation=activation, name=name + "/activ")
w = conv1x1(x=w,
in_channels=mid_cannels,
out_channels=channels,
use_bias=True,
name=name + "/conv2")
w = nn.Activation("sigmoid", name=name + "/sigmoid")(w)
x = nn.multiply([x, w], name=name + "/mul")
return x
def define_generator(latent_dim=50, nclasses=10):
label = layers.Input(shape=(1, ))
li = layers.Embedding(nclasses, latent_dim)(label)
li = layers.Flatten()(li)
noise = layers.Input(shape=(latent_dim, ))
input = layers.multiply([li, noise])
x = layers.Dense(7*7*128, activation="relu")(input)
x = layers.Reshape((7, 7, 128))(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.UpSampling2D()(x)
x = layers.Conv2D(filters=128, kernel_size=3, padding="same", activation="relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.UpSampling2D()(x)
x = layers.Conv2D(filters=64, kernel_size=3, padding="same", activation="relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
out = layers.Conv2D(filters=1, kernel_size=3, padding="same", activation="tanh")(x)
model = tf.keras.Model([noise, label], out)
return model
def squeeze_excite_block(input):
''' Create a squeeze-excite block
Args:
input: input tensor
filters: number of output filters
k: width factor
Returns: a keras tensor
'''
# channel_axis = -1 for TF
filters = input._shape_val[-1]
se = GlobalAveragePooling1D()(input)
se = Reshape((1, filters))(se)
se = Dense(filters // 16,
activation='relu',
kernel_initializer='he_normal',
use_bias=False)(se)
se = Dense(filters,
activation='sigmoid',
kernel_initializer='he_normal',
use_bias=False)(se)
se = multiply([input, se])
return se
def build_discriminator(self, config_discr):
a = float(config_discr['leaky_relu_alpha'])
d = float(config_discr['dropout'])
model = Sequential()
model.add(
Dense(int(config_discr['dense1']),
input_dim=np.prod(self.img_shape)))
model.add(LeakyReLU(alpha=a))
model.add(Dense(int(config_discr['dense2'])))
model.add(LeakyReLU(alpha=a))
model.add(Dropout(d))
model.add(Dense(int(config_discr['dense3'])))
model.add(LeakyReLU(alpha=a))
model.add(Dropout(d))
model.add(Dense(1, activation=config_discr['activation']))
img = Input(shape=self.img_shape)
label = Input(shape=(1, ), dtype='int32')
label_embedding = Flatten()(Embedding(
self.embeddings_vocab,
self.embeddings_dim,
weights=[self.word_embeddings.vectors],
trainable=False)(label))
label_expansion = Dense(self.img_rows * self.img_cols *
self.channels)(label_embedding)
flat_img = Flatten()(img)
model_input = multiply([flat_img, label_expansion])
validity = model(model_input)
return Model([img, label], validity)
def NFFM(
linear_feature_columns,
dnn_feature_columns,
embedding_size=4,
dnn_hidden_units=(128, 128),
l2_reg_embedding=1e-5,
l2_reg_linear=1e-5,
l2_reg_dnn=0,
dnn_dropout=0,
init_std=0.0001,
seed=1024,
use_bn=True,
reduce_sum=False,
task='binary',
):
"""Instantiates the Operation-aware Neural Networks architecture.
:param linear_feature_columns: An iterable containing all the features used by linear part of the model.
:param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
:param embedding_size: positive integer,sparse feature embedding_size
:param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param l2_reg_linear: float. L2 regularizer strength applied to linear part.
:param l2_reg_dnn: float . L2 regularizer strength applied to DNN
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
:param use_bn: bool,whether use bn after ffm out or not
:param reduce_sum: bool,whether apply reduce_sum on cross vector
:param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss
:return: A Keras model instance.
"""
features = build_input_features(linear_feature_columns +
dnn_feature_columns)
inputs_list = list(features.values())
linear_logit = get_linear_logit(features,
linear_feature_columns,
l2_reg=l2_reg_linear,
init_std=init_std,
seed=seed,
prefix='linear')
sparse_feature_columns = list(
filter(lambda x: isinstance(x, SparseFeat),
dnn_feature_columns)) if dnn_feature_columns else []
varlen_sparse_feature_columns = list(
filter(lambda x: isinstance(x, VarLenSparseFeat),
dnn_feature_columns)) if dnn_feature_columns else []
sparse_embedding = {
fc_j.embedding_name: {
fc_i.embedding_name:
Embedding(fc_j.dimension,
embedding_size,
embeddings_initializer=RandomNormal(mean=0.0,
stddev=0.0001,
seed=seed),
embeddings_regularizer=l2(l2_reg_embedding),
mask_zero=isinstance(fc_j, VarLenSparseFeat),
name='sparse_emb_' + str(fc_j.embedding_name) + '_' +
fc_i.embedding_name)
for fc_i in sparse_feature_columns + varlen_sparse_feature_columns
}
for fc_j in sparse_feature_columns + varlen_sparse_feature_columns
}
dense_value_list = get_dense_input(features, dnn_feature_columns)
embed_list = []
for fc_i, fc_j in itertools.combinations(
sparse_feature_columns + varlen_sparse_feature_columns, 2):
i_input = features[fc_i.name]
if fc_i.use_hash:
i_input = Hash(fc_i.dimension)(i_input)
j_input = features[fc_j.name]
if fc_j.use_hash:
j_input = Hash(fc_j.dimension)(j_input)
fc_i_embedding = feature_embedding(fc_i, fc_j, sparse_embedding,
i_input)
fc_j_embedding = feature_embedding(fc_j, fc_i, sparse_embedding,
j_input)
element_wise_prod = multiply([fc_i_embedding, fc_j_embedding])
if reduce_sum:
element_wise_prod = Lambda(lambda element_wise_prod: K.sum(
element_wise_prod, axis=-1))(element_wise_prod)
embed_list.append(element_wise_prod)
ffm_out = tf.keras.layers.Flatten()(concat_fun(embed_list, axis=1))
if use_bn:
ffm_out = tf.keras.layers.BatchNormalization()(ffm_out)
dnn_input = combined_dnn_input([ffm_out], dense_value_list)
dnn_out = DNN(dnn_hidden_units,
l2_reg=l2_reg_dnn,
dropout_rate=dnn_dropout)(dnn_input)
dnn_logit = Dense(1, use_bias=False)(dnn_out)
if len(linear_feature_columns) > 0 and len(dnn_feature_columns) > 0:
final_logit = add([dnn_logit, linear_logit])
elif len(linear_feature_columns) > 0:
final_logit = linear_logit
elif len(dnn_feature_columns) > 0:
final_logit = dnn_logit
else:
raise NotImplementedError
output = PredictionLayer(task)(final_logit)
model = Model(inputs=inputs_list, outputs=output)
return model
def block(inputs,
activation='swish',
drop_rate=0.,
name='',
filters_in=32,
filters_out=16,
kernel_size=3,
strides=1,
expand_ratio=1,
se_ratio=0.,
id_skip=True):
"""An inverted residual block.
Arguments:
inputs: input tensor.
activation: activation function.
drop_rate: float between 0 and 1, fraction of the input units to drop.
name: string, block label.
filters_in: integer, the number of input filters.
filters_out: integer, the number of output filters.
kernel_size: integer, the dimension of the convolution window.
strides: integer, the stride of the convolution.
expand_ratio: integer, scaling coefficient for the input filters.
se_ratio: float between 0 and 1, fraction to squeeze the input filters.
id_skip: boolean.
Returns:
output tensor for the block.
"""
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
# Expansion phase
filters = filters_in * expand_ratio
if expand_ratio != 1:
x = layers.Conv2D(
filters,
1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'expand_conv')(
inputs)
x = layers.BatchNormalization(axis=bn_axis, name=name + 'expand_bn')(x)
x = layers.Activation(activation, name=name + 'expand_activation')(x)
else:
x = inputs
# Depthwise Convolution
if strides == 2:
x = layers.ZeroPadding2D(
padding=imagenet_utils.correct_pad(x, kernel_size),
name=name + 'dwconv_pad')(x)
conv_pad = 'valid'
else:
conv_pad = 'same'
x = layers.DepthwiseConv2D(
kernel_size,
strides=strides,
padding=conv_pad,
use_bias=False,
depthwise_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'dwconv')(x)
x = layers.BatchNormalization(axis=bn_axis, name=name + 'bn')(x)
x = layers.Activation(activation, name=name + 'activation')(x)
# Squeeze and Excitation phase
if 0 < se_ratio <= 1:
filters_se = max(1, int(filters_in * se_ratio))
se = layers.GlobalAveragePooling2D(name=name + 'se_squeeze')(x)
se = layers.Reshape((1, 1, filters), name=name + 'se_reshape')(se)
se = layers.Conv2D(
filters_se,
1,
padding='same',
activation=activation,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'se_reduce')(
se)
se = layers.Conv2D(
filters,
1,
padding='same',
activation='sigmoid',
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'se_expand')(se)
x = layers.multiply([x, se], name=name + 'se_excite')
# Output phase
x = layers.Conv2D(
filters_out,
1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'project_conv')(x)
x = layers.BatchNormalization(axis=bn_axis, name=name + 'project_bn')(x)
if id_skip and strides == 1 and filters_in == filters_out:
if drop_rate > 0:
x = layers.Dropout(
drop_rate, noise_shape=(None, 1, 1, 1), name=name + 'drop')(x)
x = layers.add([x, inputs], name=name + 'add')
return x
