def SE_residual_block(x, num_features, size, dilation_rate, name):
# 残差
original_x = x
num_features1 = num_features // 4
merged_list = []
rate = dilation_rate
x = Conv1D(num_features1,
size,
dilation_rate=rate,
padding='same',
kernel_regularizer=l2(WEIGHT_DECAY),
name=f'conv_{name}_{rate}')(x)
x = BatchNormalization()(x)
x_tanh = Activation('tanh')(x)
x_sigmoid = Activation('sigmoid')(x)
merged = Multiply()([x_tanh, x_sigmoid])
x = Conv1D(num_features, 1, padding='same')(merged)
x = BatchNormalization()(x)
x = Multiply(name=f'scale_{name}')(
[x, squeeze_exciation(x, AMPLIFYING_RATIO, name)])
x = Add()([original_x, x])
x = Activation('relu')(x)
return x
def init_model(self):
input_question = Input(shape = (None, self.num_features))
input_ans = Input(shape = (None, self.num_features))
with tf.name_scope("question_encoder"):
question_encoder = Bidirectional(LSTM(self.question_encoder_shape))(input_question)
question_attention = Dense(2*self.question_encoder_shape, activation = 'softmax')(question_encoder)
question_attention = Multiply()([question_encoder, question_attention])
with tf.name_scope("answer_encoder"):
conv1d_2 = Conv1D(128, 5, activation = "tanh")(input_ans)
max_pooling_2 = MaxPooling1D()(conv1d_2)
conv1d_3 = Conv1D(128, 5, activation = "tanh")(max_pooling_2)
max_pooling_3 = MaxPooling1D()(conv1d_3)
answer_encoder_1 = Bidirectional(LSTM(self.answer_encoder_shape))(max_pooling_3)
answer_attention = Dense(2*self.answer_encoder_shape, activation = 'softmax')(answer_encoder_1)
answer_attention = Multiply()([answer_encoder_1, answer_attention])
merge = concatenate([question_attention, answer_attention], axis = 1)
#merge = concatenate([question_encoder, answer_encoder_1])
dense_1 = Dense(128, activation = "relu")(merge)
dropout_1 = Dropout(0.5)(dense_1)
#dense_2 = Dense(256, activation = "relu")(dropout_1)
#dropout_2 = Dropout(0.25)(dense_2)
dense_3 = Dense(1, activation ="sigmoid")(dropout_1)
model = Model(inputs = [input_question, input_ans], outputs = dense_3)
metrics = ["accuracy"]
model.compile(optimizer = Adam(self.learning_rate), loss = "binary_crossentropy", metrics = metrics)
model.summary()
return model
def se_res_block(X , filters , units , type_resnet, halving=1):
#halving=1 means that the first convolution layer in the first block of each macro-block halves the size with a stride 2x2
if type_resnet==34:
N=1
if type_resnet==50:
N=2
if halving==0:
stride=(1,1)
input_reshaped=Conv2D(N*filters,(1,1), activation=None, strides=(1,1), padding='valid')(X)
else:
stride=(2,2)
input_reshaped=Conv2D(N*filters,(1,1),activation=None, strides=(2,2), padding='valid')(X)
res=residual(X, filters, type_resnet, stride=stride)
output_SE=SE(res, filters, N)
scale= Multiply()([res, output_SE])
X=Add()([input_reshaped, scale])
X=Activation('relu')(X)
for i in range(1, units):
input_reshaped=X
res=residual(X,filters, type_resnet, stride=(1,1))
output_SE=SE(res, filters, N)
scale= Multiply()([res, output_SE])
X=Add()([input_reshaped, scale])
X=Activation('relu')(X)
return X
def build_model(n_class, n_context):
#--------------------------------------------
# モデルをビルド
# params:
# - n_class:int -> 分類するクラス数
# - n_context:int -> コンテキスト数
# return:
# - tensorflow.python.keras.engine.training.Model
#--------------------------------------------
# 入力層
main_input = Input(shape=(32, ))
sub_input = Input(shape=(20, ))
# サブネットワーク
sub_x = Dense(64, activation='relu')(sub_input)
sub_x = Dropout(0.5)(sub_x)
sub_y1 = Dense(1, activation='relu')(sub_x)
sub_y2 = Dense(1, activation='relu')(sub_x)
# メインネットワーク
main_x1 = Dense(128, activation='relu')(main_input)
main_x2_1 = Dense(128, activation='relu')(main_x1)
main_x2_1 = Multiply()([main_x2_1, sub_y1])
main_x2_2 = Dense(128, activation='relu')(main_x1)
main_x2_2 = Multiply()([main_x2_2, sub_y2])
main_x2 = Add()([main_x2_1, main_x2_2])
main_x3 = Dense(128, activation='relu')(main_x2)
main_x3 = Dropout(0.5)(main_x3)
main_x4 = Dense(64, activation='relu')(main_x3)
main_x4 = Dropout(0.5)(main_x4)
main_y = Dense(n_class, activation='softmax')(main_x4)
model = Model(inputs=[main_input, sub_input], outputs=main_y)
return model
def build_model():
n_pin_vec = 128
n_sku_vec = 128
pin_vec = Input(shape=(n_pin_vec, ), dtype = 'float32')
sku_vec = Input(shape=(n_sku_vec, ), dtype = 'float32')
# ctr_part
ctr_pin_part = Dense(64, activation='relu')(pin_vec)
ctr_sku_part = Dense(64, activation='relu')(sku_vec)
ctr_prod = Multiply()([ctr_pin_part, ctr_sku_part])
ctr_prob = Dense(1, activation='sigmoid', name='ctr_prob')(ctr_prod)
# ctcvr_part
cvr_pin_part = Dense(64, activation='relu')(pin_vec)
cvr_sku_part = Dense(64, activation='relu')(sku_vec)
cvr_prod = Multiply()([cvr_pin_part, cvr_sku_part])
cvr_prob = Dense(1, activation='sigmoid', name='cvr_prob')(cvr_prod)
#ctcvr_prob = ctr_prob * cvr_prob
ctcvr_prob = Multiply(name = 'ctcvr_prob')([ctr_prob, cvr_prob])
model = Model(inputs = [pin_vec, sku_vec], outputs = [ctr_prob, ctcvr_prob])
model.compile(optimizer = 'adam', \
loss = {'ctr_prob' : 'binary_crossentropy', 'ctcvr_prob' : 'binary_crossentropy'}, \
#metrics = {'ctr_prob' : 'accuracy', 'ctcvr_prob' : 'accuracy'})
metrics = ['accuracy'])
return model
def baseline_model(seq_dim=3):
input_1 = Input(shape=(None, 3))
input_2 = Input(shape=(None, seq_dim))
base_model = encoder(seq_dim=3)
x1 = base_model(input_1)
x2 = base_model(input_2)
x1 = Concatenate(axis=-1)([GlobalMaxPool1D()(x1), GlobalAvgPool1D()(x1)])
x2 = Concatenate(axis=-1)([GlobalMaxPool1D()(x2), GlobalAvgPool1D()(x2)])
x3 = Subtract()([x1, x2])
x3 = Multiply()([x3, x3])
x = Multiply()([x1, x2])
x = Concatenate(axis=-1)([x, x3])
x = Dropout(0.1)(x)
x = Dense(100, activation="relu")(x)
x = Dropout(0.1)(x)
out = Dense(1, activation="sigmoid")(x)
model = Model([input_1, input_2], out)
model.compile(loss="binary_crossentropy", metrics=[acc], optimizer=Adam(0.0001))
model.summary()
return model
def conv_lstm(input_tensor, input_cell_state, name):
if type(name) is str:
name = name + '/'
sigmoid_input = conv_activat('sigmoid', 'i', name)(input_tensor)
sigmoid_forget = conv_activat('sigmoid', 'f', name)(input_tensor)
tanh_cell_state = conv_activat('tanh', 'c', name)(input_tensor)
sigmoid_output = conv_activat('sigmoid', 'o', name)(input_tensor)
cell_state = Add(name=name + 'add_c')([
Multiply(name=name + 'mul_f_c')([sigmoid_forget, input_cell_state]),
Multiply(name=name + 'mul_i_c')([sigmoid_input, tanh_cell_state])
])
lstm_feats = Multiply(name=name + 'mul_lf')([
sigmoid_output,
Activation('tanh', name=name + 'tanh_c')(cell_state)
])
attention_map = conv_activat('sigmoid', 'attention_map', name, out_channel=1)(lstm_feats)
ret = {
'attention_map': attention_map,
'cell_state': cell_state,
'lstm_feats': lstm_feats
}
return ret
def build_base_model(input_size):
in_1 = Input(shape=(input_size, ), name="input_1")
in_2 = Input(shape=(input_size, ), name="input_2")
norm_1 = Lambda(lambda tensor: tf.norm(tensor, axis=1, keepdims=True),
name="norm_input_1")(in_1)
norm_2 = Lambda(lambda tensor: tf.norm(tensor, axis=1, keepdims=True),
name="norm_input_2")(in_2)
norm_mul = Multiply(name="multiply_norms")([norm_1, norm_2])
model = Multiply(name="pointwise_multiply")([in_1, in_2])
model = Lambda(lambda tensor: tf.reduce_sum(tensor, axis=1, keepdims=True),
name="sum")(model)
model = Lambda(lambda tensors: tf.divide(tensors[0], tensors[1]),
name="divide")([model, norm_mul])
model = ValueMinusInput(1, name="one_minus_input")(model)
model = LessThan(0.4)(model)
model_out = Lambda(lambda tensor: tf.cast(tensor, tf.float32),
name="cast")(model)
model = Model([in_1, in_2], model_out)
model.compile(loss=MeanSquaredError(),
optimizer=SGD(),
metrics=[
BinaryAccuracy(),
Precision(),
Recall(),
TrueNegatives(),
FalsePositives(),
FalseNegatives(),
TruePositives()
])
return model
def highway_layer(value, gate_bias=-3):
# https://towardsdatascience.com/review-highway-networks-gating-function-to-highway-image-classification-5a33833797b5
nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel()
dim = K.int_shape(value)[-1]
# gate_bias_initializer = tensorflow.keras.initializers.Constant(gate_bias)
# gate = Dense(units=dim, bias_initializer=gate_bias_initializer)(value)
# gate = Activation("sigmoid")(gate)
# TODO (just for yellow color...) NOTE: to keep dimensions matched, convolution gate instead of regular sigmoid
# gate (T in paper)
gate = Conv2D(size_list[i_hidden + config.CNN_ResBlock_conv_per_block -
1],
kernel_size=filt_list[-1],
padding='same',
activation='sigmoid',
bias_initializer=tensorflow.keras.initializers.Constant(
gate_bias))(value)
# negated (C in paper)
negated_gate = Lambda(lambda x: 1.0 - x,
output_shape=(size_list[-1], ))(gate)
# use ResBlock as the Transformation
transformed = ResBlock(x=value)
transformed_gated = Multiply()([gate, transformed])
# UpSample value if needed
if value.shape.as_list()[-1] != negated_gate.shape.as_list()[-1]:
r = negated_gate.shape.as_list()[-1] / value.shape.as_list()[-1]
assert not (bool(r % 1))
value = tf.keras.layers.UpSampling3D(size=(1, 1, int(r)))(value)
identity_gated = Multiply()([negated_gate, value])
value = Add()([transformed_gated, identity_gated])
return value
def weighted_sum(out_u, x_u, out_i, x_i, dropout_keep_prob, random_seed):
"""
Return the weighted sum of the reviews
Args:
-------
out_u: weights of user's reviews
x_u: semantics of user's reviews
out_i: weights of item's reviews
x_i: semantics of item's reviews
other_args: model parameters
Outputs:
-------
: weighted sums of user's reviews and item's reviews
"""
feas_u = tf.reduce_sum(Multiply()([out_u, x_u]), axis=1)
feas_i = tf.reduce_sum(Multiply()([out_i, x_i]), axis=1)
# Dropout layers here to reduce overfitting
feas_u = Dropout(1 - dropout_keep_prob, seed=random_seed)(feas_u)
feas_i = Dropout(1 - dropout_keep_prob, seed=random_seed)(feas_i)
return feas_u, feas_i
def build(self):
y = Input((self.dim, ))
h = Input((1, ))
alpha = Input((
self.target.alpha_num,
self.target.alpha_dim,
self.target.alpha_dim,
))
f = lambda u: self.target.f(u[0], u[1]) # Wrap f as a lambda
fy = Lambda(f)([y, alpha])
self.ks = [Multiply()([fy, h])]
for j in range(self.order - 1):
# modify the model similar to the general RK.
current_ks = []
for i in range(len(self.ks)):
current_ks.append(self.ks[i])
temp = MultiDense(name=f'k_{j}')(
current_ks
) # this is a fully connected layer, using all the previous k_i calculated before.
# temp = MultiDense(name=f'k_{j}')([self.ks[-1]]) # only use k_i from last step
temp = Add()([y, temp])
temp = Lambda(f)([temp, alpha])
k_next = Multiply()([temp, h])
self.ks.append(k_next)
ks_dense = SoftmaxDense(name='final')(self.ks)
y_next = Add()([y, ks_dense])
outputs = Concatenate()([y_next, h])
self.model = Model(inputs=[y, h, alpha], outputs=outputs)
def cbam_block(x, ratio=8):
# channel_attention
filters = x.shape[-1]
avg_pool_ch = GlobalAveragePooling2D()(x)
avg_pool_ch = Reshape((1, 1, filters))(avg_pool_ch)
avg_pool_ch = Dense(filters // ratio)(avg_pool_ch)
avg_pool_ch = Activation('relu')(avg_pool_ch)
avg_pool_ch = Dense(filters)(avg_pool_ch)
max_pool_ch = GlobalMaxPooling2D()(x)
max_pool_ch = Reshape((1, 1, filters))(max_pool_ch)
max_pool_ch = Dense(filters // ratio)(max_pool_ch)
max_pool_ch = Activation('relu')(max_pool_ch)
max_pool_ch = Dense(filters)(max_pool_ch)
cbam_ch = Add()([avg_pool_ch, max_pool_ch])
cbam_ch = Activation('sigmoid')(cbam_ch)
cbam_ch = Multiply()([x, cbam_ch])
# spatial_attention
kernel_size = 7
avg_pool_s = Lambda(lambda x: K.mean(x, axis=3, keepdims=True))(cbam_ch)
max_pool_s = Lambda(lambda x: K.max(x, axis=3, keepdims=True))(cbam_ch)
concat = Concatenate(axis=3)([avg_pool_s, max_pool_s])
cbam_s = Conv2D(filters=1, kernel_size=kernel_size, strides=1, padding='same')(concat)
cbam_s = Activation('sigmoid')(cbam_s)
cbam = Multiply()([cbam_ch, cbam_s])
return cbam
def baseline_model():
input_1 = Input(shape=(None, None, 3))
input_2 = Input(shape=(None, None, 3))
base_model = MobileNetV2(weights="imagenet", include_top=False)
x1 = base_model(input_1)
x2 = base_model(input_2)
x1 = Concatenate(axis=-1)([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)])
x2 = Concatenate(axis=-1)([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)])
x3 = Subtract()([x1, x2])
x3 = Multiply()([x3, x3])
x = Multiply()([x1, x2])
x = Concatenate(axis=-1)([x, x3])
x = Dense(100, activation="relu")(x)
x = Dropout(0.01)(x)
out = Dense(1, activation="sigmoid")(x)
model = Model([input_1, input_2], out)
model.compile(loss="binary_crossentropy", metrics=[acc], optimizer=Adam(0.00001))
model.summary()
return model
def build_model(self):
outputhelper = []
for j in range(self.N):
strategy = self.price
strategyeval = self.tradeeval
for k in range(self.d):
strategy = self.layers[k + (j) * self.d](strategy) # strategy at j is the alpha at j
strategyeval = self.layers[k + (j) * self.d](strategyeval)
incr = Input(shape=(self.m,))
logprice = Lambda(lambda x: K.log(x))(self.price)
logprice = Add()([logprice, incr])
pricenew = Lambda(lambda x: K.exp(x))(logprice)
self.price = pricenew
logwealth = Lambda(lambda x: K.log(x))(self.wealth)
logwealth = Lambda(lambda x: x + self.r * self.T / self.N)(logwealth)
helper1 = Multiply()([strategy, incr])
# helper1 = Lambda()(lambda x : K.sum(x,axis=1))([helper1])
logwealth = Add()([logwealth, helper1])
helper2 = Multiply()([strategy, strategy])
# helper2 = Lambda()(lambda x : K.sum(x,axis=1))([helper1])
helper3 = Lambda(lambda x: x * self.sigma ** 2 / 2 * self.T / self.N)(helper2)
logwealth = Subtract()([logwealth, helper3])
helper4 = Lambda(lambda x: x * self.r * self.T / self.N)(strategy)
logwealth = Subtract()([logwealth, helper4])
wealthnew = Lambda(lambda x: K.exp(x))(logwealth) # creating the wealth at time j+1
self.inputs = self.inputs + [incr]
outputhelper = outputhelper + [strategyeval] # here we collect the strategies
self.wealth = wealthnew
self.outputs = self.wealth
randomendowment = Lambda(lambda x: -0.0 * (K.abs(x - 1.0) + x - 1.0))(self.price)
self.outputs = Add()([self.wealth, randomendowment])
self.outputs = [self.outputs] + outputhelper
self.outputs = Concatenate()(self.outputs)
# Now return the model
return Model(inputs=self.inputs, outputs=self.outputs)
def CFF(input_list, input_size, filters, i):
out_shape = input_size / pow(2, i)
y = tf.zeros_like(input_list[i - 1])
for j, x in enumerate(input_list):
if j < i - 1:
down_factor = int((input_size / pow(2, j + 1)) / out_shape)
x = AveragePooling3D((down_factor, down_factor, down_factor))(x)
x = Conv3D(filters, (1, 1, 1), padding='same')(x)
sigm = Activation('sigmoid')(x)
x = Multiply()([x, sigm])
y = Add()([y, x])
if j > i - 1:
up_factor = int(out_shape / (input_size / pow(2, j + 1)))
x = Conv3D(filters, (1, 1, 1), padding='same')(x)
x = UpSampling3D((up_factor, up_factor, up_factor))(x)
sigm = Activation('sigmoid')(x)
x = Multiply()([x, sigm])
y = Add()([y, x])
x_i = input_list[i - 1]
x_i_sigm = Activation('sigmoid')(x_i)
x_i_sigm = -1 * x_i_sigm + 1
out = Multiply()([x_i_sigm, y])
out = Add()([out, x_i])
return out
def FTA_Module(x, shape, kt, kf):
x = BatchNormalization()(x)
## Residual
x_r = Conv2D(shape[2], (1, 1), padding='same', activation='relu')(x)
## Time Attention
# Attn Map (1, T, C), FC
a_t = Lambda(K.mean, arguments={'axis': -3})(x)
a_t = Conv1D(shape[2], kt, padding='same', activation='selu')(a_t)
a_t = Conv1D(shape[2], kt, padding='same', activation='selu')(a_t) #2
a_t = Softmax(axis=-2)(a_t)
a_t = Reshape((1, shape[1], shape[2]))(a_t)
# Reweight
x_t = Conv2D(shape[2], (3, 3), padding='same', activation='selu')(x)
x_t = Conv2D(shape[2], (5, 5), padding='same', activation='selu')(x_t)
x_t = Multiply()([x_t, a_t])
###TODO:参考Strip pooling，将两个map相乘后再使用
# Frequency Attention
# Attn Map (F, 1, C), Conv1D
a_f = Lambda(K.mean, arguments={'axis': -2})(x)
a_f = Conv1D(shape[2], kf, padding='same', activation='selu')(a_f)
a_f = Conv1D(shape[2], kf, padding='same', activation='selu')(a_f)
a_f = Softmax(axis=-2)(a_f)
a_f = Reshape((shape[0], 1, shape[2]))(a_f)
# Reweight
x_f = Conv2D(shape[2], (3, 3), padding='same', activation='selu')(x)
x_f = Conv2D(shape[2], (5, 5), padding='same', activation='selu')(x_f)
x_f = Multiply()([x_f, a_f])
x = Add()([x_r, x_t, x_f])
return x
def _topology_to_model(self):
"""Extend `self.topology` to produce an action-value output (action as input).
Returns
-------
keras.Model
a compiled Q model.
"""
if self.topology.output.shape.ndims > 2:
base_out = Flatten()(self.topology.output)
else:
base_out = self.topology.output
if self.action_type == 'discrete':
action_in = Input((self.action_space.n,))
out = Dense(self.action_space.n)(base_out)
out = Multiply()([out, action_in])
model = Model([self.topology.input, action_in], out)
elif self.action_type == 'multidiscrete':
action_ins = [Input((n,)) for n in self.action_space.nvec]
outs = [Dense(n)(base_out) for n in self.action_space.nvec]
outs = [Multiply()([out, action_in]) for out, action_in in zip(outs, action_ins)]
model = Model([self.topology.input] + action_ins, outs)
elif self.action_type == 'multibinary':
action_ins = [Input((2,)) for _ in range(self.action_space.n)]
outs = [Dense(2)(base_out) for _ in range(self.action_space.n)]
outs = [Multiply()([out, action_in]) for out, action_in in zip(outs, action_ins)]
model = Model([self.topology.input] + action_ins, outs)
model.compile(loss='mse', optimizer=self.optimizer)
return model
def self_attention(input_feature, num_channel, base):
"""
args:
1. input_feature: Input to the self-attention block.
2. num_channel: Number of channels after the dense operation.
3. base: layer name identifier.
"""
bn_1 = BatchNormalization(axis=-1, name=base + '/bn_1')(input_feature)
dense_1 = Dense(num_channel, name=base + '/dense_1')(bn_1)
act_1 = Activation('relu', name=base + '/act_1')(dense_1)
bn_2 = BatchNormalization(axis=-1, name=base + '/bn_2')(input_feature)
dense_2 = Dense(num_channel, name=base + '/dense_2')(bn_2)
act_2 = Activation('relu', name=base + '/act_2')(dense_2)
bn_3 = BatchNormalization(axis=-1, name=base + '/bn_3')(input_feature)
dense_3 = Dense(num_channel, name=base + '/dense_3')(bn_3)
act_3 = Activation('relu', name=base + '/act_3')(dense_3)
mul_1 = Multiply(name=base + '/mul_1')([act_2, act_3])
mask_part = Activation('softmax', name=base + '/act_4')(mul_1)
mul_2 = Multiply(name=base + '/mul_2')([act_1, mask_part])
output_feature = Add(name=base + '/add_1')([mul_2, input_feature])
return output_feature
def conv_block(inp,
channels,
output_name,
block_name='Block',
dropout=0.2,
depth=2,
kernel_size=(3, 3, 3),
activation='relu',
sSE=False,
cSE=False,
scSE=False,
bn=False):
with K.name_scope(block_name):
c_1 = Conv3D(channels[0],
kernel_size,
activation='relu',
kernel_initializer='glorot_uniform',
padding='same')(inp)
c_1 = Dropout(dropout)(c_1)
c_1 = concatenate([inp, c_1])
if scSE or (sSE and cSE):
c_2 = Conv3D(channels[1],
kernel_size,
activation=activation,
kernel_initializer='glorot_uniform',
padding='same')(c_1)
# cSE
cse = GlobalAveragePooling3D()(c_2)
cse = Dense(c_2.shape[-1] // 2, activation='relu')(cse)
cse = Dense(c_2.shape[-1], activation='sigmoid')(cse)
c_2_cse = Multiply()([c_2, cse])
# sSE
sse = Conv3D(1, (1, 1, 1),
activation='sigmoid',
kernel_initializer='glorot_uniform')(c_2)
c_2_sse = Multiply()([c_2, sse])
return Add(name=output_name)([c_2_cse, c_2_sse])
elif cSE:
sse = Conv3D(1, (1, 1, 1),
activation='sigmoid',
kernel_initializer='glorot_uniform')(c_2)
return Multiply([c_2, sse], name=output_name)
elif sSE:
# cSE
cse = GlobalAveragePooling3D()(c_2)
cse = Dense(c_2.shape[-1] // 2, activation='relu')(cse)
cse = Dense(c_2.shape[-1], activation='sigmoid')(cse)
return Multiply([c_2, cse], name=output_name)
else:
c_2 = Conv3D(channels[1],
kernel_size,
activation=activation,
kernel_initializer='glorot_uniform',
padding='same',
name=output_name)(c_1)
return c_2
def apply_mask(x, mask1, mask2, num_p, stage, branch):
w_name = "weight_stage%d_L%d" % (stage, branch)
if num_p == np_branch1:
w = Multiply(name=w_name)([x, mask1]) # vec_weight
elif num_p == np_branch2:
w = Multiply(name=w_name)([x, mask2]) # vec_heat
else:
assert False, "wrong number of layers num_p=%d " % num_p
return w
def train_net(self):
global decoder_outdim
adam_rate = 1e-4
if self.skip == True: #Handling case where Keras expects two inputs
train_data = [self.X_train[:, :2049], self.X_train[:, 2049:4098]
] #,self.X_train[:,4098:4113],self.X_train[:,4113:]]
train_target = [self.X_train[:, :2049], self.X_train[:, 2049:4098]]
val_data = [self.X_val[:, :2049], self.X_val[:, 2049:4098]
] #,self.X_val[:,4098:4113],self.X_val[:,4113:]]
val_target = [self.X_val[:, :2049], self.X_val[:, 2049:4098]]
else:
a = 1
if self.net_type == 'vae':
a = 1
else:
self.network.compile(optimizer=Adam(lr=adam_rate),
loss=self.my_mse2)
self.network.fit(x=train_data,
y=train_target,
epochs=int(self.epochs.get()),
batch_size=200,
shuffle=True,
validation_data=(val_data, val_target))
modalpha1 = Input(shape=(self.encoder_widths[-1], ))
modnegalpha1 = Input(shape=(self.encoder_widths[-1], ))
modalpha2 = Input(shape=(self.encoder_widths[-1], ))
modnegalpha2 = Input(shape=(self.encoder_widths[-1], ))
final_spec_1 = Input(shape=(decoder_outdim, )) #drum track 1
final_spec_2 = Input(shape=(decoder_outdim, )) #drum track 2
final_spec_3 = Input(shape=(decoder_outdim, )) #bass track 1
final_spec_4 = Input(shape=(decoder_outdim, )) #bass track 2
my_batch0 = [final_spec_1, final_spec_3] #drum track 1, bass track 1
my_encoded0 = self.encoder(my_batch0)
my_batch1 = [final_spec_2, final_spec_4] #drum track 2, bass track 2
my_encoded1 = self.encoder(my_batch1)
blarg0 = Multiply()([my_encoded0[0], modalpha1])
blarg1 = Multiply()([my_encoded1[0], modnegalpha1])
mod_latent1 = Add()([blarg0, blarg1])
belch0 = Multiply()([my_encoded0[1], modalpha2])
belch1 = Multiply()([my_encoded1[1], modnegalpha2])
mod_latent2 = Add()([belch0, belch1])
final_decoded = self.decoder([mod_latent1, mod_latent2])
self.dnb_net = Model(inputs=[
final_spec_1, final_spec_2, final_spec_3, final_spec_4, modalpha1,
modnegalpha1, modalpha2, modnegalpha2
],
outputs=final_decoded)
self.dnb_net.save('models/' + self.model_name.get() +
'_trained_dnb.h5')
print('Done training!')
def call(self, x):
transform_gate = self.dense_1(x)
transform_gate = Activation("sigmoid")(transform_gate)
carry_gate = Lambda(lambda x: 1.0 - x,
output_shape=(self.dim, ))(transform_gate)
transformed_data = self.dense_2(x)
transformed_data = Activation(self.activation)(transformed_data)
transformed_gated = Multiply()([transform_gate, transformed_data])
identity_gated = Multiply()([carry_gate, x])
value = Add()([transformed_gated, identity_gated])
return value
def call(self, inputs):
x, a, e = inputs
assert len(x.shape) == 3
assert len(e.shape) == 4
N = tf.shape(x)[1]
xi_shape = [-1, N, 1, x.shape[2]]
xj_shape = [-1, 1, N, x.shape[2]]
xi = tf.reshape(x, xi_shape) # b n 1 f
xj = tf.reshape(x, xj_shape) # b 1 n f
xi = tf.repeat(xi, N, axis=2)
xj = tf.repeat(xj, N, axis=1)
e_transpose = tf.transpose(e, perm=[0, 2, 1, 3])
stack = tf.concat([xi, xj, e, e_transpose], axis=-1)
for i in range(0, len(self.stack_models)):
stack = self.stack_models[i](stack)
stack = self.stack_model_acts[i](stack)
e_mask_shape = [-1, tf.shape(a)[1], tf.shape(a)[2], 1]
e_mask = tf.reshape(a, e_mask_shape)
# zero-out elements that aren't edges in the adjacency matrix
stack = Multiply()([stack, e_mask])
if self.attention:
att1 = self.incoming_att_sigmoid(stack)
incoming_e = self.incoming_att_multiply([stack, att1])
incoming_e = tf.keras.backend.sum(incoming_e,
axis=-2,
keepdims=False)
att2 = self.outgoing_att_sigmoid(stack)
outgoing_e = self.outgoing_att_multiply([stack, att2])
outgoing_e = tf.keras.backend.sum(outgoing_e,
axis=-3,
keepdims=False)
else:
incoming_e = tf.keras.backend.sum(stack, axis=-2, keepdims=False)
outgoing_e = tf.keras.backend.sum(stack, axis=-3, keepdims=False)
final_stack = Concatenate(axis=-1)([x, incoming_e, outgoing_e])
x = self.node_model(final_stack)
e = self.edge_model(stack)
# zero-out elements that aren't edges in the adjacency matrix
e = Multiply()([e, e_mask])
return x, e
def train_net(self):
global decoder_outdim
global sample_length
adam_rate = 1e-4
train_data = [self.X_train[:, :sample_length, :decoder_outdim]]
train_target = [self.X_train[:, :sample_length, :decoder_outdim]]
val_data = [self.X_val[:, :sample_length, :decoder_outdim]]
val_target = [self.X_val[:, :sample_length, :decoder_outdim]]
self.network.compile(optimizer=Adam(lr=adam_rate), loss=mse)
self.network.fit(x=train_data,
y=train_target,
epochs=self.n_epochs,
batch_size=200,
shuffle=True,
validation_data=(val_data, val_target))
mode = Input(shape=(1, ))
modalpha1 = Input(shape=(self.encoder_widths[-1], ))
modnegalpha1 = Input(shape=(self.encoder_widths[-1], ))
final_spec_1 = Input(shape=(
sample_length,
decoder_outdim,
))
final_spec_2 = Input(shape=(
sample_length,
decoder_outdim,
))
my_batch0 = [final_spec_1]
my_batch1 = [final_spec_2]
# synthesis
if mode == 0:
latent = [modalpha1]
final_decoded = self.decoder(latent)
# effects
elif mode == 1:
my_encoded = self.encoder(my_batch0)
mod_latent = Multiply()([my_encoded, modalpha1])
final_decoded = self.decoder([mod_latent])
# mixing
else:
my_encoded0 = self.encoder(my_batch0)
my_encoded1 = self.encoder(my_batch1)
blarg0 = Multiply()([my_encoded0, modalpha1])
blarg1 = Multiply()([my_encoded1, modnegalpha1])
mod_latent1 = Add()([blarg0, blarg1])
final_decoded = self.decoder([mod_latent1])
self.dnb_net = Model(
inputs=[mode, final_spec_1, final_spec_2, modalpha1, modnegalpha1],
outputs=final_decoded)
self.dnb_net.save('models/' + self.filename_out +
'_trained_network.h5')
def build_cos_model(input_size,
cos_dist_lvl,
n_neurons,
n_layers,
batch_norm=True,
loss=MeanSquaredError(),
optimizer=SGD(learning_rate=0.05, momentum=0.025)):
in_1 = Input(shape=(input_size, ), name="input_1")
in_2 = Input(shape=(input_size, ), name="input_2")
if cos_dist_lvl == 0:
model = Concatenate(name="concatenate")([in_1, in_2])
else:
model = Multiply(name="pointwise_multiply")([in_1, in_2])
if cos_dist_lvl >= 2:
norm_1 = Lambda(
lambda tensor: tf.norm(tensor, axis=1, keepdims=True),
name="norm_input_1")(in_1)
norm_2 = Lambda(
lambda tensor: tf.norm(tensor, axis=1, keepdims=True),
name="norm_input_2")(in_2)
norm_mul = Multiply(name="multiply_norms")([norm_1, norm_2])
model = Lambda(lambda tensors: tf.divide(tensors[0], tensors[1]),
name="divide")([model, norm_mul])
if cos_dist_lvl >= 3:
model = Lambda(
lambda tensor: tf.reduce_sum(tensor, axis=1, keepdims=True),
name="sum")(model)
if cos_dist_lvl >= 4:
model = ValueMinusInput(1, name="one_minus_input")(model)
if batch_norm:
model = BatchNormalization(name="input_normalization")(model)
for i in range(n_layers):
model = Dense(n_neurons,
activation='sigmoid',
name="dense_{}".format(i))(model)
model_out = Dense(1, activation='sigmoid', name="classify")(model)
model = Model([in_1, in_2], model_out)
model.compile(loss=loss,
optimizer=optimizer,
metrics=[
BinaryAccuracy(),
Precision(),
Recall(),
TrueNegatives(),
FalsePositives(),
FalseNegatives(),
TruePositives()
])
return model
def call(self, x):
dim = K.int_shape(x)[-1]
transform_gate = self.dense_1(x)
transform_gate = Activation("sigmoid")(transform_gate)
carry_gate = Lambda(lambda x: 1.0 - x,
output_shape=(dim, ))(transform_gate)
hidden = self.dense_2(x)
hidden = Activation(self.activation)(hidden)
transformed_gated = Multiply()([transform_gate, hidden])
identity_gated = Multiply()([carry_gate, x])
value = Add()([transformed_gated, identity_gated])
return value
def __init__(self, model_params):
super(SiameseModel, self).__init__()
max_len = model_params['max_len']
embedding_matrix = model_params['embedding_matrix']
num_unique_words = model_params['num_unique_words']
embedding_dim = model_params['embedding_dim']
self.sentence_1 = Input(shape=(None, ))
self.sentence_2 = Input(shape=(None, ))
self.sentence_1_embedding = Embedding(num_unique_words,embedding_dim,\
embeddings_initializer=keras.initializers.Constant(embedding_matrix),\
trainable=False)(self.sentence_1)
self.sentence_2_embedding = Embedding(num_unique_words,embedding_dim,\
embeddings_initializer=keras.initializers.Constant(embedding_matrix),\
trainable=False)(self.sentence_2)
self.len_sent_1 = Input(shape=(None, 1), dtype='int32')
self.len_sent_2 = Input(shape=(None, 1), dtype='int32')
#self.squared_diff = Multiply([tf_sum(self.difference),tf_sum(self.difference)])
self.shared_lstm = LSTM(1000)
self.question_1_encoding = self.shared_lstm(self.sentence_1_embedding)
self.question_2_encoding = self.shared_lstm(self.sentence_2_embedding)
self.Hadamard = Multiply()(
[self.question_1_encoding, self.question_2_encoding])
self.difference = subtract(
[self.question_1_encoding, self.question_2_encoding])
self.squared_euclidean_distance = tf_sum(
Multiply()([self.difference, self.difference]), axis=1)
self.squared_euclidean_distance = k.expand_dims(
self.squared_euclidean_distance, axis=-1)
self.features = concatenate([self.question_1_encoding, self.question_2_encoding,\
self.Hadamard, self.squared_euclidean_distance],)
self.h1 = Dense(800, activation='relu')(self.features)
self.dropout1 = keras.layers.Dropout(0.7)(self.h1)
self.h2 = Dense(800, activation='relu')(self.dropout1)
self.dropout2 = keras.layers.Dropout(0.7)(self.h2)
self.out = Dense(2, activation='softmax')(self.dropout2)
initial_learning_rate = model_params['lr']
decay_rate = model_params['lr_decay_rate']
learning_rate = keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate,
decay_steps=10000,
decay_rate=decay_rate,
staircase=True)
optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
self.model = Model(inputs=[
self.sentence_1, self.sentence_2, self.len_sent_1, self.len_sent_2
],
outputs=[self.out])
self.model.compile(loss="binary_crossentropy",
optimizer=optimizer,
metrics=['binary_accuracy', f1_m])
def build_DTLN_model(self, norm_stft=False):
'''
Method to build and compile the DTLN model. The model takes time domain
batches of size (batchsize, len_in_samples) and returns enhanced clips
in the same dimensions. As optimizer for the Training process the Adam
optimizer with a gradient norm clipping of 3 is used.
The model contains two separation cores. The first has an STFT signal
transformation and the second a learned transformation based on 1D-Conv
layer.
'''
# input layer for time signal
time_dat = Input(batch_shape=(None, None))
# calculate STFT
mag, angle = Lambda(self.stftLayer)(time_dat)
# normalizing log magnitude stfts to get more robust against level variations
if norm_stft:
mag_norm = InstantLayerNormalization()(tf.math.log(mag + 1e-7))
else:
# behaviour like in the paper
mag_norm = mag
# predicting mask with separation kernel
mask_1 = self.seperation_kernel(self.numLayer,
(self.blockLen // 2 + 1), mag_norm)
# multiply mask with magnitude
estimated_mag = Multiply()([mag, mask_1])
# transform frames back to time domain
estimated_frames_1 = Lambda(self.ifftLayer)([estimated_mag, angle])
# encode time domain frames to feature domain
encoded_frames = Conv1D(self.encoder_size,
1,
strides=1,
use_bias=False)(estimated_frames_1)
# normalize the input to the separation kernel
encoded_frames_norm = InstantLayerNormalization()(encoded_frames)
# predict mask based on the normalized feature frames
mask_2 = self.seperation_kernel(self.numLayer, self.encoder_size,
encoded_frames_norm)
# multiply encoded frames with the mask
estimated = Multiply()([encoded_frames, mask_2])
# decode the frames back to time domain
decoded_frames = Conv1D(self.blockLen,
1,
padding='causal',
use_bias=False)(estimated)
# create waveform with overlap and add procedure
estimated_sig = Lambda(self.overlapAddLayer)(decoded_frames)
# create the model
self.model = Model(inputs=time_dat, outputs=estimated_sig)
# show the model summary
print(self.model.summary())
def attention(query, key, value, att_hidden_units=(64, 16)):
# 旧实现，不推荐
"""
:param query: (None,D)
:param key: (None,slt_api_num,D)
:param value: (None,slt_api_num,D) 一般等于key
:return:
"""
slt_api_num = key.shape[
1].value # shape[-1]不要直接当做实数，type是Dimension！使用value!!!
# query = Lambda(lambda x: K.repeat_elements(x,slt_api_num,1))(query)
query = RepeatVector(slt_api_num)(query) # (None,slt_api_num,D)
outer_prod = Multiply()([query, key])
sub = Subtract()([query, key])
att_score = Concatenate(name='att_info_concate')(
[query, key, outer_prod, sub]) # (None,slt_api_num,4*D)
# 使用卷积对每个slt api进行相同的操作，分别得到若干个分值：
# 'channels_last' (None,slt_api_num,4*D,1)-> (None,slt_api_num,1,1) -> (None,slt_api_num,1)
kernel_size = att_score.shape[-1].value
att_score = Lambda(lambda x: K.expand_dims(x, axis=3))(att_score)
att_score = Conv2D(att_hidden_units[0],
kernel_size=(1, kernel_size),
activation='relu',
name='att_fc_{}'.format(1))(att_score)
# 结果是(None,slt_api_num,1,64)
for index, unit_num in enumerate(att_hidden_units[1:]): #
att_score = Conv2D(unit_num,
kernel_size=(1, 1),
activation='relu',
name='att_fc_{}'.format(index + 2))(
att_score) # 2维怎么使用一维dense？ 可以conv
# print(att_score)
# h层之后再进行softmax
att_score = Conv2D(1,
kernel_size=(1, 1),
activation='linear',
use_bias=False,
name='att_fc_h')(att_score) # (None,slt_api_num,1,1)
att_score = Reshape((slt_api_num, 1))(att_score) # (None,slt_api_num,1)
# att_score = Softmax(axis=1) (att_score) 有问题！！！
att_result = Multiply()([value, att_score]) # (None,slt_api_num,D)
att_result = Lambda(lambda x: tf.reduce_sum(x, axis=1))(
att_result) # (None,D)
print(att_result)
return att_result
def get_highway_output(highway_input, nb_layers, activation="tanh", bias=-3):
dim = K.int_shape(highway_input)[-1] # dimension must be the same
initial_bias = k_init.Constant(bias)
for n in range(nb_layers):
H = Dense(units=dim, bias_initializer=initial_bias)(highway_input)
H = Activation("sigmoid")(H)
carry_gate = Lambda(lambda x: 1.0 - x, output_shape=(dim, ))(H)
transform_gate = Dense(units=dim)(highway_input)
transform_gate = Activation(activation)(transform_gate)
transformed = Multiply()([H, transform_gate])
carried = Multiply()([carry_gate, highway_input])
highway_output = Add()([transformed, carried])
return highway_output
