def MT_attention_model():
input = Input((MAX_SIZE, 300))
# encode with MT_attention
transformed_input = attention_MT_LSTM(input, MAX_SIZE)
result = Dropout(0.2)(Bidirectional(LSTM(300, dropout=0.2, recurrent_dropout=0.2, return_sequences=True, kernel_regularizer=regularizers.l2(0.01)))(transformed_input))
attention_mul = attention_3d_block(result, MAX_SIZE)
summed_attention = Lambda(lambda x: K.sum(x, axis=1))(attention_mul)
fc = Dropout(0.2)(Dense(400, activation="relu", kernel_regularizer=regularizers.l2(0.01))(summed_attention))
output = Dense(NR_CLASS, activation="softmax")(fc)
model = Model(inputs=input, outputs=output)
return model, "MT_attention"
def attention_model():
input = Input((MAX_SIZE,300))
bi_lstm = Bidirectional(LSTM(300,dropout=0.2, recurrent_dropout=0.2, return_sequences=True))(input)
#bi_lstm2 = Bidirectional(LSTM(128, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))(bi_lstm)
attention_mul = attention_3d_block(bi_lstm)
summed_attention = Lambda(lambda x: K.sum(x, axis=1))(attention_mul)
fc = Dense(400, activation="relu", kernel_regularizer=regularizers.l2(0.0001))(summed_attention)
fc = Dropout(0.2)(fc)
output = Dense(NR_CLASS, activation="softmax")(fc)
model = Model(inputs=input, outputs=output)
return model
def bytenet_model():
inp = Input((60,300))
out = inp
outs = []
for dilation in [1,2,4,8]:
out = Dropout(0.2)(bytenet_resblock(out, dilation))
outs.append(out)
attentions = []
for out in outs:
attention_mul = attention_3d_block(out, out._keras_shape[1])
summed_attention = Dropout(0.2)(Lambda(lambda x: K.sum(x, axis=1))(attention_mul))
attentions.append(summed_attention)
concate_attention = Concatenate()(attentions)
fc = Dense(400, activation="relu", kernel_regularizer=regularizers.l2(0.01))(concate_attention)
out = Dense(5, activation="softmax")(fc)
model = Model(inputs=inp, outputs=out)
return model
def convolution_attention_model():
inp = Input(shape=(MAX_SIZE,300))
conv_in = inp
conv_outs = []
for _ in range(4):
conv_in = res_conv_pool(conv_in, 5, 300)
conv_outs.append(conv_in)
# apply attention for each layer
attentions = []
for conv_out in conv_outs:
attention_mul = attention_3d_block(conv_out, conv_out._keras_shape[1])
summed_attention = Dropout(0.2)(Lambda(lambda x: K.sum(x, axis=1))(attention_mul))
attentions.append(summed_attention)
concate = Concatenate()(attentions)
fc = Dense(1200, activation="relu", kernel_regularizer=regularizers.l2(0.01))(concate)
out = Dense(5, activation="softmax")(fc)
model = Model(inputs = inp, outputs = out)
return model
def conv_lstm_model():
inp = Input(shape=(MAX_SIZE,300))
dropout_inp = Dropout(0.2)(inp)
# get from high-level feat
pool1 = res_conv_pool(dropout_inp, 3, 300)
pool2 = res_conv_pool(pool1, 3, 300)
pool3 = res_conv_pool(pool2, 3, 300)
attentions = []
for pool in [pool1, pool2, pool3]:
lstm = Bidirectional(LSTM(300, dropout=0.2, recurrent_dropout=0.2, return_sequences=True, kernel_regularizer=regularizers.l2(0.01)))(pool)
attention_mul = attention_3d_block(lstm,lstm._keras_shape[1])
summed_attention = Dropout(0.2)(Lambda(lambda x: K.sum(x, axis=1))(attention_mul))
attentions.append(summed_attention)
concate_attention = Concatenate()(attentions)
fc = Dense(1200, activation="relu", kernel_regularizer=regularizers.l2(0.01))(concate_attention)
out = Dense(NR_CLASS, activation="softmax")(fc)
model = Model(inputs = inp, outputs = out)
return model
def get_feat(inp):
attention_mul = attention_3d_block(inp, 60)
self_attention = Lambda(lambda x: K.sum(x, axis=1))(attention_mul)
return self_attention