def gru_keras(max_features,
maxlen,
bidirectional,
dropout_rate,
embed_dim,
rec_units,
mtype='GRU',
reduction=None,
classes=4,
lr=0.001):
if K.backend == 'tensorflow':
K.clear_session()
input_layer = Input(shape=(maxlen, ))
embedding_layer = Embedding(max_features,
output_dim=embed_dim,
trainable=True)(input_layer)
x = SpatialDropout1D(dropout_rate)(embedding_layer)
if reduction:
if mtype == 'GRU':
if bidirectional:
x = Bidirectional(
CuDNNGRU(units=rec_units, return_sequences=True))(x)
else:
x = CuDNNGRU(units=rec_units, return_sequences=True)(x)
elif mtype == 'LSTM':
if bidirectional:
x = Bidirectional(
CuDNNLSTM(units=rec_units, return_sequences=True))(x)
else:
x = CuDNNLSTM(units=rec_units, return_sequences=True)(x)
if reduction == 'average':
x = GlobalAveragePooling1D()(x)
elif reduction == 'maximum':
x = GlobalMaxPool1D()(x)
else:
if mtype == 'GRU':
if bidirectional:
x = Bidirectional(
CuDNNGRU(units=rec_units, return_sequences=False))(x)
else:
x = CuDNNGRU(units=rec_units, return_sequences=False)(x)
elif mtype == 'LSTM':
if bidirectional:
x = Bidirectional(
CuDNNLSTM(units=rec_units, return_sequences=False))(x)
else:
x = CuDNNLSTM(units=rec_units, return_sequences=False)(x)
output_layer = Dense(classes, activation="sigmoid")(x)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(learning_rate=lr, clipvalue=1, clipnorm=1),
metrics=['acc'])
return model
def NN_huaweiv1(maxlen, embedding_matrix=None, class_num1=17, class_num2=12):
emb_layer = Embedding(
embedding_matrix.shape[0],
embedding_matrix.shape[1],
input_length=maxlen,
weights=[embedding_matrix],
trainable=False,
)
seq1 = Input(shape=(maxlen, ))
x1 = emb_layer(seq1)
sdrop = SpatialDropout1D(rate=0.2)
lstm_layer = Bidirectional(CuDNNGRU(128, return_sequences=True))
gru_layer = Bidirectional(CuDNNGRU(128, return_sequences=True))
cnn1d_layer = Conv1D(64,
kernel_size=3,
padding="same",
kernel_initializer="he_uniform")
x1 = sdrop(x1)
lstm1 = lstm_layer(x1)
gru1 = gru_layer(lstm1)
att_1 = Attention(maxlen)(lstm1)
att_2 = Attention(maxlen)(gru1)
cnn1 = cnn1d_layer(lstm1)
avg_pool = GlobalAveragePooling1D()
max_pool = GlobalMaxPooling1D()
x1 = concatenate([
att_1, att_2,
Attention(maxlen)(cnn1),
avg_pool(cnn1),
max_pool(cnn1)
])
x = Dropout(0.2)(Activation(activation="relu")(BatchNormalization()(
Dense(128)(x1))))
x = Activation(activation="relu")(BatchNormalization()(Dense(64)(x)))
pred1_d = Dense(class_num1)(x)
pred1 = Activation(activation='sigmoid', name='pred1')(pred1_d)
y = concatenate([x1, x])
y = Activation(activation="relu")(BatchNormalization()(Dense(64)(x)))
pred2_d = Dense(class_num2)(y)
pred2 = Activation(activation='sigmoid', name='pred2')(pred2_d)
z = Dropout(0.2)(Activation(activation="relu")(BatchNormalization()(
Dense(128)(x1))))
z = concatenate([pred1_d, pred2_d, z])
pred3 = Dense(class_num1 + class_num2, activation='sigmoid',
name='pred3')(z)
model = Model(inputs=seq1, outputs=[pred1, pred2, pred3])
return model
def Build_RNN_layer(Parametre_layer):
if Parametre_layer["type_cell"] == "LSTM":
cell = L.LSTM(units=Parametre_layer["units"],
dropout=Parametre_layer["dropout"],
recurrent_dropout=Parametre_layer["recurrent_dropout"],
return_sequences=True,
return_state=True,
stateful=Parametre_layer["stateful"],
unroll=Parametre_layer["unroll"])
elif Parametre_layer["type_cell"] == "CuDNNGRU":
cell = CuDNNGRU(units=Parametre_layer["units"],
dropout=Parametre_layer["dropout"],
recurrent_dropout=Parametre_layer["recurrent_dropout"],
return_sequences=True,
return_state=True,
stateful=Parametre_layer["stateful"],
unroll=Parametre_layer["unroll"])
elif Parametre_layer["type_cell"] == "GRU":
cell = L.GRU(units=Parametre_layer["units"],
return_sequences=True,
return_state=True,
stateful=Parametre_layer["stateful"])
else: #by default CuDNNLSTM
cell = CuDNNLSTM(units=Parametre_layer["units"],
return_sequences=True,
return_state=True,
stateful=Parametre_layer["stateful"])
return cell
def NN_huaweiv1(maxlen, embedding_matrix=None, class_num1=17, class_num2=12):
emb_layer = Embedding(
embedding_matrix.shape[0],
embedding_matrix.shape[1],
input_length=maxlen,
weights=[embedding_matrix],
trainable=False,
)
seq1 = Input(shape=(maxlen, ))
emb = emb_layer(seq1)
sdrop = SpatialDropout1D(rate=0.2)
lstm_layer = Bidirectional(CuDNNGRU(128, return_sequences=True))
gru_layer = Bidirectional(CuDNNGRU(128, return_sequences=True))
cnn1d_layer = Conv1D(64,
kernel_size=3,
padding="same",
kernel_initializer="he_uniform")
sd = sdrop(emb)
lstm1 = lstm_layer(sd)
gru1 = gru_layer(lstm1)
cnn1 = cnn1d_layer(gru1)
gru1 = concatenate([lstm1, gru1, cnn1])
att_1 = Attention(maxlen)(gru1)
att_2 = Attention(maxlen)(gru1)
att_3 = Attention(maxlen)(gru1)
att_4 = Attention(maxlen)(gru1)
x1 = Activation(activation="relu")(BatchNormalization()(Dense(128)(att_1)))
x2 = Activation(activation="relu")(BatchNormalization()(Dense(128)(att_2)))
x3 = Activation(activation="relu")(BatchNormalization()(Dense(128)(att_3)))
x4 = Activation(activation="relu")(BatchNormalization()(Dense(128)(att_4)))
pred1_1 = Dense(class_num1 - 10, activation='sigmoid')(x1)
pred1_2 = Dense(10, activation='sigmoid')(x2)
pred1 = concatenate([pred1_1, pred1_2], axis=-1, name='pred1')
pred2_1 = Dense(class_num2 - 9, activation='sigmoid')(x3)
pred2_2 = Dense(9, activation='sigmoid')(x4)
pred2 = concatenate(
[pred2_1, pred2_2], axis=-1, name='pred2'
) # Dense(class_num2, activation='sigmoid',name='pred2')(y)
model = Model(inputs=seq1, outputs=[pred1, pred2])
return model
def create_model():
embedding_layer = Embedding(num_words,
EMBEDDING_DIM,
input_length=MAX_POST_LENGTH,
weights=[embedding_matrix],
trainable=False)
sequence_input = Input(shape=(MAX_POST_LENGTH, ))
embedded_sequences = embedding_layer(sequence_input)
l_lstm_sent = Bidirectional(CuDNNGRU(
50, return_sequences=True))(embedded_sequences)
l_lstm_sent = Dropout(0.2)(l_lstm_sent)
l_lstm_sent = AttentionWithContext()(l_lstm_sent)
l_lstm_sent = Dropout(0.2)(l_lstm_sent)
preds = Dense(units=2, activation='softmax')(l_lstm_sent)
sentEncoder = Model(sequence_input, preds)
print(sentEncoder.summary())
ana_input = Input(shape=(MAX_POSTS, len(i_data[0][0])))
review_input = Input(shape=(MAX_POSTS, MAX_POST_LENGTH))
l_lstm_sent = TimeDistributed(sentEncoder)(review_input)
l_lstm_sent = concatenate([l_lstm_sent, ana_input
]) # combine time series and categories
l_lstm_sent = BatchNormalization()(l_lstm_sent)
l_lstm_sent = Dropout(0.2)(l_lstm_sent)
l_lstm_sent = Bidirectional(CuDNNGRU(16,
return_sequences=True))(l_lstm_sent)
l_lstm_sent = Dropout(0.2)(l_lstm_sent)
l_lstm_sent = AttentionWithContext()(l_lstm_sent)
l_lstm_sent = Dropout(0.2)(l_lstm_sent)
preds = Dense(2, activation='softmax')(l_lstm_sent)
model = Model([review_input, ana_input], preds)
print(model.summary())
from keras.optimizers import Adam, AdaMod
adam = AdaMod()
model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['acc'])
return model
def get_model(embedding_matrix, max_len, max_features, embed_size):
inp = Input(shape=(max_len, ))
x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)
x = CuDNNGRU(64, return_sequences=True)(x)
x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
conc = concatenate([avg_pool, max_pool])
conc = Dense(64, activation="relu")(conc)
conc = Dropout(0.1)(conc)
outp = Dense(1, activation="sigmoid")(conc)
model = Model(inputs=inp, outputs=outp)
model.compile(loss='binary_crossentropy',
optimizer=tf.optimizers.Adam(learning_rate=0.005),
metrics=['AUC'])
return model
def new_lpcnet_model(rnn_units1=384,
rnn_units2=16,
nb_used_features=38,
training=False,
adaptation=False,
quantize=False):
pcm = Input(shape=(None, 3))
feat = Input(shape=(None, nb_used_features))
pitch = Input(shape=(None, 1))
dec_feat = Input(shape=(None, 128))
dec_state1 = Input(shape=(rnn_units1, ))
dec_state2 = Input(shape=(rnn_units2, ))
padding = 'valid' if training else 'same'
fconv1 = Conv1D(128,
3,
padding=padding,
activation='tanh',
name='feature_conv1')
fconv2 = Conv1D(128,
3,
padding=padding,
activation='tanh',
name='feature_conv2')
embed = Embedding(256,
embed_size,
embeddings_initializer=PCMInit(),
name='embed_sig')
cpcm = Reshape((-1, embed_size * 3))(embed(pcm))
pembed = Embedding(256, 64, name='embed_pitch')
cat_feat = Concatenate()([feat, Reshape((-1, 64))(pembed(pitch))])
cfeat = fconv2(fconv1(cat_feat))
fdense1 = Dense(128, activation='tanh', name='feature_dense1')
fdense2 = Dense(128, activation='tanh', name='feature_dense2')
cfeat = fdense2(fdense1(cfeat))
rep = Lambda(lambda x: K.repeat_elements(x, frame_size, 1))
quant = quant_regularizer if quantize else None
if training:
rnn = CuDNNGRU(rnn_units1,
return_sequences=True,
return_state=True,
name='gru_a',
recurrent_constraint=constraint,
recurrent_regularizer=quant)
rnn2 = CuDNNGRU(rnn_units2,
return_sequences=True,
return_state=True,
name='gru_b',
kernel_constraint=constraint,
kernel_regularizer=quant)
else:
rnn = GRU(rnn_units1,
return_sequences=True,
return_state=True,
recurrent_activation="sigmoid",
reset_after='true',
name='gru_a',
recurrent_constraint=constraint,
recurrent_regularizer=quant)
rnn2 = GRU(rnn_units2,
return_sequences=True,
return_state=True,
recurrent_activation="sigmoid",
reset_after='true',
name='gru_b',
kernel_constraint=constraint,
kernel_regularizer=quant)
rnn_in = Concatenate()([cpcm, rep(cfeat)])
md = MDense(pcm_levels, activation='sigmoid', name='dual_fc')
gru_out1, _ = rnn(rnn_in)
gru_out2, _ = rnn2(Concatenate()([gru_out1, rep(cfeat)]))
ulaw_prob = Lambda(tree_to_pdf_train)(md(gru_out2))
if adaptation:
rnn.trainable = False
rnn2.trainable = False
md.trainable = False
embed.Trainable = False
model = Model([pcm, feat, pitch], ulaw_prob)
model.rnn_units1 = rnn_units1
model.rnn_units2 = rnn_units2
model.nb_used_features = nb_used_features
model.frame_size = frame_size
encoder = Model([feat, pitch], cfeat)
dec_rnn_in = Concatenate()([cpcm, dec_feat])
dec_gru_out1, state1 = rnn(dec_rnn_in, initial_state=dec_state1)
dec_gru_out2, state2 = rnn2(Concatenate()([dec_gru_out1, dec_feat]),
initial_state=dec_state2)
dec_ulaw_prob = Lambda(tree_to_pdf_infer)(md(dec_gru_out2))
decoder = Model([pcm, dec_feat, dec_state1, dec_state2],
[dec_ulaw_prob, state1, state2])
return model, encoder, decoder
x = Conv1D(16, 11, padding='valid', activation='relu', strides=1)(x)
x = MaxPooling1D(3)(x)
x = Dropout(0.3)(x)
# Third Conv1D layer
x = Conv1D(32, 9, padding='valid', activation='relu', strides=1)(x)
x = MaxPooling1D(3)(x)
x = Dropout(0.3)(x)
x = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(x)
x = Bidirectional(CuDNNGRU(128, return_sequences=True), merge_mode='sum')(x)
x = Bidirectional(CuDNNGRU(128, return_sequences=True), merge_mode='sum')(x)
x = Bidirectional(CuDNNGRU(128, return_sequences=False), merge_mode='sum')(x)
x = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(x)
# Flatten layer
# x = Flatten()(x)
# Dense Layer 1
x = Dense(256, activation='relu')(x)
outputs = Dense(len(labels), activation="softmax")(x)
xte,yte = test xtr = sequence.pad_sequences(xtr, padding='pre', maxlen=max_review_length) xte = sequence.pad_sequences(xte, padding='pre', maxlen=max_review_length) embedding_vecor_length =64 bs=32 ep=2 num_classes=2 ytr1 = keras.utils.to_categorical(ytr, num_classes) yte1 = keras.utils.to_categorical(yte, num_classes) rl=LeakyReLU(alpha=.01) model = Sequential() model.add(Embedding(NUM_WORDS, embedding_vecor_length, input_length=max_review_length)) # model.add(Dropout(0.5)) model.add(Conv1D(filters=64, kernel_size=3, padding='same', activation=rl)) model.add(MaxPooling1D(pool_size=2)) model.add(Dropout(0.5)) model.add(CuDNNGRU(50, return_sequences=True)) model.add(CuDNNGRU(50)) model.add(Dropout(0.3)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # print(model.summary()) model.fit(xtr, ytr1, epochs=ep, batch_size=bs, validation_data=(xte, yte1))