def bigru_with_max_aver_pool_model(self,
units_gru: int,
dense_size: int,
coef_reg_gru: float = 0.,
coef_reg_den: float = 0.,
dropout_rate: float = 0.,
rec_dropout_rate: float = 0.,
**kwargs) -> Model:
"""
Method builds uncompiled model Bidirectional GRU with concatenation of max and average pooling after BiGRU.
Args:
units_gru: number of units for GRU.
dense_size: number of units for dense layer.
coef_reg_gru: l2-regularization coefficient for GRU. Default: ``0.0``.
coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
dropout_rate: dropout rate to be used after BiGRU and between dense layers. Default: ``0.0``.
rec_dropout_rate: dropout rate for GRU. Default: ``0.0``.
kwargs: other non-used parameters
Returns:
keras.models.Model: uncompiled instance of Keras Model
"""
inp = Input(shape=(self.opt['text_size'], self.opt['embedding_size']))
output = Dropout(rate=dropout_rate)(inp)
output, state1, state2 = Bidirectional(
GRU(units_gru,
activation='tanh',
return_sequences=True,
return_state=True,
kernel_regularizer=l2(coef_reg_gru),
dropout=dropout_rate,
recurrent_dropout=rec_dropout_rate))(output)
output1 = GlobalMaxPooling1D()(output)
output2 = GlobalAveragePooling1D()(output)
output = Concatenate()([output1, output2, state1, state2])
output = Dropout(rate=dropout_rate)(output)
output = Dense(dense_size,
activation=None,
kernel_regularizer=l2(coef_reg_den))(output)
output = Activation('relu')(output)
output = Dropout(rate=dropout_rate)(output)
output = Dense(self.n_classes,
activation=None,
kernel_regularizer=l2(coef_reg_den))(output)
act_output = Activation(
self.opt.get("last_layer_activation", "sigmoid"))(output)
model = Model(inputs=inp, outputs=act_output)
return model
def GRUConvdeep3(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
x = Activation('tanh')(embed_input_)
x = SpatialDropout1D(0.1, name='embed_drop')(x)
if params['bidirectional']:
x_g1 = Bidirectional(
CuDNNGRU(params['lstm_units'], return_sequences=True))(x)
x_l1 = Bidirectional(
CuDNNLSTM(params['lstm_units'], return_sequences=True))(x_g1)
x_g2 = Bidirectional(
CuDNNGRU(params['lstm_units'], return_sequences=True))(x_l1)
x1 = GlobalMaxPooling1D()(x_g2)
x2 = GlobalAveragePooling1D()(x_g2)
merge_layer = concatenate([x_g1, x_l1, x_g2])
x_conv = Conv1D(64,
kernel_size=3,
padding='valid',
kernel_initializer='glorot_uniform')(merge_layer)
x1_conv = GlobalMaxPooling1D()(x_conv)
x2_conv = GlobalAveragePooling1D()(x_conv)
merge_layer2 = concatenate([x1_conv, x2_conv])
x = Dropout(params['dropout_rate'])(merge_layer2)
x = Dense(256, activation='relu')(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def model_conv_glob(data, paramdims):
'''
Conv1D:
{} x {}, relu
GlobPool
'''
input = Input(shape=data['input_1'].shape, name='input_1')
layer = Conv1D(paramdims[0], kernel_size=(paramdims[1]),
activation='relu')(input)
output = GlobalAveragePooling1D()(layer)
return input, output
def get_model(num_users, num_items, layers=[20, 10], reg_layers=[0, 0]):
assert len(layers) == len(reg_layers)
num_layer = len(layers) # Number of layers in the MLP
# Input variables
user_input = Input(shape=(44, ), name='user_input')
item_input = Input(shape=(16, ), name='item_input')
vector = concatenate([user_input, item_input])
'''
15d:
vector = Embedding(input_dim=vocabulary_lenth + 1, output_dim=100)(vector)
print('1:', vector.shape)
vector = Reshape((15, 4, 100))(vector)
print('2:', vector.shape)
vector = Lambda(lambda x: K.mean(x, axis=2))(vector)
print('3:', vector.shape)
O_seq = Attention(5, 120)([vector, vector, vector])
print('5', O_seq.shape)
O_seq = GlobalAveragePooling1D()(O_seq)
print('6', O_seq.shape)
O_seq = Dropout(0.5)(O_seq)
outputs = Dense(1, activation='sigmoid')(O_seq)
print('7', outputs.shape)
'''
vector = Embedding(input_dim=vocabulary_lenth + 1, output_dim=500)(vector)
print(vector.shape)
out_u = Lambda(lambda x: x[:, 0:44])(vector)
out_i = Lambda(lambda x: x[:, 44:60])(vector)
uilist = []
for vector, size in ((out_u, 11), (out_i, 4)):
print('0:', vector.shape)
vector = Reshape((size, 4, 500))(vector)
print('1:', vector.shape)
vector = Lambda(lambda x: K.mean(x, axis=1))(vector)
# vector = GlobalAveragePooling1D()(vector)
print('2:', vector.shape)
uilist.append(vector)
vector = Concatenate(axis=1)(uilist)
print('3:', vector.shape)
O_seq = Attention(5, 240)([vector, vector, vector])
print('4', O_seq.shape)
O_seq = GlobalAveragePooling1D()(O_seq)
print('5', O_seq.shape)
O_seq = Dropout(0.5)(O_seq)
outputs = Dense(1, activation='sigmoid')(O_seq)
print('6', outputs.shape)
model = Model(inputs=[user_input, item_input], outputs=outputs)
return model
def _createModel(self, outDim):
modelLeft = Sequential()
#modelLeft.add(Masking(mask_value=0., input_shape=(self._phraseLen, self._vecLen)))
modelLeft.add(LSTM(outDim, input_shape=(self._phraseLen, self._vecLen), dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True))
modelLeft.add(LSTM(outDim, dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True))
modelLeft.add(GlobalAveragePooling1D())
modelRight = Sequential()
#modelRight.add(Masking(mask_value=0., input_shape=(self._phraseLen, self._vecLen)))
modelRight.add(LSTM(outDim, input_shape=(self._phraseLen, self._vecLen), dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True))
modelRight.add(LSTM(outDim, dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True))
modelRight.add(GlobalAveragePooling1D())
model = Sequential()
model.add(Merge([modelLeft, modelRight], mode='concat'))
model.add(Dense(outDim, activation='relu'))
model.add(Dense(outDim, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
def build_model():
model = Sequential()
model.add(TimeDistributed(Conv2D(num_filters, kernel_size, padding=padding_mode, data_format="channels_last"), input_shape=(NUM_FRAMES, IMG_HEIGHT, IMG_WIDTH, 3), name='Conv')) # shape = (frames, width, height, channel)
model.add(TimeDistributed(MaxPooling2D(pool_size=(2, 2), name='MaxPooling')))
model.add(TimeDistributed(Flatten(), name='Flatten_1'))
model.add(LSTM(20, return_sequences=True, name='LSTM'))
model.add(TimeDistributed(Dense(NUM_CLASSES, activation='linear'), name='Dense'))
model.add(GlobalAveragePooling1D(name='average'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
return model
def FastText():
input_q = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='float64')
q = embedding_raw()(input_q)
q = GlobalAveragePooling1D()(q)
q = Dropout(DROPOUT_RATE)(q)
input_a = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='float64')
a = embedding_raw()(input_a)
a = GlobalAveragePooling1D()(a)
a = Dropout(DROPOUT_RATE)(a)
merged = concatenate([q, a])
merged = Dense(64)(merged)
merged = BatchNormalization()(merged)
merged = Activation('relu')(merged)
merged = Dropout(DROPOUT_RATE)(merged)
merged = Dense(1, activation="sigmoid")(merged)
model = Model([input_q, input_a], [merged])
return model
def build_model():
desc_input = Input(shape=(length, dim), name='title1')
x = CuDNNGRU(3 * dim, return_sequences=True)(desc_input)
x = CuDNNGRU(3 * dim, return_sequences=True)(x)
x = CuDNNGRU(3 * dim, return_sequences=True)(x)
x = CuDNNGRU(3 * dim, return_sequences=True)(x)
x = CuDNNGRU(3 * dim, return_sequences=True)(x)
x = CuDNNGRU(2 * dim, return_sequences=True)(x)
h = Dense(2 * dim, activation='tanh')(x)
s = Reshape((1, 2 * dim))(GlobalAveragePooling1D()(x))
a = Dot(axes=-1)([x, s])
a = Reshape((length, 1))(Activation('softmax')(a))
x = Multiply()([x, a])
x = Lambda(lambda z: z * length)(x)
# AVG
x = GlobalAveragePooling1D()(x)
x = Dense(dim, activation=None)(x)
output = Dense(dim, activation=None)(x)
model = Model(inputs=[desc_input], outputs=[output])
return model
def model_lstm(data, paramdims):
'''
LSTM:
{}, relu
GlobPool
'''
input = Input(shape=data['input_1'].shape, name='input_1')
layer = LSTM(paramdims[0], return_sequences=True)(input)
#layer = Conv1D(paramdims[0], kernel_size=(paramdims[1]), activation = 'relu')(input)
output = GlobalAveragePooling1D()(layer)
return input, output
def get_capsule_model():
maxlen = 100
learning_rate = 0.00001
input1 = Input(shape=(maxlen, ))
embed_layer = Embedding(len(word2index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=maxlen,
trainable=True)(input1)
embed_layer = SpatialDropout1D(rate_drop_dense)(embed_layer)
x = Bidirectional(
GRU(gru_len,
activation='relu',
dropout=dropout_p,
recurrent_dropout=dropout_p,
return_sequences=True))(embed_layer)
# capsule是一个卷积网络
capsule = Capsule(num_capsule=Num_capsule,
dim_capsule=Dim_capsule,
routings=Routings,
share_weights=True)(x)
# output_capsule = Lambda(lambda x: K.sqrt(K.sum(K.square(x), 2)))(capsule)
avg_pool = GlobalAveragePooling1D()(capsule)
max_pool = GlobalMaxPooling1D()(capsule)
capsule_concat = Concatenate()([avg_pool, max_pool])
capsule = Flatten()(capsule)
caspule = BatchNormalization()(capsule_concat)
capsule = Activation('relu')(capsule)
capsule = Dropout(dropout_p)(capsule)
output = Dense(30, activation='sigmoid')(capsule)
model = Model(inputs=input1, outputs=output)
# print(capsule.shape, output.shape)
adam = Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0)
sgd = SGD(lr=learning_rate, momentum=0.0, decay=0.0, nesterov=False)
rmsprop = RMSprop(lr=learning_rate, rho=0.9, epsilon=None, decay=0.0)
nadam = keras.optimizers.Nadam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=[f1_score])
return model
def build_bidirectional_model(opts, vocab_size=0, maxnum=50, maxlen=50, embedd_dim=50, embedding_weights=None, verbose=False, init_mean_value=None):
N = maxnum
L = maxlen
logger = get_logger("Build bidirectional model")
logger.info("Model parameters: max_sentnum = %d, max_sentlen = %d, embedding dim = %s, lstm_units = %s, drop rate = %s, l2 = %s" % (N, L, embedd_dim,
opts.lstm_units, opts.dropout, opts.l2_value))
word_input = Input(shape=(N*L,), dtype='int32', name='word_input')
x = Embedding(output_dim=embedd_dim, input_dim=vocab_size, input_length=N*L, weights=embedding_weights, name='x')(word_input)
drop_x = Dropout(opts.dropout, name='drop_x')(x)
resh_W = Reshape((N, L, embedd_dim), name='resh_W')(drop_x)
z_fwd = TimeDistributed(LSTM(opts.lstm_units, return_sequences=True), name='z_fwd')(resh_W)
z_bwd = TimeDistributed(LSTM(opts.lstm_units, return_sequences=True, go_backwards=True), name='z_bwd')(resh_W)
z_merged = merge([z_fwd, z_bwd], mode='concat', name='z_merged')
avg_z = TimeDistributed(GlobalAveragePooling1D(), name='avg_z')(z_merged)
hz_fwd = LSTM(opts.lstm_units, return_sequences=True, name='hz_fwd')(avg_z)
hz_bwd = LSTM(opts.lstm_units, return_sequences=True, go_backwards=True, name='hz_bwd')(avg_z)
hz_merged = merge([hz_fwd, hz_bwd], mode='concat', name='hz_merged')
# avg_h = MeanOverTime(mask_zero=True, name='avg_h')(hz)
avg_hz = GlobalAveragePooling1D(name='avg_hz')(hz_merged)
y = Dense(output_dim=1, activation='sigmoid', name='output')(avg_hz)
model = Model(input=word_input, output=y)
if opts.init_bias and init_mean_value:
logger.info("Initialise output layer bias with log(y_mean/1-y_mean)")
bias_value = (np.log(init_mean_value) - np.log(1 - init_mean_value)).astype(K.floatx())
model.layers[-1].bias = bias_value
if verbose:
model.summary()
start_time = time.time()
model.compile(loss='mse', optimizer='rmsprop')
total_time = time.time() - start_time
logger.info("Model compiled in %.4f s" % total_time)
return model
def build_model(maxlen, vector_dim, dropout=0.4):
"""build model
Build the model according to the arguments.
# Arguments
maxlen: The max length of a sample
vector_dim: The size of token's dim
dropout : the rate of dropout
winlen : the length of average pooling windows
# Returns
model : model just build
"""
print('Build model...')
inputs = Input(shape=(maxlen, vector_dim))
mask_1 = Masking(mask_value=0.0, name='mask_1')(inputs)
bgru_1 = Bidirectional(GRU(units=512,
activation='tanh',
recurrent_activation='hard_sigmoid',
return_sequences=True),
name='bgru_1')(mask_1)
dropout_1 = Dropout(dropout, name='dropout_1')(bgru_1)
bgru_2 = Bidirectional(GRU(units=512,
activation='tanh',
recurrent_activation='hard_sigmoid',
return_sequences=True),
name='bgru_2')(dropout_1)
dropout_2 = Dropout(dropout, name='dropout_2')(bgru_2)
dense_1 = TimeDistributed(Dense(1), name='dense1')(dropout_2)
activation_1 = Activation('sigmoid', name='activation_1')(dense_1)
unmask_1 = NonMasking()(activation_1)
vulner_mask_input = Input(shape=(maxlen, maxlen), name='vulner_mask_input')
multiply_1 = Multiply(name='multiply_1')([vulner_mask_input, unmask_1])
reshape_1 = Reshape((1, maxlen**2))(multiply_1)
k_max_1 = KMaxPooling(k=1, name='k_max_1')(reshape_1)
average_1 = GlobalAveragePooling1D(name='average_1')(k_max_1)
print("begin compile")
model = Model(inputs=[inputs, vulner_mask_input], outputs=average_1)
model.compile(loss='binary_crossentropy',
optimizer='adamax',
metrics=[
'TP_count', 'FP_count', 'FN_count', 'precision',
'recall', 'fbeta_score'
])
model.summary()
return model
def build_model(self):
data = self.data
inputs = Input(shape=(data.seq_length, ))
embeddings = Embedding(
data.max_feature,
data.embed_dim,
weights=[data.embed_matrix],
trainable=False)(inputs)
x = SpatialDropout1D(self.dropout)(embeddings)
x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
# x = Bidirectional(CuDNNGRU(data.embed_dim, return_sequences=True))(x)
x = TimeDistributed(Dense(100, activation="relu"))(x)
attention = AttLayer()(x)
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
conc = concatenate([avg_pool, max_pool, attention])
encoder = Model(inputs=inputs, outputs=conc)
inputs2 = Input(shape=(self.max_sent, data.seq_length, ))
x2 = TimeDistributed(encoder)(inputs2)
x2 = Bidirectional(CuDNNGRU(100, return_sequences=True))(x2)
x2 = Bidirectional(CuDNNGRU(100, return_sequences=True))(x2)
# x2 = Bidirectional(CuDNNGRU(data.embed_dim, return_sequences=True))(x2)
x2 = TimeDistributed(Dense(100, activation="relu"))(x2)
attention2 = AttLayer()(x2)
avg_pool2 = GlobalAveragePooling1D()(x2)
max_pool2 = GlobalMaxPooling1D()(x2)
conc2 = concatenate([avg_pool2, max_pool2, attention2])
x2 = Dense(self.dense_size, activation="relu")(conc2)
outputs2 = Dense(6, activation="sigmoid")(x2)
model = Model(inputs=inputs2, outputs=outputs2)
optimizer = self.get_optimizer(self.lr, self.optim_name)
model.compile(
loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
self.model = model
def get_cnn_model(num_amino_acids, max_sequence_size, max_num_functions, target_function):
logging.info('Building CNN model using functional API ...')
logging.debug("Embedding dims = " + str(embedding_dims))
# no need to mention the num_amino_acids when using functional API
input = Input(shape=(max_sequence_size,))
embedding = Embedding(num_amino_acids, embedding_dims, input_length=max_sequence_size)(input)
x = Convolution1D(250, 15, activation='relu', subsample_length=1)(embedding)
x = Dropout(0.3)(x)
x = Convolution1D(100, 15, activation='relu', subsample_length=1)(x)
x = Dropout(0.3)(x)
# x = ZeroPadding1D((0,20))(x)
#
# z = Convolution1D(100, 5, subsample_length=1)(embedding)
# z = BatchNormalization()(z)
# z = Activation('sigmoid')(z)
# z = Dropout(0.5)(z)
#
# z = Convolution1D(50, 5, subsample_length=1)(z)
# z = BatchNormalization()(z)
# z = Activation('sigmoid')(z)
# z = Dropout(0.5)(z)
# residual connection
# x = merge([x, z], mode='concat')
x = GlobalAveragePooling1D()(x)
# x = Flatten()(x)
if target_function != '':
output_nodes = max_num_functions# 2
output_activation = 'sigmoid' # 'softmax'
else:
output_nodes = max_num_functions
output_activation = 'sigmoid'
x = Dense(output_nodes)(x)
x = BatchNormalization()(x) # can also try to do this after the activation (didn't work)
output = Activation(output_activation)(x)
model = Model([input], output)
model.summary()
return model
def BiGRU(X_train, y_train, X_test, y_test, gru_units, dense_units, input_shape, \
batch_size, epochs, drop_out, patience):
model = Sequential()
reg = L1L2(l1=0.2, l2=0.2)
model.add(
Bidirectional(GRU(units=gru_units,
dropout=drop_out,
activation='relu',
recurrent_regularizer=reg,
return_sequences=True),
input_shape=input_shape,
merge_mode="concat"))
model.add(BatchNormalization())
model.add(TimeDistributed(Dense(dense_units, activation='relu')))
model.add(BatchNormalization())
model.add(
Bidirectional(GRU(units=gru_units,
dropout=drop_out,
activation='relu',
recurrent_regularizer=reg,
return_sequences=True),
merge_mode="concat"))
model.add(BatchNormalization())
model.add(Dense(units=1))
model.add(GlobalAveragePooling1D())
print(model.summary())
early_stopping = EarlyStopping(monitor="val_loss", patience=patience)
model.compile(loss='mse', optimizer='adam')
#date_time = time.strftime('%Y-%m-%d-%H', time.localtime(time.time()))
#tensorboard = TensorBoard(log_dir="logs/{}".format(data_time))
#history_callback = model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, \
# verbose=2, callbacks=[tensorboard], validation_data=[X_test, y_test], shuffle=True)
history_callback = model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2, \
callbacks=[early_stopping], validation_data=[X_test, y_test], shuffle=True)
return model, history_callback
def cnn_melspect_urban_sound(input_shape):
kernel_size = 4
# activation_func = LeakyReLU()
activation_func = Activation('relu')
inputs = Input(input_shape)
# Convolutional block_1
conv1 = Conv1D(8, kernel_size)(inputs)
act1 = activation_func(conv1)
bn1 = BatchNormalization()(act1)
pool1 = MaxPooling1D(pool_size=2, strides=2)(bn1)
# Convolutional block_2
conv2 = Conv1D(16, kernel_size)(pool1)
act2 = activation_func(conv2)
bn2 = BatchNormalization()(act2)
pool2 = MaxPooling1D(pool_size=2, strides=2)(bn2)
# Convolutional block_3
conv3 = Conv1D(32, kernel_size)(pool2)
act3 = activation_func(conv3)
pool3 = BatchNormalization()(act3)
# Convolutional block_4
conv4 = Conv1D(64, kernel_size)(pool3)
act4 = activation_func(conv4)
bn4 = BatchNormalization()(act4)
# Global Layers
gmaxpl = GlobalMaxPooling1D()(bn4)
gmeanpl = GlobalAveragePooling1D()(bn4)
mergedlayer = concatenate([gmaxpl, gmeanpl], axis=1)
# Regular MLP
dense1 = Dense(512,
kernel_initializer='glorot_normal',
bias_initializer='glorot_normal')(mergedlayer)
actmlp = activation_func(dense1)
reg = Dropout(0.5)(actmlp)
dense2 = Dense(512,
kernel_initializer='glorot_normal',
bias_initializer='glorot_normal')(reg)
actmlp = activation_func(dense2)
reg = Dropout(0.5)(actmlp)
dense2 = Dense(10, activation='softmax')(reg)
model = Model(inputs=[inputs], outputs=[dense2])
return model, 'Kernel size 4 - 8/16/32/64'
def _createModel(self, outDim, vecLen):
model = Sequential()
#model.add(Masking(mask_value=0., input_shape=(self._phraseLen, vecLen)))
model.add(Embedding(vecLen, self._embeddingSize, input_length=self._phraseLen))
model.add(Bidirectional(LSTM(self._lstmCells, dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True)))
model.add(Bidirectional(LSTM(self._lstmCells, dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True)))
model.add(GlobalAveragePooling1D())
model.add(Dense(outDim, activation='relu'))
model.add(Dense(outDim, activation='relu'))
model.add(Dense(outDim, activation='softmax'))
opt = RMSprop(lr = self._learningRate)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
return model
def Conv1DQuora(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
conv1 = Conv1D(filters=128, kernel_size=1,
padding='same', activation='relu')
conv2 = Conv1D(filters=128, kernel_size=2,
padding='same', activation='relu')
conv3 = Conv1D(filters=128, kernel_size=3,
padding='same', activation='relu')
conv4 = Conv1D(filters=128, kernel_size=4,
padding='same', activation='relu')
conv5 = Conv1D(filters=32, kernel_size=5,
padding='same', activation='relu')
conv6 = Conv1D(filters=32, kernel_size=6,
padding='same', activation='relu')
conv1a = conv1(embed_input_)
glob1a = GlobalAveragePooling1D()(conv1a)
conv2a = conv2(embed_input_)
glob2a = GlobalAveragePooling1D()(conv2a)
conv3a = conv3(embed_input_)
glob3a = GlobalAveragePooling1D()(conv3a)
conv4a = conv4(embed_input_)
glob4a = GlobalAveragePooling1D()(conv4a)
conv5a = conv5(embed_input_)
glob5a = GlobalAveragePooling1D()(conv5a)
conv6a = conv6(embed_input_)
glob6a = GlobalAveragePooling1D()(conv6a)
merge_layer = concatenate([glob1a, glob2a, glob3a, glob4a, glob5a, glob6a])
x = Dropout(0.2)(merge_layer)
x = BatchNormalization()(x)
x = Dense(256)(x)
x = PReLU()(x)
x = Dropout(0.2)(x)
x = BatchNormalization()(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss='binary_crossentropy', optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def BuildModelKerasMn(self):
start = time.time()
model = Sequential()
model.add(
Bidirectional(LSTM(256, return_sequences=True),
input_shape=(self.n_steps, self.n_input)))
model.add(GlobalAveragePooling1D())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(10, activation='softmax'))
model.summary()
# compile
model.compile(optimizer=Adam(lr=self.learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0),
loss='categorical_crossentropy',
metrics=['accuracy'])
# data
no_of_samples = np.shape(self.X)[0]
step = 1
Loss = []
Elapsed = []
model.fit(self.X,
self.Y,
epochs=self.training_iters,
batch_size=self.batch_size,
shuffle=True,
validation_data=(self.X, self.Y))
# Calculate accuracy for test data
score = model.evaluate(self.testX, self.testY)
# Save the model
file_suffix = str(self.n_steps) + '_segments_' + str(self.n_input)
model_name = 'model_' + file_suffix + '.h5'
model.save(model_name)
print("Testing Loss:", score[0])
print("Testing Accuracy:", score[1])
elapsed = time.time() - start
"""
if self.deg_of_logsig==0:
plt.plot(Elapsed, Loss, label='Time series')
else:
plt.plot(Elapsed, Loss, label='degree %d' %self.deg_of_logsig)
"""
return {'Accuracy': score[1], 'Time': elapsed, 'NStep': step}
def create_conv1d_model(X_train):
"""
"""
model = Sequential()
# example here: https://gist.github.com/jkleint/1d878d0401b28b281eb75016ed29f2ee
model.add(
Conv1D(64,
30,
strides=1,
padding='valid',
kernel_initializer='glorot_normal',
activation=None,
input_shape=(X_train.shape[1], 1)))
model.add(BatchNormalization())
model.add(LeakyReLU())
# https://github.com/fchollet/keras/issues/4403 note on TimeDistributed
model.add(MaxPooling1D(pool_size=2, strides=2, padding='valid'))
model.add(
Conv1D(256,
30,
strides=1,
padding='valid',
kernel_initializer='glorot_normal',
activation=None,
input_shape=(X_train.shape[1], 1)))
model.add(BatchNormalization())
model.add(LeakyReLU())
# https://github.com/fchollet/keras/issues/4403 note on TimeDistributed
model.add(MaxPooling1D(pool_size=2, strides=2, padding='valid'))
# model.add(Flatten()) # dimensions were too big with this
model.add(GlobalAveragePooling1D())
model.add(Dense(256, kernel_initializer='glorot_normal'))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dropout(0.5))
model.add(Dense(128, kernel_initializer='glorot_normal'))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dropout(0.5))
model.add(Dense(1))
# build model using keras documentation recommended optimizer initialization
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
# compile the model
model.compile(loss='mean_squared_error', optimizer=optimizer)
return model
def build_model():
seq_input1 = Input(shape=(seq_size, dim), name='seq1')
seq_input2 = Input(shape=(seq_size, dim), name='seq2')
l1 = Conv1D(hidden_dim, 3)
r1 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True))
l2 = Conv1D(hidden_dim, 3)
r2 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True))
l3 = Conv1D(hidden_dim, 3)
r3 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True))
l4 = Conv1D(hidden_dim, 3)
r4 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True))
l5 = Conv1D(hidden_dim, 3)
r5 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True))
l6 = Conv1D(hidden_dim, 3)
s1 = MaxPooling1D(2)(l1(seq_input1))
s1 = concatenate([r1(s1), s1])
s1 = MaxPooling1D(2)(l2(s1))
s1 = concatenate([r2(s1), s1])
s1 = MaxPooling1D(3)(l3(s1))
s1 = concatenate([r3(s1), s1])
s1 = l6(s1)
s1 = GlobalAveragePooling1D()(s1)
s2 = MaxPooling1D(2)(l1(seq_input2))
s2 = concatenate([r1(s2), s2])
s2 = MaxPooling1D(2)(l2(s2))
s2 = concatenate([r2(s2), s2])
s2 = MaxPooling1D(3)(l3(s2))
s2 = concatenate([r3(s2), s2])
s2 = l6(s2)
s2 = GlobalAveragePooling1D()(s2)
merge_text = multiply([s1, s2])
x = Dense(100, activation='linear')(merge_text)
x = keras.layers.LeakyReLU(alpha=0.3)(x)
main_output = Dense(1, activation='sigmoid')(x)
merge_model = Model(inputs=[seq_input1, seq_input2], outputs=[main_output])
return merge_model
def conv_filt_v2(input_a, no_filters, filt_width):
''' base module for text data
Inputs: input_a -- input tensor
no_filters -- number of filter maps used in conv layer (details)
filt_width -- convolution window width
outputs: output tensor of base module
'''
# 20000 is the vocabulary size. It should be greater than or equal to your tokenizer vocabulary size
embed = Embedding(vocab_size, embed_size)(input_a)
conv = Conv1D(no_filters, filt_width, strides=1)(embed)
conv = Activation('relu')(conv)
conv = AveragePooling1D(pool_size=2)(conv)
conv = Dropout(0.5)(conv)
conv = GlobalAveragePooling1D()(conv)
return conv
def initialize_net(train_params):
act = "relu"
model = Sequential()
# Layer 1
model.add(Conv1D(nb_filter=5,
filter_length=10,
init='glorot_uniform',
border_mode='same',
input_shape=(train_params['max_size'], 3),
bias=True))
model.add(Activation(act))
model.add(MaxPooling1D(pool_size=2))
# Layer 2
model.add(Conv1D(nb_filter=10,
filter_length=20,
init='glorot_uniform',
border_mode='same',
bias=True))
model.add(Activation(act))
model.add(MaxPooling1D(pool_size=2))
# Layer 3
model.add(Conv1D(nb_filter=20,
filter_length=20,
init='glorot_uniform',
border_mode='same',
bias=True))
model.add(Activation(act))
model.add(MaxPooling1D(pool_size=2))
model.add(GlobalAveragePooling1D(input_shape=model.output_shape[1:]))
model.add(Dense(input_dim=20,
output_dim=2,
init='glorot_uniform'))
model.add(Activation(act))
model.add(Dropout(0.3))
model.add(Dense(input_dim=2,
output_dim=2,
init='glorot_uniform'))
model.add(Activation('softmax'))
return model
def LSTMdeep2(params):
embed_dropout_rate = 0.1
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
x = Activation('tanh')(embed_input_)
x = SpatialDropout1D(embed_dropout_rate, name='embed_drop')(x)
if params['bidirectional']:
x = Bidirectional(
CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform'))(x)
x = Bidirectional(
CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform'))(x)
x1 = GlobalMaxPooling1D()(x)
x2 = GlobalAveragePooling1D()(x)
x3 = AttentionWithContext()(x)
else:
x = CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform')(x)
x = CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform')(x)
x = GlobalMaxPooling1D()(x)
merge_layer = concatenate([x1, x2, x3])
x = Dropout(params['dropout_rate'])(merge_layer)
x = Dense(256)(x)
x = PReLU()(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def build_model(opts,
overall_maxlen,
vocab_size=0,
embedd_dim=50,
embedding_weights=None,
verbose=True,
init_mean_value=None):
word_input = Input(shape=(overall_maxlen, ),
dtype='int32',
name='word_input')
x = Embedding(output_dim=embedd_dim,
input_dim=vocab_size,
input_length=overall_maxlen,
weights=[embedding_weights],
mask_zero=True,
name='x')(word_input)
x_maskedout = ZeroMaskedEntries(name='x_maskedout')(x)
drop_x = Dropout(opts.dropout, name='drop_x')(x_maskedout)
position_embedding = Position_Embedding(name='position_embedding')(drop_x)
self_att = MutilHeadAttention(nb_head=8, size_per_head=16,
name='self_att')(position_embedding)
lstm = LSTM(opts.rnn_dim, return_sequences=True, name='lstm')(self_att)
avg_pooling = GlobalAveragePooling1D(name='avg_pooling')(lstm)
y = Dense(output_dim=1,
activation='sigmoid',
name='y',
W_regularizer=l2(opts.l2_value))(avg_pooling)
model = Model(input=[word_input], output=y)
if opts.init_bias and init_mean_value:
bias_value = (np.log(init_mean_value) -
np.log(1 - init_mean_value)).astype(K.floatx())
model.layers[-1].b.set_value(bias_value)
if verbose:
model.summary()
optimize = optimizers.rmsprop(lr=0.001)
model.compile(loss='mse', optimizer=optimize)
return model
def fGlove_avg(MAX_NB_WORDS, MAX_WORDS, MAX_SENTS, EMBEDDING_DIM, WORDGRU, embedding_matrix, DROPOUTPER):
wordInputs = Input(shape=(MAX_SENTS+1, MAX_WORDS), name="wordInputs", dtype='float32')
wordInp = Flatten()(wordInputs)
wordEmbedding = Embedding(MAX_NB_WORDS, EMBEDDING_DIM, weights=[embedding_matrix], mask_zero=False, trainable=True, name='wordEmbedding')(wordInp)
head = GlobalAveragePooling1D()(wordEmbedding)
v6 = Dense(1, activation="sigmoid", name="dense")(head)
model = Model(inputs=[wordInputs] , outputs=[v6])
model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
return model
def build_model(opts, vocab_size=0, maxnum=50, maxlen=50, embedd_dim=50, embedding_weights=None, verbose=False, init_mean_value=None):
N = maxnum
L = maxlen
p = Input(shape=(4, 2048), dtype='float32', name='p')
# img_vector = Dense(name='img_vector', units=128)(p)
word_input = Input(shape=(N * L,), dtype='int32', name='word_input')
x = Embedding(output_dim=embedd_dim, input_dim=vocab_size, input_length=N * L, weights=embedding_weights,
mask_zero=True, trainable=False, name='x')(word_input)
x_maskedout = ZeroMaskedEntries(name='x_maskedout')(x)
drop_x = Dropout(opts.dropout, name='drop_x')(x_maskedout)
resh_W = Reshape((N, L, embedd_dim), name='resh_W')(drop_x)
cnn_e = TimeDistributed(Conv1D(opts.nbfilters, opts.filter1_len, border_mode='valid'), name='cnn_e')(resh_W)
att_cnn_e = TimeDistributed(Attention(), name='att_cnn_e')(cnn_e)
lstm_e = LSTM(opts.lstm_units, return_sequences=True, name='lstm_e')(att_cnn_e)
G = CoAttention(name='essay')([lstm_e, p])
avg = GlobalAveragePooling1D()(G)
final_vec_drop = Dropout(rate=0.5, name='final_vec_drop')(avg)
if opts.l2_value:
logger.info("Use l2 regularizers, l2 value = %s" % opts.l2_value)
y = Dense(units=1, activation='sigmoid', name='output', W_regularizer=l2(opts.l2_value))(final_vec_drop)
else:
y = Dense(units=1, activation='sigmoid', name='output')(final_vec_drop)
model = Model(input=[word_input, p], output=y)
if opts.init_bias and init_mean_value:
logger.info("Initialise output layer bias with log(y_mean/1-y_mean)")
bias_value = (np.log(init_mean_value) - np.log(1 - init_mean_value)).astype(K.floatx())
model.layers[-1].b.set_value(bias_value
)
if verbose:
model.summary()
start_time = time.time()
model.compile(loss='mse', optimizer='adam')
total_time = time.time() - start_time
logger.info("Model compiled in %.4f s" % total_time)
return model
def _build_network(self,
vocab_size,
maxlen,
embedding_dimension=256,
hidden_units=256,
trainable=False):
print('Build model...')
model = Sequential()
model.add(
Embedding(vocab_size,
embedding_dimension,
input_length=maxlen,
embeddings_initializer='glorot_normal'))
model.add(
Convolution1D(hidden_units,
2,
kernel_initializer='he_normal',
padding='valid',
activation='sigmoid'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.25))
model.add(
LSTM(hidden_units,
kernel_initializer='he_normal',
activation='sigmoid',
dropout=0.5,
recurrent_activation=0.5,
unroll=True,
return_sequences=True))
model.add(GlobalAveragePooling1D())
model.add(Dropout(0.5))
model.add(Dense(2))
model.add(Activation('softmax'))
adam = Adam(lr=0.001)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
print('No of parameter:', model.count_params())
print(model.summary())
return model
def cnn_model_max_and_aver_pool(params):
if type(params['kernel_sizes_cnn']) is str:
params['kernel_sizes_cnn'] = [
int(x) for x in params['kernel_sizes_cnn'].split(' ')
]
inp = Input(shape=(params['text_size'], params['embedding_size']))
outputs = []
for i in range(len(params['kernel_sizes_cnn'])):
output_i = Conv1D(params['filters_cnn'],
kernel_size=params['kernel_sizes_cnn'][i],
activation=None,
kernel_regularizer=l2(params['coef_reg_cnn']),
padding='same')(inp)
output_i = BatchNormalization()(output_i)
output_i = Activation('relu')(output_i)
output_i_0 = GlobalMaxPooling1D()(output_i)
output_i_1 = GlobalAveragePooling1D()(output_i)
output_i = Concatenate()([output_i_0, output_i_1])
outputs.append(output_i)
output = concatenate(outputs, axis=1)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(params['dense_size'],
activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
output = BatchNormalization()(output)
output = Activation('relu')(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(nb_classes,
activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
output = BatchNormalization()(output)
act_output = Activation(params.get("last_layer_activation",
"sigmoid"))(output)
model = Model(inputs=inp, outputs=act_output)
model.compile(
optimizer="Adam",
loss="binary_crossentropy",
metrics=['accuracy'],
#loss_weights=loss_weights,
#sample_weight_mode=sample_weight_mode,
# weighted_metrics=weighted_metrics,
# target_tensors=target_tensors
)
return model
def get_model(num_users, num_items, layers=[20, 10], reg_layers=[0, 0]):
assert len(layers) == len(reg_layers)
num_layer = len(layers) # Number of layers in the MLP
# Input variables
user_input = Input(shape=(1, ), dtype='int32', name='user_input')
item_input = Input(shape=(1, ), dtype='int32', name='item_input')
MLP_Embedding_User = Embedding(
input_dim=num_users,
output_dim=50,
name='user_embedding',
embeddings_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.05,
seed=None),
embeddings_regularizer=l2(reg_layers[0]),
input_length=1)
MLP_Embedding_Item = Embedding(
input_dim=num_items,
output_dim=50,
name='item_embedding',
embeddings_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.05,
seed=None),
embeddings_regularizer=l2(reg_layers[0]),
input_length=1)
# Crucial to flatten an embedding vector!
# user_latent = Flatten()(MLP_Embedding_User(user_input))
# item_latent = Flatten()(MLP_Embedding_Item(item_input))
user_latent = MLP_Embedding_User(user_input)
item_latent = MLP_Embedding_Item(item_input)
user_latent = Reshape((10, 5))(user_latent)
item_latent = Reshape((10, 5))(item_latent)
# The 0-th layer is the concatenation of embedding layers
vector = concatenate([user_latent, item_latent], axis=1)
#vector = K.permute_dimensions(vector, (0, 2, 1))
O_seq = Attention(32, 32)([vector, vector, vector])
# O_seq = Dense(20,activation='Relu')(O_seq)
O_seq = GlobalAveragePooling1D()(O_seq)
O_seq = Dropout(0.5)(O_seq)
outputs = Dense(1, activation='sigmoid')(O_seq)
model = Model(inputs=[user_input, item_input], outputs=outputs)
return model