def __DenseNet(
inp, #input_shape=None,
dense_blocks=3,
dense_layers=-1,
growth_rate=12,
nb_classes=None,
dropout_rate=None,
bottleneck=False,
compression=1.0,
weight_decay=1e-4,
depth=40):
"""
Creating a DenseNet
Arguments:
input_shape : shape of the input images. E.g. (28,28,1) for MNIST
dense_blocks : amount of dense blocks that will be created (default: 3)
dense_layers : number of layers in each dense block. You can also use a list for numbers of layers [2,4,3]
or define only 2 to add 2 layers at all dense blocks. -1 means that dense_layers will be calculated
by the given depth (default: -1)
growth_rate : number of filters to add per dense block (default: 12)
nb_classes : number of classes
dropout_rate : defines the dropout rate that is accomplished after each conv layer (except the first one).
In the paper the authors recommend a dropout of 0.2 (default: None)
bottleneck : (True / False) if true it will be added in convolution block (default: False)
compression : reduce the number of feature-maps at transition layer. In the paper the authors recomment a compression
of 0.5 (default: 1.0 - will have no compression effect)
weight_decay : weight decay of L2 regularization on weights (default: 1e-4)
depth : number or layers (default: 40)
Returns:
Model : A Keras model instance
"""
if nb_classes == None:
raise Exception(
'Please define number of classes (e.g. num_classes=10). This is required for final softmax.'
)
if compression <= 0.0 or compression > 1.0:
raise Exception(
'Compression have to be a value between 0.0 and 1.0.')
if type(dense_layers) is list:
if len(dense_layers) != dense_blocks:
raise AssertionError(
'Number of dense blocks have to be same length to specified layers'
)
elif dense_layers == -1:
dense_layers = int((depth - 4) / 3)
if bottleneck:
dense_layers = int(dense_layers / 2)
dense_layers = [dense_layers for _ in range(dense_blocks)]
else:
dense_layers = [dense_layers for _ in range(dense_blocks)]
#img_input = Input(shape=input_shape)
nb_channels = growth_rate
#print('Creating DenseNet %s' % __version__)
#print('#############################################')
#print('Dense blocks: %s' % dense_blocks)
#print('Layers per dense block: %s' % dense_layers)
#print('#############################################')
# Initial convolution layer
#x = Convolution2D(filters=2 * growth_rate, kernel_size=(3,3), padding='same',strides=(1,1),
# use_bias=False, kernel_regularizer=l2(weight_decay))(img_input)
x = Convolution1D(filters=growth_rate,
kernel_size=32,
padding='same',
strides=1,
use_bias=False,
kernel_regularizer=l2(weight_decay))(
inp) #(img_input)
# Building dense blocks
for block in range(dense_blocks - 1):
#print("block:::",block)
# Add dense block
x, nb_channels = Net.__dense_block(x, dense_layers[block],
nb_channels, growth_rate,
dropout_rate, bottleneck,
weight_decay)
# Add transition_block
x = Net.__transition_layer(x, nb_channels, dropout_rate,
compression, weight_decay)
nb_channels = int(nb_channels * compression)
# Add last dense block without transition but for that with global average pooling
x, nb_channels = Net.__dense_block(x, dense_layers[-1], nb_channels,
growth_rate, dropout_rate,
weight_decay)
x = BatchNormalization()(x)
x = Activation('relu')(
x) #LeakyReLU(alpha=0.2)(x)#Activation('relu')(x)
#x = GlobalAveragePooling2D()(x)
#x = GlobalAveragePooling1D()(x)
x = AveragePooling1D(int(x.shape[1]), int(x.shape[1]))(x)
#x = Dense(nb_classes, activation='softmax')(x)
return x #Model(img_input, x, name='densenet')
model2.add(TimeDistributed(Dense(dim, activation='relu')))
model2.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(dim, )))
#===============
model3 = Sequential()
model3.add(
Embedding(len(word_index) + 1,
dim,
weights=[embedding_matrix],
input_length=max_len,
trainable=False))
model3.add(
Convolution1D(filters=nb_filter,
kernel_size=filter_length,
padding='valid',
activation='relu',
strides=1))
model3.add(Dropout(0.2))
model3.add(
Convolution1D(filters=nb_filter,
kernel_size=filter_length,
padding='valid',
activation='relu',
strides=1))
model3.add(GlobalMaxPooling1D())
model3.add(Dropout(0.2))
model3.add(Dense(dim))
def trian_cnn():
# set parameters:
max_features = 20000
maxlen = 8
batch_size = 32
embedding_dims = 50
nb_filter = 250
filter_length = 3
hidden_dims = 250
nb_epoch = 2
print('Loading data...')
X_train, y_train, X_test, y_test = load_data()
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')
print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen, value=3)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen, value=3)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
y_train = np.array(y_train)
print('y_train shape', y_train.shape)
y_test = np.array(y_test)
print('y_train shape', y_test.shape)
print('Build model...')
model = Sequential()
model.add(
Embedding(max_features,
embedding_dims,
input_length=maxlen,
dropout=0.2))
# we add a Convolution1D, which will learn nb_filter
# word group filters of size filter_length:
model.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
# we use max pooling:
model.add(GlobalMaxPooling1D())
# We add a vanilla hidden layer:
model.add(Dense(hidden_dims))
model.add(Dropout(0.2))
model.add(Activation('relu'))
# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)
#save the model &serialize model to JSON
model_json = model.to_json()
with open("./dict/model_cnn.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("./dict/model_cnn_weights.h5")
print("Saved model to disk")
del model
print('Test score:', score)
print('Test accuracy:', acc)
def default_model(input_dim, useBatchNorm):
reg = l2(0.0001)
model = Sequential()
model.add(
InputLayer(input_shape=(input_dim[0], input_dim[1], input_dim[2])))
model.add(Reshape((input_dim[0], input_dim[2])))
model.add(
Convolution1D(filters=8,
kernel_size=7,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=16,
kernel_size=5,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=32,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=32,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=64,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=64,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=128,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=256,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(Flatten())
#
#
model.add(Dense(units=512, kernel_regularizer=reg, bias_regularizer=reg))
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(Dense(units=128, kernel_regularizer=reg, bias_regularizer=reg))
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
return model
scaler = Normalizer().fit(T)
testT = scaler.transform(T)
y_train = np.array(Y)
y_test = np.array(C)
X_train = np.reshape(trainX, (trainX.shape[0], trainX.shape[1], 1))
X_test = np.reshape(testT, (testT.shape[0], testT.shape[1], 1))
lstm_output_size = 128
cnn = Sequential()
cnn.add(
Convolution1D(64,
3,
border_mode="same",
activation="relu",
input_shape=(43, 1)))
cnn.add(Convolution1D(64, 3, border_mode="same", activation="relu"))
cnn.add(MaxPooling1D(pool_length=(2)))
cnn.add(Convolution1D(128, 3, border_mode="same", activation="relu"))
cnn.add(Convolution1D(128, 3, border_mode="same", activation="relu"))
cnn.add(MaxPooling1D(pool_length=(2)))
cnn.add(Flatten())
cnn.add(Dense(128, activation="relu"))
cnn.add(Dropout(0.5))
cnn.add(Dense(1, activation="sigmoid"))
# define optimizer and objective, compile cnn
cnn.compile(loss="binary_crossentropy", optimizer="adam", metrics=['accuracy'])
def build(self):
assert self.config['question_len'] == self.config['answer_len']
question = self.question
answer = self.get_answer()
# add embedding layers
embedding = Embedding(self.config['n_words'],
self.model_params.get('n_embed_dims', 100))
question_embedding = embedding(question)
answer_embedding = embedding(answer)
# turn off layer updating
embedding.params = []
embedding.updates = []
# dropout
dropout = Dropout(0.25)
question_dropout = dropout(question_embedding)
answer_dropout = dropout(answer_embedding)
# dense
dense = TimeDistributed(
Dense(self.model_params.get('n_hidden', 200), activation='tanh'))
question_dense = dense(question_dropout)
answer_dense = dense(answer_dropout)
# regularization
question_dense = ActivityRegularization(l2=0.0001)(question_dense)
answer_dense = ActivityRegularization(l2=0.0001)(answer_dense)
# dropout
question_dropout = dropout(question_dense)
answer_dropout = dropout(answer_dense)
# cnn
cnns = [
Convolution1D(filter_length=filter_length,
nb_filter=self.model_params.get('nb_filters', 1000),
activation=self.model_params.get(
'conv_activation', 'relu'),
border_mode='same')
for filter_length in [2, 3, 5, 7]
]
question_cnn = merge([cnn(question_dropout) for cnn in cnns],
mode='concat')
answer_cnn = merge([cnn(answer_dropout) for cnn in cnns],
mode='concat')
# regularization
question_cnn = ActivityRegularization(l2=0.0001)(question_cnn)
answer_cnn = ActivityRegularization(l2=0.0001)(answer_cnn)
# dropout
question_dropout = dropout(question_cnn)
answer_dropout = dropout(answer_cnn)
# maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False),
output_shape=lambda x: (x[0], x[2]))
question_pool = maxpool(question_dropout)
answer_pool = maxpool(answer_dropout)
# activation
activation = Activation('tanh')
question_output = activation(question_pool)
answer_output = activation(answer_pool)
return question_output, answer_output
def seq_attentionmodel(enhancer_length=600,
promoter_length=400,
n_kernels=256,
filter_length=8,
dense_layer_size=800):
opt = Adam(lr=1e-5)
input_enh_sequence = Input(shape=(enhancer_length, 4))
enh_seq = Convolution1D(input_dim=4,
input_length=enhancer_length,
nb_filter=n_kernels,
filter_length=filter_length,
border_mode="valid",
subsample_length=1,
kernel_regularizer=l2(2e-5))(input_enh_sequence)
enh_seq = relu(enh_seq)
enh_seq = MaxPooling1D(pool_length=int(filter_length / 2),
stride=int(filter_length / 2))(enh_seq)
input_prom_sequence = Input(shape=(promoter_length, 4))
prom_seq = Convolution1D(input_dim=4,
input_length=promoter_length,
nb_filter=n_kernels,
filter_length=filter_length,
border_mode="valid",
subsample_length=1,
kernel_regularizer=l2(2e-5))(input_prom_sequence)
#prom_seq = BatchNormalization(momentum=0.997)(prom_seq)
prom_seq = relu(prom_seq)
prom_seq = MaxPooling1D(pool_length=int(filter_length / 2),
stride=int(filter_length / 2))(prom_seq)
seq_mixed = Concatenate(axis=1)([enh_seq, prom_seq])
seq_mixed = BatchNormalization(momentum=0.997)(seq_mixed)
seq_mixed = Dropout(0.2)(seq_mixed)
seq_mixed = Flatten()(seq_mixed)
seq_mixed = Dense(output_dim=dense_layer_size,
init="glorot_uniform",
activity_regularizer=l2(1e-6))(seq_mixed)
seq_mixed = BatchNormalization(momentum=0.997)(seq_mixed)
seq_mixed = relu(seq_mixed)
a_prob = Dense(dense_layer_size,
activation='softmax',
name='attention_vec',
kernel_regularizer=l2(weightDecay))(seq_mixed)
attention_mul = multiply([seq_mixed, a_prob], name='attention_mul')
seq_mixed = Dense(256, kernel_regularizer=l2(weightDecay))(attention_mul)
a_prob = Dense(256,
activation='softmax',
name='attention_vec1',
kernel_regularizer=l2(weightDecay / 10))(seq_mixed)
attention_mul = multiply([seq_mixed, a_prob], name='attention_mul1')
attention_mul = Dense(128, kernel_regularizer=l2(weightDecay /
10))(attention_mul)
attention_mul = BatchNormalization(momentum=0.997)(attention_mul)
attention_mul = relu(attention_mul)
#seq_mixed = Dropout(0.5)(seq_mixed)
seq_mixed = Dense(1)(attention_mul)
seq_mixed = BatchNormalization(momentum=0.997, scale=False)(seq_mixed)
seq_mixed = Activation('sigmoid')(seq_mixed)
model = Model([input_enh_sequence, input_prom_sequence], seq_mixed)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[f1])
model.summary()
return model
return (input_shape[0], (input_shape[2] * self.k))
def call(self, inputs):
# swap last two dimensions since top_k will be applied along the last dimension
shifted_input = tf.transpose(inputs, [0, 2, 1])
# extract top_k, returns two tensors [values, indices]
top_k = tf.nn.top_k(shifted_input, k=self.k, sorted=True, name=None)[0]
# return flattened output
return Flatten()(top_k)
main_input = Input(shape=(MAX_LENGTH, ), dtype='float64')
embed = Embedding(353717 + 1, 378, input_length=MAX_LENGTH)(main_input)
cnn = Convolution1D(256, 3, padding='same', strides=1,
activation='relu')(embed)
cnn = MaxPool1D(pool_size=4)(cnn)
cnn = Flatten()(cnn)
cnn = Dense(256)(cnn)
rnn = Bidirectional(GRU(256, dropout=0.2, recurrent_dropout=0.1))(embed)
#rnn = Bidirectional(keras.layers.CuDNNGRU(256))(embed)
rnn = Dense(256)(rnn)
con = concatenate([cnn, rnn], axis=-1)
main_output = Dense(125, activation='softmax')(con)
model = Model(inputs=main_input, outputs=main_output)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=[f1])
history = model.fit(x=x_train,
y=y_train,
batch_size=32,
def CLSTMWordEmbed(nb_labels,
nb_filters=1200,
n_gram=2,
maxlen=15,
vecsize=300,
cnn_dropout=0.0,
nb_rnnoutdim=1200,
rnn_dropout=0.2,
final_activation='softmax',
dense_wl2reg=0.0,
optimizer='adam'):
""" Returns the C-LSTM neural networks for word-embedded vectors.
Reference: Chunting Zhou, Chonglin Sun, Zhiyuan Liu, Francis Lau,
"A C-LSTM Neural Network for Text Classification,"
(arXiv:1511.08630). [`arXiv
<https://arxiv.org/abs/1511.08630>`_]
:param nb_labels: number of class labels
:param nb_filters: number of filters (Default: 1200)
:param n_gram: n-gram, or window size of CNN/ConvNet (Default: 2)
:param maxlen: maximum number of words in a sentence (Default: 15)
:param vecsize: length of the embedded vectors in the model (Default: 300)
:param cnn_dropout: dropout rate for CNN/ConvNet (Default: 0.0)
:param nb_rnnoutdim: output dimension for the LSTM networks (Default: 1200)
:param rnn_dropout: dropout rate for LSTM (Default: 0.2)
:param final_activation: activation function. Options: softplus, softsign, relu, tanh, sigmoid, hard_sigmoid, linear. (Default: 'softmax')
:param dense_wl2reg: L2 regularization coefficient (Default: 0.0)
:param optimizer: optimizer for gradient descent. Options: sgd, rmsprop, adagrad, adadelta, adam, adamax, nadam. (Default: adam)
:return: keras sequantial model for CNN/ConvNet for Word-Embeddings
:type nb_labels: int
:type nb_filters: int
:type n_gram: int
:type maxlen: int
:type vecsize: int
:type cnn_dropout: float
:type nb_rnnoutdim: int
:type rnn_dropout: float
:type final_activation: str
:type dense_wl2reg: float
:type optimizer: str
:rtype: keras.model.Sequential
"""
model = Sequential()
model.add(Convolution1D(nb_filter=nb_filters,
filter_length=n_gram,
border_mode='valid',
activation='relu',
input_shape=(maxlen, vecsize)))
if cnn_dropout > 0.0:
model.add(Dropout(cnn_dropout))
model.add(MaxPooling1D(pool_length=maxlen - n_gram + 1))
model.add(LSTM(nb_rnnoutdim))
if rnn_dropout > 0.0:
model.add(Dropout(rnn_dropout))
model.add(Dense(nb_labels,
activation=final_activation,
W_regularizer=l2(dense_wl2reg)
)
)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model
def _model_constructor(self):
########################################
## sample train/validation data
########################################
data_1, data_2, labels, _, _, _ = self._preprocess_data()
perm = np.random.permutation(len(data_1))
idx_train = perm[:int(len(data_1) * (1 - self.VALIDATION_SPLIT))]
idx_val = perm[int(len(data_1) * (1 - self.VALIDATION_SPLIT)):]
# data_1_train = np.vstack((data_1[idx_train], data_2[idx_train]))
# data_2_train = np.vstack((data_2[idx_train], data_1[idx_train]))
data_1_train = np.vstack((data_1[idx_train], data_2[idx_train]))
data_2_train = np.vstack((data_2[idx_train], data_1[idx_train]))
self.train_custom_features = self.train_custom_features.as_matrix()
features_train = np.vstack((self.train_custom_features[idx_train],
self.train_custom_features[idx_train]))
labels_train = np.concatenate((labels[idx_train], labels[idx_train]))
data_1_val = np.vstack((data_1[idx_val], data_2[idx_val]))
data_2_val = np.vstack((data_2[idx_val], data_1[idx_val]))
features_val = np.vstack((self.train_custom_features[idx_val],
self.train_custom_features[idx_val]))
labels_val = np.concatenate((labels[idx_val], labels[idx_val]))
weight_val = np.ones(len(labels_val))
if self.REWEIGH:
weight_val *= 0.472001959
weight_val[labels_val == 0] = 1.309028344
########################################
## define the model structure
########################################
embedding_layer = self._create_embedding_layer()
# lstm_layer = LSTM(self.NUM_LSTM,
# dropout=self.RATE_DROP_LSTM,
# recurrent_dropout=self.RATE_DROP_LSTM)
convolution_layer = Convolution1D(nb_filter=NB_FILTER,
filter_length=FILTER_LENGTH,
border_mode='valid',
subsample_length=1)
custom_dim = int(self.train_custom_features.shape[1])
custom_input = Input(shape=(custom_dim, ), dtype='float32')
#custom_features = Dropout(self.RATE_DROP_DENSE)(custom_input)
custom_features = Dense(self.NUM_DENSE,
kernel_initializer='normal')(custom_input)
custom_features = PReLU()(custom_features)
custom_features = Dropout(self.RATE_DROP_DENSE)(custom_features)
custom_features = BatchNormalization()(custom_features)
sequence_1_input = Input(shape=(self.MAX_SEQUENCE_LENGTH, ),
dtype='int32')
embedded_sequences_1 = embedding_layer(sequence_1_input)
# Q1 = lstm_layer(embedded_sequences_1)
Q1 = convolution_layer(embedded_sequences_1)
Q1 = PReLU()(Q1)
Q1 = GlobalMaxPooling1D()(Q1)
Q1 = Dropout(self.RATE_DROP_DENSE)(Q1)
sequence_2_input = Input(shape=(self.MAX_SEQUENCE_LENGTH, ),
dtype='int32')
embedded_sequences_2 = embedding_layer(sequence_2_input)
#Q2 = lstm_layer(embedded_sequences_2)
Q2 = convolution_layer(embedded_sequences_2)
Q2 = PReLU()(Q2)
Q2 = GlobalMaxPooling1D()(Q2)
Q2 = Dropout(self.RATE_DROP_DENSE)(Q2)
# Q2 = convolution_layer(embedded_sequences_2)
# # Q2 = Dropout(self.RATE_DROP_DENSE)(Q2)
# # Q2 = convolution_layer(Q2)
# Q2 = GlobalMaxPooling1D()(Q2)
# Q2 = Dropout(self.RATE_DROP_DENSE)(Q2)
merged = concatenate([Q1, Q2, custom_features])
merged = BatchNormalization()(merged)
merged = Dense(self.NUM_DENSE, kernel_initializer='normal')(merged)
merged = PReLU()(merged)
merged = Dropout(self.RATE_DROP_DENSE)(merged)
merged = BatchNormalization()(merged)
merged = Dense(round(0.9 * self.NUM_DENSE),
kernel_initializer='normal')(merged)
merged = PReLU()(merged)
merged = Dropout(self.RATE_DROP_DENSE)(merged)
merged = BatchNormalization()(merged)
merged = Dense(round(0.7 * self.NUM_DENSE),
kernel_initializer='normal')(merged)
merged = PReLU()(merged)
merged = Dropout(self.RATE_DROP_DENSE)(merged)
merged = BatchNormalization()(merged)
merged = Dense(round(0.5 * self.NUM_DENSE),
kernel_initializer='normal')(merged)
merged = PReLU()(merged)
merged = Dropout(self.RATE_DROP_DENSE)(merged)
merged = BatchNormalization()(merged)
preds = Dense(1, activation='sigmoid')(merged)
########################################
## construct the model
########################################
model = Model(inputs=[sequence_1_input,
sequence_2_input,
custom_input], \
outputs=preds)
adam = optimizers.Adam(clipnorm=1.)
model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['acc'])
#model.summary()
print("The model {} is built.".format(self.STAMP))
########################################
## add class weight
########################################
if self.REWEIGH:
class_weight = {0: 1.309028344, 1: 0.472001959}
else:
class_weight = None
early_stopping = EarlyStopping(monitor='val_loss', patience=3)
bst_model_path = self.STAMP + '.h5'
model_checkpoint = ModelCheckpoint(bst_model_path,
save_best_only=True,
save_weights_only=True)
hist = model.fit(
[data_1_train, data_2_train, features_train],
labels_train,
validation_data=([data_1_val, data_2_val,
features_val], labels_val, weight_val),
epochs=10,
batch_size=512,
shuffle=True,
class_weight=class_weight,
callbacks=[early_stopping, model_checkpoint])
model.load_weights(bst_model_path)
bst_val_score = min(hist.history['val_loss'])
return (model, hist, bst_val_score)
model = Sequential()
# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
# 先从一个高效的嵌入层开始，它将词汇的索引值映射为 embedding_dims 维度的词向量
model.add(Embedding(len(abc), embedding_dims, input_length=maxlen,
dropout=0.5))
# we add a Convolution1D, which will learn nb_filter
# word group filters of size filter_length:
# 添加一个 1D 卷积层，它将学习 nb_filter 个 filter_length 大小的词组卷积核
model.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
# we use max pooling:
# 使用最大池化
model.add(GlobalMaxPooling1D())
# We add a vanilla hidden layer:
# 添加一个原始隐藏层
model.add(Dense(hidden_dims))
model.add(Activation('relu'))
model.add(Dropout(0.5))
# We project onto a single unit output layer, and squash it with a sigmoid:
# 投影到一个单神经元的输出层，并且使用 sigmoid 压缩它
model.add(Dense(1))
factor=0.2,
verbose=1,
patience=5,
min_lr=0.001)
checkpoint = ModelCheckpoint(model_weight_file,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='min',
period=1)
data_vis = DataVisualized()
model = Sequential([
Convolution1D(88,
4,
border_mode='same',
input_shape=(timesteps, len(features)),
name="conv_1"),
BatchNormalization(),
Activation('tanh'),
Dropout(0.3),
Convolution1D(60, 4, border_mode='same', name="conv_2"),
BatchNormalization(),
Activation('tanh'),
Dropout(0.3),
Convolution1D(20, 4, border_mode='same', name="conv_4"),
BatchNormalization(),
Activation('tanh'),
Dropout(0.3),
Convolution1D(8, 4, border_mode='same', name="conv_5"),
BatchNormalization(),
def lstm(self):
for seq_len in [30]:
print(self.seq_length)
keyer = self.solve_data()
toperr = 0.0
topfar = 0.0
topfrr = 0.0
for nflod in range(1):
train_list = []
test_list = []
for j in range(self.usernum):
one = keyer[j]
random.shuffle(one) # 随机打乱list
train, test = self.split_list(nflod, one)
train_list.append(train)
test_list.append(test)
totalerr = 0.0
totalfar = 0.0
totalfrr = 0.0
for userid in range(self.usernum - 1, self.usernum):
print("第%d个用户" % (userid), "固定长度:", self.seq_length,
"开始训练。。。")
x_train_1 = train_list[userid]
nb_word = 0
y_train_1 = [1] * len(x_train_1)
x_test_1 = test_list[userid]
y_test_1 = [1] * len(x_test_1)
leave_train = []
samplenum = len(y_train_1) * 2
print(samplenum)
if samplenum % 10 >= 5:
samplenum = (int(samplenum / 10) + 1) * 10
else:
samplenum = int(samplenum / 10) * 10
len1 = samplenum - len(y_train_1)
for k in range(self.usernum):
if k != userid:
leave_train = leave_train + train_list[k]
print(len1)
x_train_1 = x_train_1 + random.sample(leave_train, len1)
y_train_1 = y_train_1 + [0] * len1
leave_test = []
realnum = len(y_test_1)
testnum = len(y_test_1) * 2
print(realnum)
if testnum % 10 >= 5:
testnum = (int(testnum / 10) + 1) * 10
else:
testnum = int(testnum / 10) * 10
len2 = testnum - len(y_test_1)
for k in range(self.usernum):
if k != userid:
leave_test = leave_test + test_list[k]
print(len2)
x_test_1 = x_test_1 + random.sample(leave_test, len2)
y_test_1 = y_test_1 + [0] * len2
y_test_1_tmp = y_test_1[:]
x_train_1 = numpy.array(x_train_1)
x_test_1 = numpy.array(x_test_1)
y_train_1 = numpy.array(y_train_1)
# print(y_train_1)
model = Sequential()
# model.add(Embedding(nb_word + 1,128,dropout=0.2))
model.add(
Convolution1D(batch_input_shape=(self.batch_size,
x_train_1.shape[1],
x_train_1.shape[2]),
filters=self.nb_filter,
kernel_size=self.filter_length,
padding='valid',
activation='relu'))
model.add(Dropout(0.5))
model.add(MaxPooling1D(pool_size=self.pool_length))
model.add(GRU(32, return_sequences=True, stateful=True))
model.add(Dropout(0.5))
model.add(GRU(32, return_sequences=False))
# model.add(Bidirectional(LSTM(32,return_sequences=True,batch_input_shape=(batch_size, x_train_1.shape[1], x_train_1.shape[2]))))
# model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
# model.add(Flatten())
model.add(Dense(1, activation="sigmoid"))
model.compile(loss="binary_crossentropy",
optimizer='adam',
metrics=['accuracy'])
# model.fit(x_train_1, y_train_1,epochs=epochs, batch_size=batch_size,verbose=2,shuffle=False,validation_data=(x_test_1, y_test_1))
minerr = 1.0
# class_weight={0:0.33,1:0.67}
# model.fit(x_train_1, y_train_1, epochs=1, batch_size=batch_size, verbose=2, shuffle=False)
'''
for nums in range(200):
model.fit(x_train_1, y_train_1, epochs=1, batch_size=batch_size, verbose=2, shuffle=False)
model.reset_states()
setThr(model, x_test_1, realnum)
#model.save('model_2.h5')
'''
model.fit(x_train_1,
y_train_1,
batch_size=10,
epochs=200,
validation_data=(x_test_1, y_test_1),
verbose=1,
shuffle=True)
model.fit(x_test_1,
y_test_1,
batch_size=10,
epochs=200,
verbose=1,
shuffle=True)
m = 'model_' + str(self.usernum) + '.h5'
model.save(m)
#k=keystroke_lstm(78)
#k.lstm()
def build(self):
# Build model
number_of_class = 4
if self.model_type == "CNN-non-static":
vocabulary_size = len(self.embed_matrix) # include 0 empty
input_shape = (self.sequence_length, )
model_input = Input(shape=input_shape)
z = Embedding(vocabulary_size,
self.embedding_dim,
input_length=self.sequence_length,
weights=[self.embed_matrix])(model_input)
elif self.model_type == "CNN-static":
input_shape = (self.sequence_length, self.embedding_dim)
model_input = Input(shape=input_shape)
z = model_input
# MaxPooling1D
#
# z = Dropout(self.dropout_prob)(z)
# # Convolutional block
# conv_blocks = []
# for sz in self.filter_sizes:
# conv = Convolution1D(filters=self.num_filters,
# kernel_size=sz,
# padding="valid",
# activation="relu",
# strides=1)(z)
# conv = MaxPooling1D(pool_size=2)(conv)
# conv = Flatten()(conv)
# conv_blocks.append(conv)
# z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
#
# z = Dropout(self.dropout_prob)(z)
# z = Dense(self.hidden_dims, activation="relu")(z)
# model_output = Dense(number_of_class, activation="sigmoid")(z)
#
# model = Model(model_input, model_output)
# model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
#
# GlobalMaxPooling1D
convolution_output = []
for sz in self.filter_sizes:
conv = Convolution1D(filters=self.num_filters,
kernel_size=sz,
activation='tanh',
name='Conv1D_{}_{}'.format(
self.num_filters, sz))(z)
pool = GlobalMaxPooling1D(name='MaxPoolingOverTime_{}_{}'.format(
self.num_filters, sz))(conv)
convolution_output.append(pool)
x = Concatenate()(convolution_output)
# Fully connected layers
for fl in [self.hidden_dims]:
x = Dense(fl, activation='selu',
kernel_initializer='lecun_normal')(x)
x = AlphaDropout(self.dropout_prob)(x)
# Output layer
predictions = Dense(number_of_class, activation='softmax')(x)
# Build and compile model
model = Model(inputs=model_input, outputs=predictions)
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.summary()
return model
text_seq_input = Input(shape=(max_seq_len, ))
embeds = Embedding(embedding_matrix.shape[0],
embedding_matrix.shape[1],
weights=[embedding_matrix],
trainable=False)(text_seq_input)
xx = SpatialDropout1D(0.08500492158254568)(embeds)
xx = Bidirectional(CuDNNLSTM(64, return_sequences=True))(xx)
xx = SpatialDropout1D(0.1733455182727272)(xx)
cnn1 = Convolution1D(256,
5,
padding='valid',
strides=3,
activation='elu',
kernel_initializer="glorot_uniform")(xx)
cnn1 = SpatialDropout1D(0.1231232556)(cnn1)
cnn1_max = GlobalMaxPooling1D()(cnn1)
cnn1_ave = GlobalAveragePooling1D()(cnn1)
cnn2 = Convolution1D(256,
5,
padding='valid',
strides=3,
activation='elu',
kernel_initializer="glorot_uniform")(xx)
cnn2 = SpatialDropout1D(0.10348347384)(cnn2)
cnn2_max = GlobalMaxPooling1D()(cnn2)
cnn2_ave = GlobalAveragePooling1D()(cnn2)
def DoubleCNNWordEmbed(nb_labels,
nb_filters_1=1200,
nb_filters_2=600,
n_gram=2,
filter_length_2=10,
maxlen=15,
vecsize=300,
cnn_dropout_1=0.0,
cnn_dropout_2=0.0,
final_activation='softmax',
dense_wl2reg=0.0,
optimizer='adam'):
""" Returns the double-layered convolutional neural network (CNN/ConvNet) for word-embedded vectors.
:param nb_labels: number of class labels
:param nb_filters_1: number of filters for the first CNN/ConvNet layer (Default: 1200)
:param nb_filters_2: number of filters for the second CNN/ConvNet layer (Default: 600)
:param n_gram: n-gram, or window size of first CNN/ConvNet (Default: 2)
:param filter_length_2: window size for second CNN/ConvNet layer (Default: 10)
:param maxlen: maximum number of words in a sentence (Default: 15)
:param vecsize: length of the embedded vectors in the model (Default: 300)
:param cnn_dropout_1: dropout rate for the first CNN/ConvNet layer (Default: 0.0)
:param cnn_dropout_2: dropout rate for the second CNN/ConvNet layer (Default: 0.0)
:param final_activation: activation function. Options: softplus, softsign, relu, tanh, sigmoid, hard_sigmoid, linear. (Default: 'softmax')
:param dense_wl2reg: L2 regularization coefficient (Default: 0.0)
:param optimizer: optimizer for gradient descent. Options: sgd, rmsprop, adagrad, adadelta, adam, adamax, nadam. (Default: adam)
:return: keras sequantial model for CNN/ConvNet for Word-Embeddings
:type nb_labels: int
:type nb_filters_1: int
:type nb_filters_2: int
:type n_gram: int
:type filter_length_2: int
:type maxlen: int
:type vecsize: int
:type cnn_dropout_1: float
:type cnn_dropout_2: float
:type final_activation: str
:type dense_wl2reg: float
:type optimizer: str
:rtype: keras.model.Sequential
"""
model = Sequential()
model.add(Convolution1D(nb_filter=nb_filters_1,
filter_length=n_gram,
border_mode='valid',
activation='relu',
input_shape=(maxlen, vecsize)))
if cnn_dropout_1 > 0.0:
model.add(Dropout(cnn_dropout_1))
model.add(Convolution1D(nb_filter=nb_filters_2,
filter_length=filter_length_2,
border_mode='valid',
activation='relu'))
if cnn_dropout_2 > 0.0:
model.add(Dropout(cnn_dropout_2))
model.add(MaxPooling1D(pool_length=maxlen - n_gram -filter_length_2 + 1))
model.add(Flatten())
model.add(Dense(nb_labels,
activation=final_activation,
W_regularizer=l2(dense_wl2reg)))
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model
def create_deepnet(self):
input_query_1 = Input(shape=(None,), name='input_query_1', dtype='int32')
input_query_2 = Input(shape=(None,), name='input_query_2', dtype='int32')
nb_words = len(self.embedding_matrix)
shared_embedding = Embedding(nb_words, EMBEDDING_DIM, weights=[self.embedding_matrix],
input_length=MAX_LEN, trainable=False)
q1_embedding = shared_embedding(input_query_1)
q1_dense = (Dense(EMBEDDING_DIM, activation='relu'))(q1_embedding)
q1_lambda = Lambda(lambda x: k.sum(x, axis=1), output_shape=(EMBEDDING_DIM,))(q1_dense)
q2_embedding = shared_embedding(input_query_2)
q2_dense = (Dense(EMBEDDING_DIM, activation='relu'))(q2_embedding)
q2_lambda = Lambda(lambda x: k.sum(x, axis=1), output_shape=(EMBEDDING_DIM,))(q2_dense)
q3_embedding = shared_embedding(input_query_1)
q3_conv_1d_1 = Convolution1D(nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='valid',
activation='relu', subsample_length=1)(q3_embedding)
q3_dropout_1 = Dropout(0.2)(q3_conv_1d_1)
q3_conv_1d_2 = Convolution1D(nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='valid',
activation='relu', subsample_length=1)(q3_dropout_1)
q3_global_pooling_1d_1 = GlobalMaxPooling1D()(q3_conv_1d_2)
q3_dropout_2 = Dropout(0.2)(q3_global_pooling_1d_1)
q3_dense = Dense(EMBEDDING_DIM)(q3_dropout_2)
q3_dropout_3 = Dropout(0.2)(q3_dense)
q3_bn = BatchNormalization()(q3_dropout_3)
q4_embedding = shared_embedding(input_query_2)
q4_conv_1d_1 = Convolution1D(nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='valid',
activation='relu', subsample_length=1)(q4_embedding)
q4_dropout_1 = Dropout(0.2)(q4_conv_1d_1)
q4_conv_1d_2 = Convolution1D(nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='valid',
activation='relu', subsample_length=1)(q4_dropout_1)
q4_global_pooling_1d_1 = GlobalMaxPooling1D()(q4_conv_1d_2)
q4_dropout_2 = Dropout(0.2)(q4_global_pooling_1d_1)
q4_dense = Dense(EMBEDDING_DIM)(q4_dropout_2)
q4_dropout_3 = Dropout(0.2)(q4_dense)
q4_bn = BatchNormalization()(q4_dropout_3)
q5_embedding = shared_embedding(input_query_1)
q5_lstm = LSTM(EMBEDDING_DIM, dropout_W=0.2, dropout_U=0.2)(q5_embedding)
q6_embedding = shared_embedding(input_query_2)
q6_lstm = LSTM(EMBEDDING_DIM, dropout_W=0.2, dropout_U=0.2)(q6_embedding)
merge_layer = Merge(mode='concat')([q1_lambda, q2_lambda, q3_bn, q4_bn, q5_lstm, q6_lstm])
bn_layer_1 = BatchNormalization()(merge_layer)
dense_layer_1 = Dense(EMBEDDING_DIM)(bn_layer_1)
prelu_layer_1 = PReLU()(dense_layer_1)
dropout_layer_1 = Dropout(0.2)(prelu_layer_1)
bn_layer_2 = BatchNormalization()(dropout_layer_1)
dense_layer_2 = Dense(EMBEDDING_DIM)(bn_layer_2)
prelu_layer_2 = PReLU()(dense_layer_2)
dropout_layer_2 = Dropout(0.2)(prelu_layer_2)
bn_layer_3 = BatchNormalization()(dropout_layer_2)
dense_layer_3 = Dense(EMBEDDING_DIM)(bn_layer_3)
prelu_layer_3 = PReLU()(dense_layer_3)
dropout_layer_3 = Dropout(0.2)(prelu_layer_3)
bn_layer_4 = BatchNormalization()(dropout_layer_3)
dense_layer_4 = Dense(EMBEDDING_DIM)(bn_layer_4)
prelu_layer_4 = PReLU()(dense_layer_4)
dropout_layer_4 = Dropout(0.2)(prelu_layer_4)
bn_layer_5 = BatchNormalization()(dropout_layer_4)
dense_layer_5 = Dense(EMBEDDING_DIM)(bn_layer_5)
prelu_layer_5 = PReLU()(dense_layer_5)
dropout_layer_5 = Dropout(0.2)(prelu_layer_5)
output = Dense(1, activation='sigmoid')(dropout_layer_5)
self.model = Model(input=[input_query_1, input_query_2], output=output)
self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
return self.model
def _build_model(self):
# Neural Net for Deep-Q learning Model
model = Sequential()
# model.add(Dense(24, input_dim=self.state_size, activation='relu'))
# model.add(Dropout(0.5))
# model.add(Dense(32, activation='relu'))
# model.add(Dropout(0.5))
# model.add(Dense(10, activation='relu'))
# model.add(Dense(self.action_size, activation='linear'))
# model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
#
if self.config['network'] == "CNNRNN":
model.add(LSTM(10, return_sequences=True, input_shape=(30, 8)))
model.add(Dropout(0.5))
model.add(LSTM(50, return_sequences=True))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(50))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='relu'))
model.add(Dense(self.action_size, activation='linear'))
model.compile(loss=self.config['loss'],
optimizer=Adam(lr=self.learning_rate))
else:
PERIODS_PER_X = 30
# model.add(LSTM(32, return_sequences=True,input_shape=(30,8)))
# model.add(Dropout(0.5))
# model.add(LSTM(32))
# model.add(Dropout(0.5))
# model.add(Dense(32, activation='relu'))
# model.add(Dropout(0.5))
# model.add(Dense(10, activation='relu'))
# model.add(Dense(self.action_size, activation='linear'))
# model.compile(loss=self.config['loss'], optimizer=Adam(lr=self.learning_rate))
# model.add(Reshape((1, PERIODS_PER_X, self.state_size), input_shape=(PERIODS_PER_X, self.state_size)))
# model.add(Input((PERIODS_PER_X, self.state_size)))
model.add(
Convolution1D(128,
3,
padding='same',
activation='relu',
input_shape=(PERIODS_PER_X, 8)))
model.add(MaxPooling1D(pool_size=2))
model.add(Convolution1D(64, 3, activation='relu', padding="same"))
model.add(MaxPooling1D(pool_size=2))
# model.add(Reshape((PERIODS_PER_X, self.state_size)))
model.add(LSTM(64, return_sequences=True))
model.add(Dropout(0.25))
model.add(LSTM(32, return_sequences=True))
model.add(Dropout(0.5))
model.add(Dense(16, activation='relu'))
model.add(Dense(self.action_size, activation='softmax'))
# if doesn't fit as well as other one, try adding more layers, or try using the old model with only price information (KISS)
model.compile(loss=self.config['loss'],
optimizer=Adam(lr=self.learning_rate))
# Create the model based on the information above
#model.compile(loss='mean_squared_error', optimizer='adam')
if os.path.isfile(self.weight_backup):
#load_model(self.weight_backup)
model.load_weights(self.weight_backup, True)
self.exploration_rate = self.exploration_min
return model
model_input = Input(shape=input_shape)
# Static model does not have embedding layer
if model_type == "CNN-static":
z = model_input
else:
z = Embedding(len(vocabulary_inv), embedding_dim, input_length=sequence_length, name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
num_seq_per_batch=num_seq_per_batch,
input_processing=[DivideBy(input_std),
IndependentlyCenter()],
target_processing=[DivideBy(target_std)])
# get the shape of X_train
(X_train, Y_train) = pipeline.train_generator().next()
starting_time = time.time()
nb_epoch = 1
# define the network architecture = Conv Net
model = Sequential()
model.add(
Convolution1D(64,
3,
border_mode='same',
input_shape=(1, X_train.shape[2]),
init='normal',
activation='relu'))
model.add(Flatten())
model.add(Dense(32))
model.add(Activation('relu'))
model.add(Dense(3, activation='sigmoid'))
# compile the model
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error', optimizer=sgd)
compiling_time = time.time() - starting_time
print('compiling time = ', compiling_time)
model.fit_generator(pipeline.train_generator(fold = ' train', source_id = 0), \
samples_per_epoch = 64064, \
nb_epoch = 2)
print('run time = ', time.time() - compiling_time)
def CRNN_1d(input_dim, useBatchNorm):
reg = l2(0.0001)
model = Sequential()
model.add(
InputLayer(input_shape=(input_dim[0], input_dim[1], input_dim[2])))
model.add(Reshape((input_dim[0], input_dim[2])))
model.add(
Convolution1D(filters=16,
kernel_size=7,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
#model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=16,
kernel_size=5,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
#model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=32,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=32,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
#model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
#
model.add(
Convolution1D(filters=64,
kernel_size=3,
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
#model.add(MaxPooling1D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(0.4))
model.add(
Bidirectional(
LSTM(units=64,
return_sequences=True,
dropout=0.4,
kernel_regularizer=reg,
recurrent_regularizer=reg)))
model.add(
Bidirectional(
LSTM(units=128,
return_sequences=True,
dropout=0.4,
kernel_regularizer=reg,
recurrent_regularizer=reg)))
model.add(
Bidirectional(
LSTM(units=64,
return_sequences=False,
dropout=0.4,
kernel_regularizer=reg,
recurrent_regularizer=reg)))
return model
def block_deepFlavourConvolutions(charged,
neutrals,
vertices,
dropoutRate,
active=True,
batchnorm=False,
batchmomentum=0.6):
'''
deep Flavour convolution part.
'''
cpf = charged
if active:
cpf = Convolution1D(64,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='cpf_conv0')(cpf)
if batchnorm:
cpf = BatchNormalization(momentum=batchmomentum,
name='cpf_batchnorm0')(cpf)
cpf = Dropout(dropoutRate, name='cpf_dropout0')(cpf)
cpf = Convolution1D(32,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='cpf_conv1')(cpf)
if batchnorm:
cpf = BatchNormalization(momentum=batchmomentum,
name='cpf_batchnorm1')(cpf)
cpf = Dropout(dropoutRate, name='cpf_dropout1')(cpf)
cpf = Convolution1D(32,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='cpf_conv2')(cpf)
if batchnorm:
cpf = BatchNormalization(momentum=batchmomentum,
name='cpf_batchnorm2')(cpf)
cpf = Dropout(dropoutRate, name='cpf_dropout2')(cpf)
cpf = Convolution1D(8,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='cpf_conv3')(cpf)
else:
cpf = Convolution1D(1, 1, kernel_initializer='zeros',
trainable=False)(cpf)
npf = neutrals
if active:
npf = Convolution1D(32,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='npf_conv0')(npf)
if batchnorm:
npf = BatchNormalization(momentum=batchmomentum,
name='npf_batchnorm0')(npf)
npf = Dropout(dropoutRate, name='npf_dropout0')(npf)
npf = Convolution1D(16,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='npf_conv1')(npf)
if batchnorm:
npf = BatchNormalization(momentum=batchmomentum,
name='npf_batchnorm1')(npf)
npf = Dropout(dropoutRate, name='npf_dropout1')(npf)
npf = Convolution1D(4,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='npf_conv2')(npf)
else:
npf = Convolution1D(1, 1, kernel_initializer='zeros',
trainable=False)(npf)
vtx = vertices
if active:
vtx = Convolution1D(64,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='vtx_conv0')(vtx)
if batchnorm:
vtx = BatchNormalization(momentum=batchmomentum,
name='vtx_batchnorm0')(vtx)
vtx = Dropout(dropoutRate, name='vtx_dropout0')(vtx)
vtx = Convolution1D(32,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='vtx_conv1')(vtx)
if batchnorm:
vtx = BatchNormalization(momentum=batchmomentum,
name='vtx_batchnorm1')(vtx)
vtx = Dropout(dropoutRate, name='vtx_dropout1')(vtx)
vtx = Convolution1D(32,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='vtx_conv2')(vtx)
if batchnorm:
vtx = BatchNormalization(momentum=batchmomentum,
name='vtx_batchnorm2')(vtx)
vtx = Dropout(dropoutRate, name='vtx_dropout2')(vtx)
vtx = Convolution1D(8,
1,
kernel_initializer='lecun_uniform',
activation='relu',
name='vtx_conv3')(vtx)
else:
vtx = Convolution1D(1, 1, kernel_initializer='zeros',
trainable=False)(vtx)
return cpf, npf, vtx
def shallow_model(input_dim, useBatchNorm, convFilters, convKernels, convPool,
convDO, denseUnits, denseDO, rnnUnits, rnnDO, convModelDim):
reg = l2(0.00001)
model = Sequential()
model.add(
InputLayer(input_shape=(input_dim[0], input_dim[1], input_dim[2])))
if convModelDim == 1:
model.add(Reshape((input_dim[0], input_dim[2])))
#Convolutional layers block:
for i in range(len(convFilters)):
if convModelDim == 1:
model.add(
Convolution1D(filters=convFilters[i],
kernel_size=convKernels[i],
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
else:
model.add(
Convolution2D(filters=convFilters[i],
kernel_size=convKernels[i],
padding='same',
kernel_regularizer=reg,
bias_regularizer=reg))
if convPool[i]:
if convModelDim == 1:
model.add(MaxPooling1D())
else:
model.add(MaxPooling2D())
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(convDO))
ret = True
for i in range(len(rnnUnits)):
if i == (len(rnnUnits) - 1):
ret = False
model.add(
Bidirectional(
LSTM(units=rnnUnits[i],
return_sequences=ret,
dropout=rnnDO,
kernel_regularizer=reg,
recurrent_regularizer=reg)))
#flattening up filters
if ret:
model.add(Flatten())
#
for i in range(len(denseUnits)):
model.add(
Dense(units=denseUnits[i],
kernel_regularizer=reg,
bias_regularizer=reg))
if useBatchNorm:
model.add(BatchNormalization(scale=False))
model.add(Activation('relu'))
else:
model.add(PReLU(shared_axes=[1]))
model.add(Dropout(denseDO[min(i, len(denseDO) - 1)]))
return model
def build_model(
input_size,
alphabet_size,
conv_layers,
fully_connected_layers,
embedding_size,
threshold,
dropout_p,
num_of_classes,
optimizer='adam',
loss='categorical_crossentropy', # binary
# loss='sparse_categorical_crossentropy', # integers
metrics=['accuracy'],
concat=False,
):
"""
Build and compile the Character Level CNN model
Returns: None
"""
# Input layer
inputs = Input(shape=(input_size, ), name='sent_input', dtype='int64')
# Embedding layers
x = Embedding(alphabet_size + 1,
embedding_size,
input_length=input_size,
trainable=True)(inputs)
# Convolution layers
if concat:
channels = []
for cl in conv_layers:
x1 = Convolution1D(cl[0],
cl[1],
activation='tahn',
padding='same',
kernel_regularizer=regularizers.l2(0.03))(x)
# x = ThresholdedReLU(threshold)(x)
if cl[2] != -1:
x1 = MaxPooling1D(cl[2], padding='valid')(x1)
if concat:
channels.append(x1)
else:
x = x1
if concat:
x1 = concatenate(channels, axis=1)
x = Flatten()(x1)
# Fully connected layers
for fl in fully_connected_layers:
x = Dense(fl, activation='relu')(x)
#x = ThresholdedReLU(threshold)(x)
x = Dropout(dropout_p)(x)
# Output layer
predictions = Dense(num_of_classes, activation='softmax')(x)
# Build and compile model
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
model = model
print("CharCNNZhang model built: ")
model.summary()
return model
distance1_input = Input(shape=(max_sentence_len, ),
dtype='int32',
name='distance1_input')
distance1 = Embedding(max_position, position_dims)(distance1_input)
distance2_input = Input(shape=(max_sentence_len, ),
dtype='int32',
name='distance2_input')
distance2 = Embedding(max_position, position_dims)(distance2_input)
output = concatenate([words, distance1, distance2])
output = Convolution1D(filters=nb_filter,
kernel_size=filter_length,
padding='same',
activation='tanh',
strides=1)(output)
# we use standard max over time pooling
output = GlobalMaxPooling1D()(output)
#output = Flatten()(output)
#output = Dropout(0.25)(output)
#output = Dense(512)(output)
#output = LeakyReLU()(output)
output = Dense(n_out, activation='softmax')(output)
model = Model(inputs=[words_input, distance1_input, distance2_input],
outputs=[output])
model.compile(loss='sparse_categorical_crossentropy',
optimizer='Adam',
metrics=['accuracy'])
for i in range(20, 1090):
X_train.append(training_set[i - 20:i, 0])
y_train.append(training_set[i, 0])
X_train = np.asarray(X_train)
y_train = np.asarray(y_train)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
from keras.models import Sequential
from keras.layers import Convolution1D
from keras.layers import MaxPooling1D
from keras.layers import Flatten
from keras.layers import Dense
model = Sequential()
model.add(
Convolution1D(filters=16,
kernel_size=3,
activation='relu',
input_shape=(20, 1)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(20, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(loss='mean_absolute_error', optimizer='adam')
# fit network
history = model.fit(X_train,
y_train,
epochs=60,
batch_size=8,
validation_split=0.02)
test_set = dataset.iloc[1100:1130, 1:2].values
X_test = []
def train_cnn_lstm():
# Embedding
max_features = 20000
maxlen = 10
embedding_size = 40
# Convolution
filter_length = 5
nb_filter = 64
pool_length = 4
# LSTM
lstm_output_size = 70
# Training
batch_size = 35
nb_epoch = 2
'''
Note:
batch_size is highly sensitive.
Only 2 epochs are needed as the dataset is very small.
'''
print('Loading data...')
#(X_train, y_train), (X_test, y_test) = load_data()
X_train, y_train, X_test, y_test = load_data()
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')
print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen, value=3)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen, value=3)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
y_train = np.array(y_train)
print('y_train shape', y_train.shape)
y_test = np.array(y_test)
print('y_train shape', y_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
model.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)
#save the model &serialize model to JSON
model_json = model.to_json()
with open("./dict/model_cnn_lstm.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("./dict/model_cnn_lstm_weights.h5")
print("Saved model to disk")
del model
print('Test score:', score)
print('Test accuracy:', acc)
y = data['y']
(x_train, x_val, y_train, y_val) = train_test_split(x,
y,
test_size=0.4,
random_state=SEED)
#print('Building model...')
n_features = x_train.shape[2]
input_shape = (None, n_features)
model_input = Input(input_shape, name='input')
layer = model_input
for i in range(N_LAYERS):
# convolutional layer names are used by extract_filters.py
layer = Convolution1D(nb_filter=CONV_FILTER_COUNT,
filter_length=FILTER_LENGTH,
name='convolution_' + str(i + 1))(layer)
layer = Activation('relu')(layer)
layer = MaxPooling1D(2)(layer)
layer = Dropout(0.5)(layer)
layer = LSTM(LSTM_COUNT, return_sequences=True)(layer)
layer = Dropout(0.5)(layer)
layer = TimeDistributed(Dense(len(GENRES)))(layer)
layer = Activation('softmax', name='output_realtime')(layer)
time_distributed_merge_layer = Lambda(function=lambda x: K.mean(x, axis=1),
output_shape=lambda shape:
(shape[0], ) + shape[2:],
name='output_merged')
model_output = time_distributed_merge_layer(layer)
model = Model(model_input, model_output)
def inceptionBlock(x):
_drop_rate_ = .1
x = BatchNormalization()(x)
conv1_1 = Convolution1D(100,
1,
activation='relu',
padding='same',
kernel_regularizer=l2(0.001))(x)
conv1_1 = Dropout(_drop_rate_)(
conv1_1
) #https://www.quora.com/Can-l-combine-dropout-and-l2-regularization
conv1_1 = BatchNormalization()(conv1_1)
conv2_1 = Convolution1D(100,
1,
activation='relu',
padding='same',
kernel_regularizer=l2(0.001))(x)
conv2_1 = Dropout(_drop_rate_)(conv2_1)
conv2_1 = BatchNormalization()(conv2_1)
conv2_2 = Convolution1D(100,
3,
activation='relu',
padding='same',
kernel_regularizer=l2(0.001))(conv2_1)
conv2_2 = Dropout(_drop_rate_)(conv2_2)
conv2_2 = BatchNormalization()(conv2_2)
conv3_1 = Convolution1D(100,
1,
activation='relu',
padding='same',
kernel_regularizer=l2(0.001))(x)
conv3_1 = Dropout(_drop_rate_)(conv3_1)
conv3_1 = BatchNormalization()(conv3_1)
conv3_2 = Convolution1D(100,
3,
activation='relu',
padding='same',
kernel_regularizer=l2(0.001))(conv3_1)
conv3_2 = Dropout(_drop_rate_)(conv3_2)
conv3_2 = BatchNormalization()(conv3_2)
conv3_3 = Convolution1D(100,
3,
activation='relu',
padding='same',
kernel_regularizer=l2(0.001))(conv3_2)
conv3_3 = Dropout(_drop_rate_)(conv3_3)
conv3_3 = BatchNormalization()(conv3_3)
conv3_4 = Convolution1D(100,
3,
activation='relu',
padding='same',
kernel_regularizer=l2(0.001))(conv3_3)
conv3_4 = Dropout(_drop_rate_)(conv3_4)
conv3_4 = BatchNormalization()(conv3_4)
concat = concatenate([conv1_1, conv2_2, conv3_4])
concat = BatchNormalization()(concat)
return concat
except Exception as e:
break
X.append(x_i)
Y.append(y_i)
X, Y = np.array(X), np.array(Y)
X_train, X_test, Y_train, Y_test = create_Xt_Yt(X, Y)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], EMB_SIZE))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], EMB_SIZE))
model = Sequential()
model.add(
Convolution1D(input_shape=(WINDOW, EMB_SIZE),
nb_filter=16,
filter_length=4,
border_mode='same'))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dropout(0.5))
model.add(Convolution1D(nb_filter=8, filter_length=4, border_mode='same'))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(64))
model.add(BatchNormalization())
model.add(LeakyReLU())
