def get_feature_model(params):
embedding_dims = params['embedding_dims']
max_features = 8001
model = Sequential()
model.add(
Embedding(max_features,
embedding_dims,
input_length=MAXLEN,
dropout=params['embedding_dropout']))
for i in xrange(params['nb_conv']):
model.add(
Convolution1D(nb_filter=params['nb_filter'],
filter_length=params['filter_length'],
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(
MaxPooling1D(pool_length=params['pool_length'],
stride=params['stride']))
model.add(Flatten())
model.summary()
return model
def build_model(features, seq_len, out):
model = Sequential()
model.add(LSTM(100, input_shape=(seq_len, features),
return_sequences=True))
model.add(Activation("tanh"))
model.add(Convolution1D(50, 10, border_mode='valid'))
model.add(Activation("relu"))
model.add(Flatten())
model.add(Dense(units=out))
model.add(Activation("linear"))
start = time.time()
adam = Adam(lr=0.25, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
#model.compile(loss = "mean_absolute_percentage_error", optimizer = 'RMSprop')
model.compile(loss="mean_absolute_percentage_error", optimizer=adam)
print("> Compilation Time : ", time.time() - start)
return model
def conv_aggregate(pad, dropout=1/2, l2reg=1e-4, cnninit='glorot_uniform', cnnact='relu',
cdim={1: 1/2, 2: 1/2, 3: 1/2, 4: 1/2, 5: 1/2, 6: 1/2, 7: 1/2}, inputs=None, input_dim=304, pfx=''):
qi_cnn_res_list = []
si_cnn_res_list = []
tot_len = 0
for fl, cd in cdim.items():
nb_filter = int(input_dim*cd)
shared_conv = Convolution1D(name=pfx+'conv%d'%(fl), input_shape=(None, conf['pad'], input_dim),
kernel_size=fl, filters=nb_filter, activation='linear', padding='same',
kernel_regularizer=l2(l2reg), kernel_initializer=cnninit)
qi_one = Activation(cnnact)(BatchNormalization()(shared_conv(inputs[0]))) # shape:(None, pad, nbfilter)
si_one = Activation(cnnact)(BatchNormalization()(shared_conv(inputs[1]))) # shape:(None, pad, nbfilter)
qi_cnn_res_list.append(qi_one)
si_cnn_res_list.append(si_one)
tot_len += nb_filter
qi_cnn = Dropout(dropout, noise_shape=(None, pad, tot_len))(concatenate(qi_cnn_res_list))
si_cnn = Dropout(dropout, noise_shape=(None, pad, tot_len))(concatenate(si_cnn_res_list))
return (qi_cnn, si_cnn, tot_len)
def create_model(nb_filters=3,
nb_conv=2,
pool=20,
dropout=0.5,
denseunits=2):
model = Sequential()
model.add(
Convolution1D(nb_filters,
nb_conv,
activation='relu',
input_shape=(X_train.shape[1], X_train.shape[2]),
padding="same"))
model.add(MaxPooling1D(pool))
model.add(Dropout(dropout))
model.add(Flatten())
model.add(Dense(denseunits, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
return model
def __init__(self, **settings):
# max_features, embedding_dim, seqlen,
# nb_filter, filter_widths, activation, dropout_p,
# l1reg, l2reg, batchnorm,
self.settings = settings
self.settings['verbosity'] = 2
seqlen = self.settings['seqlen']
l1reg = self.settings['l1reg']
l2reg = self.settings['l2reg']
conv_filters = []
for n_gram in settings['filter_widths']:
sequential = Sequential()
conv_filters.append(sequential)
sequential.add(Embedding(input_dim=self.settings['max_features'],
output_dim=self.settings['embedding_dim'],
input_length=seqlen))
sequential.add(Dropout(self.settings['dropout_p']))
sequential.add(Convolution1D(self.settings['nb_filter'],
n_gram,
activation=self.settings['activation']
))
sequential.add(MaxPooling1D(pool_length=seqlen - n_gram + 1))
sequential.add(Flatten())
self.nn = Sequential()
self.nn.add(Merge(conv_filters, mode='concat'))
self.nn.add(Dropout(self.settings['dropout_p']))
if (l1reg is not None and l1reg is float and l2reg is not
None and l2reg is float):
self.nn.add(Dense(1), W_regularizer=l1l2(l1reg, l2reg))
elif (l2reg is not None and l2reg is float):
self.nn.add(Dense(1), W_regularizer=l2(l2reg))
elif (l1reg is not None and l1reg is float):
self.nn.add(Dense(1), W_regularizer=l1(l1reg))
else:
self.nn.add(Dense(1))
if (self.settings['batchnorm'] is True):
self.nn.add(BatchNormalization())
self.nn.add(Activation('sigmoid'))
def get_models():
models = dict()
filter_sizes = (3, 8)
dropout_prob = (0.5, 0.8)
hidden_dims = 50
models = dict()
# Build model
input_shape = (170, 1)
model_input = Input(shape=input_shape)
# Static model do not have embedding layer
z = Dropout(dropout_prob[0])(model_input)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=2,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(7, activation="sigmoid")(z)
models['convnet'] = keras.models.Model(model_input, model_output)
models['convnet'].compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
return models
def cnn(W):
nb_filter = 250
filter_length = 3
hidden_dims = 250
maxlen = 200
model = Sequential()
# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
model.add(Embedding(W.shape[0], W.shape[1], input_length=maxlen, weights=[W]))
model.add(Dropout(0.25))
# we add a Convolution1D, which will learn nb_filter
# word group filters of size filter_length:
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
# we use standard max pooling (halving the output of the previous layer):
model.add(MaxPooling1D(pool_length=2))
# We flatten the output of the conv layer,
# so that we can add a vanilla dense layer:
model.add(Flatten())
# We add a vanilla hidden layer:
model.add(Dense(hidden_dims))
model.add(Dropout(0.25))
model.add(Activation('relu'))
# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(2))
model.add(Activation('softmax'))
return model
def document_embedding(name, n_in, n_em, n_out, n_words):
"""
document input embedding
Args:
name: layer name
n_in: input max numbers of words
n_em: embedding size
n_out: output dimension
n_words: num words
Return:
(input_layer, reshape_layer
"""
input_layer = Input(shape=(n_in, ), name=name)
embedding_layer = Embedding(n_words, n_em)(input_layer)
convolution_layer = Convolution1D(n_out,
1,
padding="same",
input_shape=(n_in, n_em),
activation="tanh")(embedding_layer)
pooling_layer = Lambda(lambda x: backend.max(x, axis=1),
output_shape=(n_out, ))(convolution_layer)
reshape_layer = Reshape((1, n_out))(pooling_layer)
return input_layer, reshape_layer
def conv_layer(x,nb_row,nb_filter,name,subsample=1,padding=None, activation='relu',border_mode='same', weight_decay=None): if weight_decay: W_regularizer = regularizers.l2(weight_decay) b_regularizer = regularizers.l2(weight_decay) else: W_regularizer = None b_regularizer = None x = Convolution1D(nb_filter, nb_row, strides=subsample, activation=activation, border_mode=border_mode, W_regularizer=W_regularizer, b_regularizer=b_regularizer, bias=False, name=name+'_Conv1')(x) if padding: for i in range(padding): x = ZeroPadding1D(padding=1, name=name+'_zp1_'+str(i))(x) return x
def cnn(train_data,train_label,test_data,test_label):
train_label = np_utils.to_categorical(train_label,label_count)
test_label_categorical = np_utils.to_categorical(test_label,label_count)
model = Sequential()
model.add(Convolution1D(200,3,border_mode='same',input_shape=
(timestep,embedding_length),W_regularizer=l2(0.01)))
model.add(GlobalMaxPooling1D())
# model.add(Dropout(0.5))
model.add(Dense(100,activation='tanh',W_regularizer=l2(0.01)))
model.add(Dense(label_count,activation='softmax',name='output'))
sgd = SGD(lr=0.001, decay=1e-5, momentum=0.9, nesterov=True)
adam = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
epoch = 10
for i in range(epoch):
print i ,'/', epoch
model.fit(train_data, train_label, batch_size=64, nb_epoch=1, shuffle=True,
verbose=1, validation_data=(test_data,test_label_categorical))
test_classes = model.predict_classes(test_data,batch_size=64)
putOut(test_classes,test_label)
def build_lstm_model(top_words,
embedding_vector_length,
max_input_length,
num_outputs,
internal_lstm_size=100,
embedding_matrix=None,
embedding_trainable=True):
model = Sequential()
model.add(
Embedding(top_words,
embedding_vector_length,
input_length=max_input_length,
weights=[embedding_matrix],
trainable=embedding_trainable))
model.add(
Convolution1D(nb_filter=32,
filter_length=3,
border_mode='same',
activation='relu'))
model.add(MaxPooling1D(pool_length=2))
model.add(LSTM(internal_lstm_size))
model.add(Dense(num_outputs, activation='softmax'))
return model
def cnnlstmModel(embeddingMatrix):
"""Constructs the architecture of the modelEMOTICONS_TOKEN[list_str[index]]
Input:
embeddingMatrix : The embedding matrix to be loaded in the embedding layer.
Output:
model : A basic LSTM model
"""
sequence = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embeddingLayer = Embedding(embeddingMatrix.shape[0],
embeddingMatrix.shape[1],
weights=[embeddingMatrix],
input_length=MAX_SEQUENCE_LENGTH,
mask_zero=emb_mask_zero,
trainable=emb_trainable)(sequence)
embedded = Dropout(0.25)(embeddingLayer)
convolution = Convolution1D(filters=nb_filter,
kernel_size=kernel_size,
padding='valid',
activation='relu',
strides=1)(embedded)
maxpooling = MaxPooling1D(pool_size=2)(convolution)
bilstm = Bidirectional(
LSTM(LSTM_DIM // 2, recurrent_dropout=0.25,
return_sequences=True))(maxpooling)
bilstm1 = Bidirectional(
LSTM(LSTM_DIM // 2, recurrent_dropout=0.25,
return_sequences=False))(bilstm)
# att = AttentionM()(bilstm1)
output = Dense(3, activation='softmax')(bilstm1)
model = Model(inputs=sequence, outputs=output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def model(X_train, X_test, y_train, y_test, maxlen, max_features):
embedding_size = 300
pool_length = 4
lstm_output_size = 100
batch_size = 200
nb_epoch = 1
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout({{uniform(0, 1)}}))
# Note that we use unnamed parameters here, which is bad style, but is used here
# to demonstrate that it works. Always prefer named parameters.
model.add(
Convolution1D({{choice([64, 128])}}, {{choice([6, 8])}},
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def build_sentence_model(self):
'''
Build the *sentence* level model, which operates over, erm, sentences.
The task is to predict which sentences are pos/neg rationales.
'''
tokens_input = Input(name='input', shape=(self.preprocessor.max_sent_len,), dtype='int32')
x = Embedding(self.preprocessor.max_features, self.preprocessor.embedding_dims,
input_length=self.preprocessor.max_sent_len,
weights=self.preprocessor.init_vectors)(tokens_input)
x = Dropout(0.1)(x)
convolutions = []
for n_gram in self.ngram_filters:
cur_conv = Convolution1D(nb_filter=self.nb_filter,
filter_length=n_gram,
border_mode='valid',
activation='relu',
subsample_length=1,
input_dim=self.preprocessor.embedding_dims,
input_length=self.preprocessor.max_sent_len)(x)
# pool
one_max = MaxPooling1D(pool_length=self.preprocessor.max_sent_len - n_gram + 1)(cur_conv)
flattened = Flatten()(one_max)
convolutions.append(flattened)
sentence_vector = merge(convolutions, name="sentence_vector") # hang on to this layer!
output = Dense(3, activation="softmax")(sentence_vector)
self.sentence_model = Model(input=tokens_input, output=output)
print("model built")
print(self.sentence_model.summary())
self.sentence_model.compile(loss='categorical_crossentropy', optimizer="adam")
self.sentence_embedding_dim = self.sentence_model.layers[-2].output_shape[1]
return self.sentence_model
def create_autoconv_lstm_network(num_frequency_dimensions, num_hidden_dimensions,
num_recurrent_units=1, stateful=False):
model = Sequential()
model.add(TimeDistributed(Convolution1D(32, 41, border_mode='same', name="1_conv1d", activation='relu'),
input_shape=(None, num_frequency_dimensions, 1), name="1_timedist"))
model.add(TimeDistributed(BatchNormalization()))
model.add(TimeDistributed(MaxPooling1D(2, border_mode='valid')))
model.add(TimeDistributed(Convolution1D(16, 17, activation='relu', border_mode='same')))
model.add(TimeDistributed(BatchNormalization()))
model.add(TimeDistributed(MaxPooling1D(2, border_mode='valid')))
model.add(TimeDistributed(Convolution1D(8, 3, activation='relu', border_mode='same')))
model.add(TimeDistributed(BatchNormalization()))
model.add(TimeDistributed(MaxPooling1D(2, border_mode='valid')))
last_2d_shape = model.output_shape[-2:]
output_num_rnn = last_2d_shape[0] * last_2d_shape[1]
model.add(TimeDistributed(Flatten()))
for cur_unit in xrange(num_recurrent_units):
model.add(LSTM(output_dim=num_hidden_dimensions, return_sequences=True, stateful=stateful))
model.add(TimeDistributed(Dense(input_dim=num_hidden_dimensions, output_dim=output_num_rnn)))
model.add(TimeDistributed(Reshape(last_2d_shape)))
model.add(TimeDistributed(Convolution1D(8, 3, activation='relu', border_mode='same')))
model.add(TimeDistributed(BatchNormalization()))
model.add(TimeDistributed(UpSampling1D(2)))
model.add(TimeDistributed(Convolution1D(16, 17, activation='relu', border_mode='same')))
model.add(TimeDistributed(BatchNormalization()))
model.add(TimeDistributed(UpSampling1D(2)))
model.add(TimeDistributed(Convolution1D(32, 41, activation='relu', border_mode='same')))
model.add(TimeDistributed(BatchNormalization()))
model.add(TimeDistributed(UpSampling1D(2)))
model.add(TimeDistributed(Convolution1D(1, 3, activation='tanh', border_mode='same')))
last_2d_shape = model.output_shape[-2]
model.add(TimeDistributed(Reshape((last_2d_shape,))))
model.compile(loss='mean_squared_error', optimizer='rmsprop')
return model
def cnn(input_shape, output_length):
""" Create and return a keras model of a CNN """
NB_FILTER = 256
NGRAM_LENGTHS = [1, 2, 3, 4, 5]
conv_layers = []
for ngram_length in NGRAM_LENGTHS:
ngram_layer = Sequential()
ngram_layer.add(
Convolution1D(
NB_FILTER,
ngram_length,
input_dim=input_shape,
input_length=SAMPLE_LENGTH,
init='lecun_uniform',
activation='tanh',
))
pool_length = SAMPLE_LENGTH - ngram_length + 1
ngram_layer.add(MaxPooling1D(pool_length=pool_length))
conv_layers.append(ngram_layer)
model = Sequential()
model.add(Merge(conv_layers, mode='concat'))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(output_length, activation='softmax'))
model.compile(
loss='categorical_crossentropy',
optimizer='adam',
metrics=['categorical_accuracy'],
)
return model
def train(index, epochs=11):
# make predictions
model = Sequential()
model.add(Convolution1D(4, 1, input_dim=1))
model.add(LSTM(4))
model.add(Activation('sigmoid'))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
trainX, trainY, scaler, dataset = create_data(index)
model.fit(trainX, trainY, nb_epoch=epochs, batch_size=1, verbose=2)
trainPredict = model.predict(trainX)
#testPredict = model.predict(testX)
# invert predictions
trainPredict = scaler.inverse_transform(trainPredict)
trainY = scaler.inverse_transform([trainY])
#testPredict = scaler.inverse_transform(testPredict)
#testY = scaler.inverse_transform([testY])
# calculate root mean squared error
trainScore = math.sqrt(mean_squared_error(trainY[0], trainPredict[:, 0]))
print('Train Score: %.2f RMSE' % (trainScore))
#testScore = math.sqrt(mean_squared_error(testY[0], testPredict[:,0]))
#print('Test Score: %.2f RMSE' % (testScore))
# shift train predictions for plotting
return model, dataset, trainPredict, scaler
def ifv5():
ii = [1682, 241]
idce = [
Embedding(ii[i], 8, input_length=1,
embeddings_regularizer=l2(0.01))(idc5[i]) for i in range(2)
]
idde = Embedding(64, 300)(idd5)
iddecl = Convolution1D(300,
1,
padding="same",
input_shape=(16, 300),
activation="tanh")(idde)
iddepl = Lambda(lambda x: backend.max(x, axis=1),
output_shape=(300, ))(iddecl)
idderl = Reshape((1, 300))(iddepl)
idbdl = Dense(8, activation="relu")(idb5)
idbrl = Reshape((1, 8))(idbdl)
y = Concatenate(axis=-1)(idce + [idderl] + [idbrl])
y = Flatten()(y)
y = BatchNormalization()(y)
iwd = concatenate([iwi5, y])
y = Dense(300,
activation='relu',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(iwd)
y = Dense(300,
activation='relu',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(iwd)
ifv = Dense(300,
activation='relu',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(iwd)
return ifv
def create_model(dropout, dim, optimizer='rmsprop'):
global train_length, readabilty_features_count
model = Sequential()
embedding_layer = create_embedding_matrix(word_index)
model.add(embedding_layer)
model.add(Dropout(dropout))
model.add(
Convolution1D(nb_filter=32,
filter_length=3,
border_mode='same',
activation='relu'))
model.add(MaxPooling1D(pool_length=2))
model.add(Dropout(dropout))
model.add(Bidirectional(LSTM(dim)))
# model_readabilty = Sequential()
# model_readabilty.add(Bidirectional(LSTM(readabilty_features_count), input_shape=(readabilty_features_count, 1)))
# Merge
# merged = Merge([model, model_readabilty], mode='concat')
# model_merged = Sequential()
# model_merged.add(merged)
# model_merged.add(Dropout(0.1))
# model_merged.add(Dense(2, activation='sigmoid'))
model.add(Dropout(dropout))
model.add(Dense(2, activation='sigmoid'))
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
return model
def rnn_training():
np.random.seed(7)
top_words = 5000
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=top_words)
max_review_length = 500
X_train = sequence.pad_sequences(X_train, maxlen=max_review_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_review_length)
# create the model
embedding_vecor_length = 32
model = Sequential()
model.add(
Embedding(top_words,
embedding_vecor_length,
input_length=max_review_length,
dropout=0.2))
model.add(Dropout(0.2))
#convolution, 1D since sequence data
model.add(
Convolution1D(nb_filter=32,
filter_length=3,
border_mode='same',
activation='relu'))
model.add(MaxPooling1D(pool_length=2))
#model.add(Embedding(top_words, embedding_vecor_length, input_length=max_review_length, dropout=0.2))
#model.add(LSTM(100, dropout_W=0.2, dropout_U=0.2))
model.add(LSTM(100))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
model.fit(X_train, y_train, nb_epoch=3, batch_size=64)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1] * 100))
def create_model_cnn(neurons=120,
dropout_rate=0.0,
weight_constraint=5,
activation='linear',
init_mode='lecun_uniform',
learn_rate=0.2,
momentum=0.4):
# create model
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded = embedding_layer(sequence_input)
embedded = Dropout(dropout_rate)(embedded)
# convolutional layer
convolution = Convolution1D(filters=nb_filter,
kernel_size=kernel_size,
padding='valid',
activation='relu',
strides=1)(embedded)
maxpooling = MaxPooling1D(pool_size=2)(convolution)
maxpooling = Flatten()(maxpooling)
# We add a vanilla hidden layer:
dense = Dense(neurons,
init=init_mode,
activation=activation,
W_constraint=maxnorm(weight_constraint))(
maxpooling) # best: 120
dense = Dropout(dropout_rate)(dense) # best: 0.25
dense = Activation('relu')(dense)
output = Dense(2, activation='sigmoid', init='lecun_uniform')(dense)
model = Model(inputs=sequence_input, outputs=output)
optimizer = SGD(lr=learn_rate, momentum=momentum)
model.compile(loss=[focal_loss([6066, 1987])],
optimizer=optimizer,
metrics=[f1])
return model
def get_32_to_1_model():
num_features = 52
model = Sequential()
model.add(
TimeDistributed(Convolution1D(128, 3, border_mode='valid'),
input_shape=(config['win_len'], 32, num_features)))
model.add(TimeDistributed(Activation('relu')))
model.add(TimeDistributed(Convolution1D(128, 3, border_mode='valid')))
model.add(TimeDistributed(MaxPooling1D(2)))
model.add(TimeDistributed(Convolution1D(256, 3, border_mode='valid')))
model.add(TimeDistributed(Activation('relu')))
model.add(TimeDistributed(Convolution1D(256, 3, border_mode='valid')))
model.add(TimeDistributed(Activation('relu')))
model.add(TimeDistributed(MaxPooling1D(2)))
model.add(TimeDistributed(Convolution1D(128, 3, border_mode='valid')))
model.add(TimeDistributed(Activation('relu')))
model.add(TimeDistributed(Convolution1D(64, 3, border_mode='valid')))
model.add(TimeDistributed(Activation('relu')))
return model
def create_model(model_type):
if (model_type == 'timedistributedcnn_lstm'):
seq_size = 1366
nb_features = 96
channels = 1
model = Sequential()
model.add(
TimeDistributed(Convolution1D(32, 3, activation='relu'),
input_shape=(seq_size, nb_features, channels)))
model.add(TimeDistributed(Convolution1D(32, 3, activation='relu')))
model.add(TimeDistributed(MaxPooling1D(2, 2)))
model.add(Dropout(0.25))
model.add(TimeDistributed(Convolution1D(64, 3, activation='relu')))
model.add(TimeDistributed(Convolution1D(64, 3, activation='relu')))
model.add(TimeDistributed(MaxPooling1D(2, 2)))
model.add(Dropout(0.25))
model.add(TimeDistributed(Convolution1D(64, 3, activation='relu')))
model.add(TimeDistributed(Convolution1D(64, 3, activation='relu')))
model.add(TimeDistributed(MaxPooling1D(2, 2)))
model.add(Dropout(0.25))
# model.summary()
model.add(TimeDistributed(Flatten()))
model.add(BatchNormalization())
model.add(LSTM(128, return_sequences=True))
model.add(LSTM(128, return_sequences=True))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(3, activation='sigmoid'))
#model.summary()
sgd = SGD(lr=0.01,
decay=1e-6,
momentum=0.9,
nesterov=True,
clipnorm=0.5)
#plt.title('model accuracy')
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['categorical_accuracy'])
return model
def build_stack(inputs, dense_outputs, nb_filter, maxlen, vocab_size,
filter_kernels):
conv = Convolution1D(nb_filter=nb_filter,
filter_length=filter_kernels[0],
border_mode='valid',
activation='relu',
input_shape=(maxlen, vocab_size))(inputs)
conv = MaxPooling1D(pool_length=3)(conv)
conv1 = Convolution1D(nb_filter=nb_filter,
filter_length=filter_kernels[1],
border_mode='valid',
activation='relu')(conv)
conv1 = MaxPooling1D(pool_length=3)(conv1)
conv2 = Convolution1D(nb_filter=nb_filter,
filter_length=filter_kernels[2],
border_mode='valid',
activation='relu')(conv1)
conv3 = Convolution1D(nb_filter=nb_filter,
filter_length=filter_kernels[3],
border_mode='valid',
activation='relu')(conv2)
conv4 = Convolution1D(nb_filter=nb_filter,
filter_length=filter_kernels[4],
border_mode='valid',
activation='relu')(conv3)
conv5 = Convolution1D(nb_filter=nb_filter,
filter_length=filter_kernels[5],
border_mode='valid',
activation='relu')(conv4)
conv5 = MaxPooling1D(pool_length=3)(conv5)
conv5 = Flatten()(conv5)
zz = Dropout(0.5)(Dense(dense_outputs, activation='relu')(conv5))
return zz
def build_cnn_architecture():
model = Sequential()
activation = 'relu'
model.add(Convolution1D(2, 9, input_shape=(500, 1), activation=activation))
model.add(AveragePooling1D())
model.add(BatchNormalization())
model.add(Convolution1D(2, 7, activation=activation))
model.add(AveragePooling1D())
model.add(BatchNormalization())
model.add(Convolution1D(4, 7, activation=activation))
model.add(AveragePooling1D())
model.add(BatchNormalization())
model.add(Convolution1D(4, 5, activation=activation))
model.add(AveragePooling1D())
model.add(BatchNormalization())
model.add(Convolution1D(8, 3, activation=activation))
model.add(AveragePooling1D())
model.add(BatchNormalization())
model.add(Dropout(0.10))
model.add(Convolution1D(3, 1))
model.add(GlobalAveragePooling1D())
model.add(Activation('softmax', name='loss'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
print("CNN Model created.")
return model
def build_model(spectral_input_size, temporal_input_size,
spectral_n_feature, temporal_n_feature,
conv_layers, dense_layers, init,
learning_rate, optimizer, pooling, dropout, atrous,
regularizer_conf, temporal, objective, penalty,
activation, last_layer, batch_size,
n_batch_per_file, lstm_dropout):
"""Build the model"""
spectral_input = Input(
batch_shape=(batch_size, spectral_input_size,
spectral_n_feature),
name='spectral_input')
temporal_input = Input(
batch_shape=(batch_size, temporal_input_size,
temporal_n_feature),
name='temporal_input')
pre_temporal_input = Input(
batch_shape=(batch_size, temporal_input_size,
temporal_n_feature),
name='pre_temporal_input')
n_conv = len(conv_layers)
n_dense = len(dense_layers)
nb_kernel = conv_layers[0][0]
he_kernel = conv_layers[0][1]
regularizer = None
if regularizer_conf['name'] == 'l1':
regularizer = l1(l=regularizer_conf['value'])
elif regularizer_conf['name'] == 'l2':
regularizer = l2(l=regularizer_conf['value'])
# spectral_conv = Conv1D(nb_kernel, he_kernel, strides=1, padding='valid',
# dilation_rate=1, activation=activation,
# use_bias=True,
# kernel_initializer=init,
# bias_initializer='zeros',
# kernel_regularizer=regularizer,
# bias_regularizer=None,
# activity_regularizer=None,
# kernel_constraint=None,
# bias_constraint=None)(spectral_input)
spectral_conv = Convolution1D(nb_kernel, he_kernel,
border_mode='valid',
activation=activation,
bias=True,
init=init,
W_regularizer=regularizer)(spectral_input)
# spectral_conv = MaxPooling1D(pool_length=2)(spectral_conv)
spectral_conv = Dropout(dropout)(spectral_conv)
# spectral_conv = MaxPooling1D(pool_size=2, strides=2,
# padding='valid')(spectral_conv)
# temporal_conv = Conv1D(nb_kernel, he_kernel, strides=1, padding='valid',
# dilation_rate=1, activation=activation,
# use_bias=True,
# kernel_initializer=init,
# bias_initializer='zeros',
# kernel_regularizer=regularizer,
# bias_regularizer=None,
# activity_regularizer=None,
# kernel_constraint=None,
# bias_constraint=None)(temporal_input)
temporal_conv = Convolution1D(nb_kernel, he_kernel,
border_mode='valid',
activation=activation,
bias=True,
init=init,
W_regularizer=regularizer)(temporal_input)
# temporal_conv = MaxPooling1D(pool_size=2, strides=2,
# padding='valid')(temporal_conv)
# temporal_conv = MaxPooling1D(pool_length=2)(temporal_conv)
temporal_conv = Dropout(dropout)(temporal_conv)
pre_temporal_conv = Convolution1D(nb_kernel, he_kernel,
border_mode='valid',
activation=activation,
bias=True,
init=init,
W_regularizer=regularizer)(pre_temporal_input)
# pre_temporal_conv = MaxPooling1D(pool_length=2)(pre_temporal_conv)
pre_temporal_conv = Dropout(dropout)(pre_temporal_conv)
for i in range(1, n_conv):
nb_kernel = conv_layers[i][0]
he_kernel = conv_layers[i][1]
regularizer = None
if regularizer_conf['name'] == 'l1':
regularizer = l1(l=regularizer_conf['value'])
elif regularizer_conf['name'] == 'l2':
regularizer = l2(l=regularizer_conf['value'])
# spectral_conv = Conv1D(nb_kernel, he_kernel, strides=1, padding='valid',
# dilation_rate=1, activation=activation,
# use_bias=True,
# kernel_initializer=init,
# bias_initializer='zeros',
# kernel_regularizer=None,
# bias_regularizer=None,
# activity_regularizer=regularizer,
# kernel_constraint=None,
# bias_constraint=None)(spectral_conv)
spectral_conv = Convolution1D(nb_kernel, he_kernel,
border_mode='valid',
activation=activation,
bias=True,
init=init,
W_regularizer=None)(
spectral_conv)
spectral_conv = Dropout(dropout)(spectral_conv)
# temporal_conv = Conv1D(nb_kernel, he_kernel, strides=1, padding='valid',
# dilation_rate=1, activation=activation,
# use_bias=True,
# kernel_initializer=init,
# bias_initializer='zeros',
# kernel_regularizer=None,
# bias_regularizer=None,
# activity_regularizer=regularizer,
# kernel_constraint=None,
# bias_constraint=None)(temporal_conv)
temporal_conv = Convolution1D(nb_kernel, he_kernel,
border_mode='valid',
activation=activation,
bias=True,
init=init,
W_regularizer=None)(
temporal_conv)
# temporal_conv = MaxPooling1D(pool_size=2, strides=2,
# padding='valid')(temporal_conv)
# if i < 3:
# temporal_conv = MaxPooling1D(pool_length=2)(temporal_conv)
temporal_conv = Dropout(dropout)(temporal_conv)
pre_temporal_conv = Convolution1D(nb_kernel, he_kernel,
border_mode='valid',
activation=activation,
bias=True,
init=init,
W_regularizer=None)(
pre_temporal_conv)
# if i < 3:
# pre_temporal_conv = MaxPooling1D(pool_length=2)(pre_temporal_conv)
pre_temporal_conv = Dropout(dropout)(pre_temporal_conv)
# spectral_conv = GRU(dense_layers[0], activation='tanh',
# recurrent_activation='hard_sigmoid',
# use_bias=True,
# kernel_initializer='glorot_uniform',
# recurrent_initializer='orthogonal',
# bias_initializer='zeros',
# kernel_regularizer=None,
# recurrent_regularizer=None,
# bias_regularizer=None,
# activity_regularizer=None,
# kernel_constraint=None,
# recurrent_constraint=None,
# bias_constraint=None,
# dropout=0.0,
# stateful=True,
# implementation=0,
# recurrent_dropout=lstm_dropout)(spectral_conv)
# temporal_conv = GRU(dense_layers[0], activation='tanh',
# recurrent_activation='hard_sigmoid',
# use_bias=True,
# kernel_initializer='glorot_uniform',
# recurrent_initializer='orthogonal',
# bias_initializer='zeros',
# kernel_regularizer=None,
# recurrent_regularizer=None,
# bias_regularizer=None,
# activity_regularizer=None,
# kernel_constraint=None,
# recurrent_constraint=None,
# bias_constraint=None,
# dropout=0.0,
# stateful=True,
# implementation=0,
# recurrent_dropout=lstm_dropout)(temporal_conv)
spectral_conv = Flatten()(spectral_conv)
temporal_conv = Flatten()(temporal_conv)
pre_temporal_conv = Flatten()(pre_temporal_conv)
# merged_conv = concatenate([spectral_conv, temporal_conv])
merged_conv = merge([spectral_conv, temporal_conv, pre_temporal_conv],
mode='concat')
for i in range(0, n_dense):
regularizer = None
if regularizer_conf['name'] == 'l1':
regularizer = l1(l=regularizer_conf['value'])
elif regularizer_conf['name'] == 'l2':
regularizer = l2(l=regularizer_conf['value'])
# merged_conv = Dense(dense_layers[i],
# activation=activation, kernel_initializer=init,
# activity_regularizer=None, use_bias=True,
# bias_initializer='zeros',
# kernel_regularizer=regularizer,
# bias_regularizer=None,
# kernel_constraint=None, bias_constraint=None
# )(merged_conv)
merged_conv = Dense(dense_layers[i],
activation=activation,
init=init, W_regularizer=l2(l=1e-02))(
merged_conv)
merged_conv = Dropout(lstm_dropout)(merged_conv)
last_regularizer = None
if last_layer['regularization']['name'] == 'l1':
last_regularizer = l1(l=last_layer['regularization']['value'])
elif last_layer['regularization']['name'] == 'l2':
last_regularizer = l2(l=last_layer['regularization']['value'])
# output = Dense(1, activation=last_layer['activation'],
# kernel_initializer=init,
# activity_regularizer=None, use_bias=True,
# bias_initializer='zeros',
# kernel_regularizer=last_regularizer,
# bias_regularizer=None,
# kernel_constraint=None, bias_constraint=None,
# name='output')(merged_conv)
output = Dense(1, activation=last_layer['activation'],
init=init,
W_regularizer=last_regularizer,name='output')(merged_conv)
# model = Model(inputs=[spectral_input, temporal_input],
# outputs=[output])
model = Model(input=[spectral_input, temporal_input,
pre_temporal_input],
output=output)
compile_model(model, objective, penalty, learning_rate, optimizer)
model_structure = ''
print(model.summary())
return [model_structure, model]
for
trainFeatures = sequence.pad_sequences(trainFeatures,maxlen=MAX_LENGTH)
trainTargets = np.asarray(trainTargets)
devFeatures = sequence.pad_sequences(devFeatures,maxlen=MAX_LENGTH)
devTargets = np.asarray(devTargets)
testFeatures = sequence.pad_sequences(testFeatures,maxlen=MAX_LENGTH)
testTargets = np.asarray(testTargets)
model = Sequential()
model.add(Embedding(nFeatures, embeddingSize,input_length=MAX_LENGTH))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=2))
model.add(LSTM(70))
model.add(Dense(70,activation='sigmoid'))
model.add(Dense(3))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam', metrics=['accuracy'])
print("Entrenando Modelo")
model.fit(trainFeatures, trainTargets, batch_size=batch_size, nb_epoch=1,
Y_train = Y_train.astype(theano.config.floatX)
X_test = X_test.astype(theano.config.floatX)
Y_test = Y_test.astype(theano.config.floatX)
from keras.models import Sequential
from keras.layers.core import Dense, Flatten, Dropout
from keras.layers.convolutional import Convolution1D
from keras.optimizers import SGD
np.random.seed(1)
model = Sequential()
model.add(
Convolution1D(32,
20,
border_mode="valid",
input_shape=(200, 4),
subsample_length=2,
activation="relu"))
model.add(Dropout(0.25))
model.add(
Convolution1D(128,
20,
border_mode="valid",
subsample_length=2,
activation="relu"))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation="sigmoid"))
embedding = Embedding(max_features,
embeddings_dim,
input_length=max_sent_len,
mask_zero=False,
weights=[embedding_weights])(main_input)
Drop1 = Dropout(dropout_prob[0])(embedding)
i = 0
conv_name = ["" for x in range(len(filter_sizes))]
pool_name = ["" for x in range(len(filter_sizes))]
flat_name = ["" for x in range(len(filter_sizes))]
for n_gram in filter_sizes:
conv_name[i] = str('conv_' + str(n_gram))
conv_name[i] = Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode='valid',
activation='relu',
subsample_length=1,
input_dim=embeddings_dim,
input_length=max_sent_len)(Drop1)
pool_name[i] = str('maxpool_' + str(n_gram))
pool_name[i] = MaxPooling1D(pool_length=max_sent_len - n_gram + 1)(
conv_name[i])
flat_name[i] = str('flat_' + str(n_gram))
flat_name[i] = Flatten()(pool_name[i])
i += 1
merged = merge([flat_name[0], flat_name[1], flat_name[2]], mode='concat')
droput_final = Dropout(dropout_prob[1])(merged)
Dense_final = Dense(1,
input_dim=nb_filter * len(filter_sizes))(droput_final)
Out = Dense(num_classes, activation='sigmoid')(Dense_final)
model = Model(inputs=main_input, outputs=Out)
def get_cnn_kmodel(size, image_dim):
"""
Generates CNN model using keras
:return: return generated model
"""
model = Sequential()
# add convolution layers
x = 32
# for i in range(1):
# model.add(Convolution2D(x, 3, 3, border_mode='full', activation='relu',
# input_shape=(image_dim, size, size)))
# model.add(Convolution2D(x, 2, 2, border_mode='full', activation='relu'))
# model.add(Convolution2D(x, 2, 2, border_mode='full', activation='relu'))
#
# model.add(MaxPooling2D(pool_size=(2, 2)))
# x *= 2
# for i in range(1):
# model.add(Convolution2D(x, 3, 3, border_mode='full',
# activation='relu',
# input_shape=(image_dim, size, size)))
# model.add(Convolution2D(x, 2, 2,activation='relu'))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# x *= 2
#
# for i in range(3):
# model.add(Convolution2D(x, 3, 3, border_mode='valid', activation='relu',
# input_shape=(image_dim, size, size)))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# x *= 2
# for i in range(1):
# model.add(Convolution1D(x, 3, border_mode='full',
# activation='relu',
# input_shape=(1,size)))
# model.add(Convolution1D(x, 2,border_mode='full',activation='relu'))
# model.add(Convolution1D(x, 2,border_mode='full',activation='relu'))
# model.add(MaxPooling1D(pool_length=2))
# x *= 2
# for i in range(1):
# model.add(Convolution1D(x, 3, border_mode='full',
# activation='relu',
# input_shape=(1,size)))
# model.add(Convolution1D(x, 2,border_mode='full',activation='relu'))
# model.add(MaxPooling1D(pool_length=2))
# x *= 2
for i in range(3):
model.add(
Convolution1D(x,
3,
border_mode='full',
activation='relu',
input_shape=(image_dim, size)))
model.add(MaxPooling1D(pool_length=2))
x *= 2
# start adding dense layers
model.add(Flatten())
# final output layer
model.add(Dense(5, activation='softmax'))
sgd = SGD(lr=0.1, momentum=0.9, decay=1e-7, nesterov=True)
# sgd = Adam(lr=0.005, beta_1=0.85, beta_2=0.99, epsilon=1e-5, verbose=1)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
# 'rmsprop'
return model
