selected = int(0.4 * len(out))
trainTextsSeq_Fr = out[:selected, :-1]
y_train_Fr = out[:selected, -1]
embedding_vector_length = 100
class_names = ['0', '1']
# Building the model on the entire dataset and testing it on the test data
runs = []
for run in range(1):
myInput = Input(shape=(max_rep_length, ),
name='input_English') # 500 vectors of size 100
x = Embedding(output_dim=100,
input_dim=vocabSize_Fr,
input_length=max_rep_length)(myInput)
CNN_h1 = Conv1D(200, kernel_size=4, strides=1, activation='relu')(x)
h1 = MaxPooling1D(pool_size=2, strides=None, padding='same')(CNN_h1)
CNN_h2 = Conv1D(200, kernel_size=3, strides=1, activation='relu')(h1)
h2 = MaxPooling1D(pool_size=2, strides=None, padding='same')(CNN_h2)
CNN_h3 = Conv1D(200, kernel_size=3, strides=1, activation='relu')(h2)
h3 = MaxPooling1D(pool_size=2, strides=None, padding='same')(CNN_h3)
CNN_h4 = Conv1D(200, kernel_size=3, strides=1, activation='relu')(h3)
h5 = MaxPooling1D(pool_size=2, strides=None, padding='same')(CNN_h4)
f = Flatten()(h1)
predictions = Dense(2, activation='softmax', name='pred_eng')(f)
model = Model(myInput, outputs=predictions)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
ytest = np.array([0 for _ in range(num if num else 12500)] +
[1 for _ in range(num if num else 12500)])
ytest = to_categorical(ytest, num_classes=2)
# Using pre-trained word embedding
print("Loading Word Embedding")
wordlist = load_emb(embedding_dir, tokenizer.word_index)
embedding_layer = Embedding(vocab_size,
300,
weights=[wordlist],
input_length=max_length,
trainable=False)
# Define model
model = Sequential()
model.add(embedding_layer)
model.add(Conv1D(filters=128, kernel_size=5, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(2, activation='softmax'))
model.summary()
# Compile network
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Fit network
model.fit(Xtrain, ytrain, epochs=20, verbose=1)
# evaluate
loss, acc = model.evaluate(Xtest, ytest, verbose=0)
print('Test Accuracy: %f' % (acc * 100))
print(i, "th Fold *****************************************")
i = i + 1
x_train_clinical, x_test_clinical = X_clinical[train_index], X_clinical[
test_index]
y_train_clinical, y_test_clinical = Y_clinical[train_index], Y_clinical[
test_index]
x_train_clinical = numpy.expand_dims(x_train_clinical, axis=2)
x_test_clinical = numpy.expand_dims(x_test_clinical, axis=2)
# first Clinical CNN Model
init = initializers.glorot_normal(seed=1)
bias_init = initializers.Constant(value=0.1)
main_input1 = Input(shape=(25, 1), name='Input')
conv1 = Conv1D(filters=25,
kernel_size=15,
strides=2,
activation='tanh',
padding='same',
name='Conv1D',
kernel_initializer=init,
bias_initializer=bias_init)(main_input1)
flat1 = Flatten(name='Flatten')(conv1)
dense1 = Dense(150,
activation='tanh',
name='dense1',
kernel_initializer=init,
bias_initializer=bias_init)(flat1)
output = Dense(1,
activation='sigmoid',
name='output',
activity_regularizer=regularizers.l2(0.01),
kernel_initializer=init,
bias_initializer=bias_init)(dense1)
# -+-+-+-+-+-+-+- BUILDING MODEL -+-+-+-+-+-+-+-
print("BUILDING MODEL")
embedding_vecor_length = 32
input_layer = Embedding(len(tokenizer.word_index) + 1,
global_emb_dim,
weights=[emb_matrix],
input_length=global_max_seq,
trainable=False)
branch_3 = Sequential()
branch_3.add(input_layer)
branch_3.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
kernel_regularizer=l2(.01)))
branch_3.add(Activation('relu'))
branch_3.add(MaxPooling1D(pool_size=2))
branch_3.add(Dropout(0.5))
branch_3.add(BatchNormalization())
branch_3.add(LSTM(100))
branch_4 = Sequential()
branch_4.add(input_layer)
branch_4.add(
Conv1D(filters=32,
kernel_size=4,
padding='same',
kernel_regularizer=l2(.01)))
branch_4.add(Activation('relu'))
D = [44, 40, 45, 70, 20, 67, 36, 43, 65, 63]
for NConv1, NConv2, NJanet1, NJanet2, NDense in zip(C1, C2, J1, J2, D):
acc_list = []
prec_list = []
recall_list = []
f1_list = []
for i in range(repetition):
early_stopping = EarlyStopping(monitor='val_acc', patience=patience)
checkpoint = ModelCheckpoint('6dmg_try.h5',
monitor='val_acc',
verbose=0,
save_weights_only=True)
nadam = Nadam(lr=1e-4, beta_1=0.9, beta_2=0.9)
model3 = Sequential()
model3.add(Conv1D(NConv1, 5, activation='relu', input_shape=(240, 6)))
model3.add(Conv1D(NConv2, 5, activation='relu'))
model3.add(JANET(NJanet1, activation='relu',
return_sequences=True)) #, input_shape=(None, 6)))
model3.add(JANET(NJanet2, activation='relu'))
model3.add(Dense(NDense, activation='relu'))
model3.add(Dense(20, activation='softmax'))
model3.compile(loss='categorical_crossentropy',
optimizer=nadam,
metrics=['accuracy'])
model3.fit(xTrain,
yTrain,
epochs=epochs,
validation_data=(xVal, yVal),
callbacks=[early_stopping],
verbose=1,
def train_cnn_lstm(x_train, x_test, y_train, y_test, embedding_matrix,
max_length):
"""
Function that trains a CNN-LSTM model using the pre-trained skip-gram embedding
Inputs:
x_train: np array
x_test: np array
y_train: np array
y_test: np array
embedding_matrix: np array of shape num_words * dim_embedding
max_length: int - maximum length of the sequence (in words)
"""
kfold = StratifiedKFold(n_splits=5, shuffle=False)
es = EarlyStopping(monitor='val_loss', patience=5, min_delta=0.01)
num_words = embedding_matrix.shape[0]
dim_embedding = embedding_matrix.shape[1]
accs = []
aucs = []
for train, valid in kfold.split(x_train, y_train):
model = Sequential()
embedding_layer = Embedding(input_dim=num_words,
output_dim=dim_embedding,
weights=[embedding_matrix],
input_length=max_length,
trainable=False)
model.add(embedding_layer)
model.add(Dropout(0.25))
model.add(
Conv1D(filters=10,
kernel_size=2,
padding='same',
activation='relu'))
model.add(
Conv1D(filters=10,
kernel_size=3,
padding='same',
activation='relu'))
model.add(Dropout(0.5))
model.add(MaxPooling1D(pool_size=4))
model.add(
LSTM(units=256,
recurrent_dropout=0.5,
recurrent_regularizer=regularizers.l2(0.01)))
model.add(Dense(1, activation='sigmoid')) # binary classification
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = model.fit(x_train[train],
y_train[train],
validation_data=(x_train[valid], y_train[valid]),
callbacks=[es],
epochs=10,
batch_size=128,
verbose=0)
score = model.evaluate(x_test, y_test, verbose=0)
accs.append(score[1])
y_proba = model.predict_proba(
x_test) # get the classification probs on the test set
# Compute ROC curve
fpr, tpr, thresholds = roc_curve(y_test, y_proba, pos_label=1)
# Compute ROC area
roc_auc = auc(fpr, tpr)
aucs.append(roc_auc)
# plot training history
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
plt.show()
return history, accs, aucs, model
sequence_length = 17
max_review_length = 17
# input dim
vocabulary_size = len(vocabulary_inv_train)
# output dim
embedding_vector_length = 220
model = Sequential()
model.add(
Embedding(vocabulary_size,
embedding_vector_length,
input_length=max_review_length))
model.add(
Conv1D(filters=128, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(
Conv1D(filters=96, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(
Conv1D(filters=64, kernel_size=3, padding='same', activation='relu'))
# print(model.summary())
model.add(MaxPooling1D(pool_size=1))
model.add(
Bidirectional(
LSTM(250,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True)))
model.add(
sequence_length = 5
max_review_length = 5
# input dim
vocabulary_size = len(vocabulary_inv_train)
# output dim
embedding_vector_length = 20
model = Sequential()
model.add(
Embedding(vocabulary_size,
embedding_vector_length,
input_length=max_review_length))
model.add(
Conv1D(filters=150, kernel_size=20, padding='same', activation='relu'))
model.add(
Bidirectional(
LSTM(250,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True)))
model.add(
Bidirectional(
LSTM(350,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True)))
model.add(Bidirectional(LSTM(250, dropout=0.2)))
model.add(Dense(50, activation='softmax'))
from scipy import fromstring, int16
import numpy as np
import os.path
from keras.models import Sequential, load_model
from keras.layers import Dense, Flatten, Reshape
from keras.layers.noise import GaussianNoise
from keras.layers.convolutional import Conv1D, UpSampling1D
from keras.layers.pooling import MaxPooling1D
from keras.layers.normalization import BatchNormalization
from keras import backend as K
from keras.layers.advanced_activations import LeakyReLU
if os.path.exists(par.l1_encoder_filename):
model = load_model(par.l1_encoder_filename)
else:
model = Sequential()
model.add(Conv1D(par.l1_first_filters, 21, strides=par.l1_first_strides, padding='same', input_shape=(par.l1_input_length, par.l1_channel_size)))
model.add(Conv1D(par.l1_second_filters, 14, strides=par.l1_second_strides, padding='same'))
model.add(UpSampling1D(par.l1_second_strides))
model.add(Conv1D(par.l1_second_filters, 14, padding='same'))
model.add(UpSampling1D(par.l1_first_strides))
model.add(Conv1D(par.l1_first_filters, 21, padding='same'))
model.add(Conv1D(par.l1_channel_size, 8, padding='same'))
train_filename = par.l1_train_filename
epochs = par.l1_epochs
encoder_file = par.l1_encoder_filename
encoder = K.function([model.layers[0].input], [model.layers[1].output])
decoder = K.function([model.layers[2].input], [model.layers[6].output])
y = np.load('../output/y_data.npy')
pos_ix = [i for i in range(len(y)) if y[i] == 1]
neg_ix = [i for i in range(len(y)) if y[i] == 0]
train_ix = range(250)
train_set = [pos_ix[ix] for ix in train_ix] + [neg_ix[ix] for ix in train_ix]
x_train = x[train_set]
y_train = y[train_set]
for i in range(2):
neural_net = Sequential()
neural_net.add(Conv1D(nb_filter=10, filter_length=5, input_shape=(17, 4)))
neural_net.add(Flatten())
neural_net.add(Dense(10))
neural_net.add(Activation('relu'))
neural_net.add(Dense(1))
neural_net.add(Activation('sigmoid'))
neural_net.compile(loss='mse', optimizer='sgd')
neural_net.fit(x=x_train, y=y_train, batch_size=32, nb_epoch=250)
# read in sequences from rap1-lieb-test.txt
filepath = '../data/rap1-lieb-test.txt'
file_text = open(filepath)
sequences = []
for line in file_text:
sequence = line.strip()
sequences.append(sequence)
X_train = sequence.pad_sequences(X_train, maxlen=max_review_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_review_length)
print("")
# -+-+-+-+-+-+-+- BUILDING MODEL -+-+-+-+-+-+-+-
print("BUILDING MODEL")
embedding_vecor_length = 32
input_layer = Embedding(top_words,
embedding_vecor_length,
input_length=max_review_length)
branch_3 = Sequential()
branch_3.add(input_layer)
branch_3.add(Conv1D(filters=32, kernel_size=3, padding='same'))
branch_3.add(Activation('relu'))
branch_3.add(MaxPooling1D(pool_size=2))
branch_3.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
branch_4 = Sequential()
branch_4.add(input_layer)
branch_4.add(Conv1D(filters=32, kernel_size=4, padding='same'))
branch_4.add(Activation('relu'))
branch_4.add(MaxPooling1D(pool_size=2))
branch_4.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
branch_5 = Sequential()
branch_5.add(input_layer)
branch_5.add(Conv1D(filters=32, kernel_size=5, padding='same'))
branch_5.add(Activation('relu'))
print("y_train_ori:", y_train_ori.sum())
y_train = np_utils.to_categorical(y_train_ori, num_classes=2)
y_test = np_utils.to_categorical(y_test_ori, num_classes=2)
print("Building model")
model_input = Input(shape=(int(SAMPLE_LENGTH / 10), 2)) # PD = 2, LCP = 3
# x= Bidirectional(LSTM(hi, return_sequences=True), merge_mode='concat')(model_input)
rnn_feature = Bidirectional(LSTM(hidden_num,
return_sequences=True))(model_input)
pooled_outputs = []
for filter_size in FILTER_SIZES:
x = Conv1D(NUM_FILTERS * 2, filter_size, padding='same')(rnn_feature)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# x = AveragePooling1D(int(x.shape[1]))(x)
# x = GlobalAveragePooling1D()(x)
x = AttLayer()(x)
x = Reshape((1, -1))(x)
pooled_outputs.append(x)
merged = concatenate(pooled_outputs)
x = Flatten()(merged)
x = BatchNormalization()(x)
x = Dense(NUM_FILTERS)(x)
x = BatchNormalization()(x)
# x = Activation('relu')(x)
model_output = Dense(num_class, activation='softmax')(x)
def build_model(vector_size):
#batch_size = 512
nb_epochs = 5
model = Sequential()
# This f*****g block will eat all of your goddamn memory
# Sacrifice 3 chickens to improve accuracy
# sparse regularization
model.add(
Conv1D(32,
kernel_size=3,
kernel_regularizer=regularizers.l2(0.005),
activation='relu',
padding='same',
input_shape=(vector_size, 3)))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Dropout(0.5))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Dropout(0.5))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Dropout(0.5))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=1, activation='relu', padding='same'))
model.add(Conv1D(32, kernel_size=1, activation='relu', padding='same'))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(64, activation='tanh'))
model.add(Dense(64, activation='tanh'))
model.add(Dense(1, activation='sigmoid'))
# Compile the model
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=0.001, decay=1e-6),
metrics=['accuracy'])
return model
half_train_length = int(round(len(train) / 2))
half_test_length = int(round(len(test) / 2))
# define train and test labels
y_train = array([0 for _ in range(half_train_length)] +
[1 for _ in range(half_train_length)])
y_test = array([0 for _ in range(half_test_length)] +
[1 for _ in range(half_test_length)])
# define vocab size (+1 for unknown words)
vocab_size = len(tokenizer.word_index) + 1
# define model
model = Sequential()
model.add(Embedding(vocab_size, 100, input_length=max_length))
model.add(Conv1D(filters=32, kernel_size=8, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
print(model.summary())
# save model
model.save('models/embedded_conv_1d')
# create dict of train and test data
data_dict = {'x_train': x_train, 'y_train': y_train,
'x_test': x_test, 'y_test': y_test}
# serialize data dict
print('saving data dict to pickle...')
def create_model():
"""Creates the neural network.
Returns:
neural network model, optimizer
"""
dense_regularizer = L1L2(l2=0.0005)
audio_input = Input(shape=(1, 128, 301), dtype="float64")
audio = Conv1D(32,
kernel_size=(128, 8),
padding="same",
activation="relu",
name="Conv1D_32/128/8_relu")(audio_input)
audio = MaxPooling1D(pool_size=(1, 4), name="MaxPooling1D_1/4_1")(audio)
audio = Conv1D(32,
kernel_size=(1, 8),
padding="same",
activation="relu",
name="Conv1D_32/1/8_relu")(audio)
audio = MaxPooling1D(pool_size=(1, 2), name="MaxPooling1D_1/2_2")(audio)
audio = Conv1D(32,
kernel_size=(1, 4),
padding="same",
activation="relu",
name="Conv1D_32/1/4_relu")(audio)
audio = MaxPooling1D(pool_size=(1, 2), name="MaxPooling1D_1/2_3")(audio)
audio = Dense(1024,
activation="relu",
name="Dense_1024",
kernel_regularizer=dense_regularizer)(audio)
audio = Dense(512,
activation=" relu",
name=" Dense_512",
kernel_regularizer=dense_regularizer)(audio)
audio_output = audio
cnn_audio_model = Model(audio_input, audio_output)
input_audio = Input(shape=(128, 301))
input_image = Input(shape=(1, 2048))
input_time = Input(shape=(1))
list_output_audio = K.map_fn(lambda feature: cnn_audio_model, input_audio)
#element wise
output_audio = np.maximum.reduce(list_output_audio)
merged = Concatenate([input_image, output_audio, input_time])
X_input = Input(merged)
X = Dense(100,
activation="relu",
name="Dense_100_1",
kernel_regularizer=dense_regularizer)(X_input)
X = Dense(100,
activation=" relu",
name=" Dense_100_2",
kernel_regularizer=dense_regularizer)(X)
X_output = X
return Model([input_audio, input_image, input_time], X_output)
#getting the vocabulary of data sentences = tokenizer.texts_to_sequences(sentences) padded_docs= pad_sequences(sentences,maxlen=max_review_len) le = preprocessing.LabelEncoder() y = le.fit_transform(y) X_train, X_test, y_train, y_test = train_test_split(padded_docs, y, test_size=0.25, random_state=1000) vocab_size= len(tokenizer.word_index)+1 num_classes = y_test.shape[0] model = Sequential() model.add(layers.Embedding(vocab_size, 50, input_length=max_review_len)) #model.add(layers.Flatten()) model.add(Conv1D(32, 3, activation='relu', padding='same')) model.add(Dropout(0.2)) model.add(Conv1D(64, 3, activation='relu', padding='same')) model.add(MaxPooling1D(pool_size=1)) model.add(Conv1D(128, 3, activation='relu', padding='same')) model.add(Dropout(0.2)) model.add(Conv1D(128, 3, activation='relu', padding='same')) model.add(MaxPooling1D(pool_size=1)) # flattening the matrix into vector form model.add(Flatten()) model.add(Dropout(0.2)) model.add(Dense(1024, activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.2))
from keras.layers.convolutional import MaxPooling1D
x = array([[10, 20, 30], [20, 30, 40], [30, 40, 50], [40, 50, 60]])
y = array([40, 50, 60, 70])
x = x.reshape((x.shape[0], x.shape[1], 1))
print("x.shape[0]", x.shape[0])
print("x.shape[1]", x.shape[1])
print("x.shape", x.shape)
print("y.shape", y.shape)
# define model
model = Sequential()
model.add(
Conv1D(filters=64, kernel_size=2, activation='relu', input_shape=(3, 1)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(50, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
model.summary()
#fit model
model.fit(x, y, epochs=1000, verbose=0)
#demonstrate prediction
x_input = array([60, 70, 90])
x_input = x_input.reshape(1, 3, 1)
# univariate cnn-lstm example from numpy import array from keras.models import Sequential from keras.layers import LSTM from keras.layers import Dense from keras.layers import Flatten from keras.layers import TimeDistributed from keras.layers.convolutional import Conv1D from keras.layers.convolutional import MaxPooling1D # define dataset X = array([[10, 20, 30, 40], [20, 30, 40, 50], [30, 40, 50, 60], [40, 50, 60, 70]]) y = array([50, 60, 70, 80]) # reshape from [samples, timesteps] into [samples, subsequences, timesteps, features] X = X.reshape((X.shape[0], 2, 2, 1)) # define model model = Sequential() model.add(TimeDistributed(Conv1D(filters=64, kernel_size=1, activation='relu'), input_shape=(None, 2, 1))) model.add(TimeDistributed(MaxPooling1D(pool_size=2))) model.add(TimeDistributed(Flatten())) model.add(LSTM(50, activation='relu')) model.add(Dense(1)) model.compile(optimizer='adam', loss='mse') # fit model model.fit(X, y, epochs=500, verbose=0) # demonstrate prediction x_input = array([50, 60, 70, 80]) x_input = x_input.reshape((1, 2, 2, 1)) yhat = model.predict(x_input, verbose=0) print(yhat)
def build_decoder(self, latent, name='conv', n_layers=1):
decoder = latent
if name == 'conv':
if n_layers == 3:
decoder = Dense(self.window_size // 64 * 256,
activation='relu',
name='decoder_dense1')(decoder)
decoder = Reshape((self.window_size // 64, 256),
name='decoder_reshape1')(decoder)
decoder = UpSampling1D(4, name='decoder_upsample1')(decoder)
decoder = Conv1D(128,
3,
padding='same',
activation='relu',
name='decoder_conv1')(decoder)
decoder = UpSampling1D(4, name='decocer_upsample2')(decoder)
decoder = Conv1D(64,
3,
padding='same',
activation='relu',
name='decoder_conv2')(decoder)
if n_layers == 2:
decoder = Dense(self.window_size // 16 * 128,
activation='relu',
name='decoder_dense1')(decoder)
decoder = Reshape((self.window_size // 16, 128),
name='decoder_reshape1')(decoder)
decoder = UpSampling1D(4, name='decocer_upsample2')(decoder)
decoder = Conv1D(64,
3,
padding='same',
activation='relu',
name='decoder_conv2')(decoder)
if n_layers == 1:
decoder = Dense(self.window_size // 4 * 64,
activation='relu',
name='decoder_dense1')(decoder)
decoder = Reshape((self.window_size // 4, 64),
name='decoder_reshape1')(decoder)
decoder = UpSampling1D(4, name='decoder_upsample3')(decoder)
decoder = Conv1D(4, 1, padding='same',
name='decoder_conv3')(decoder)
decoder = Lambda(lambda x: K.softmax(x, axis=-1),
name='output_softmax')(decoder)
decoder = Lambda(lambda x: K.mean(K.reshape(
x, (-1, self.n_sampler, self.window_size, self.n_channels)),
axis=1),
name='output_mean')(decoder)
elif name == 'mlp':
if n_layers >= 2:
decoder = Dense(128, activation='relu',
name='decoder_dense2')(decoder)
decoder = Dense(self.window_size * self.n_channels,
name='decoder_dense3')(decoder)
decoder = Lambda(lambda x: K.softmax(x, axis=-1),
name='output_softmax')(decoder)
decoder = Lambda(lambda x: K.mean(K.reshape(
x, (-1, self.n_sampler, self.window_size, self.n_channels)),
axis=1),
name='output_mean')(decoder)
elif name == 'lstm':
decoder = LSTM(64, name='encoder_lstm1',
return_sequences=True)(decoder)
return decoder
label, sent = line.strip().split('\t')
ys.append(label)
words = [w.lower() for w in nltk.word_tokenize(sent)]
wids = [word2index[w] for w in words]
xs.append(wids)
fin.close()
X = pad_sequences(xs, maxlen=maxlen)
Y = np_utils.to_categorical(ys)
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.3, random_state=42)
print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape)
model = Sequential()
model.add(Embedding(input_dim=vocab_sz, output_dim=EMBED_SIZE, input_length=maxlen))
model.add(SpatialDropout1D(0.2))
model.add(Conv1D(filters=NUM_FILTERS, kernel_size=NUM_WORDS, activation='relu'))
model.add(GlobalMaxPooling1D())
model.add(Dense(2, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, verbose=VERBOSE, validation_data=[Xtest, Ytest])
plt.subplot(211)
plt.title("accuracy")
plt.plot(history.history['acc'], color='r', label='train')
plt.plot(history.history['val_acc'], color='b', label='validation')
plt.legend(loc='best')
plt.subplot(212)
plt.title('loss')
plt.plot(history.history['loss'], color='r', label='train')
def simple_LSTM(X_train_reshape, y_train_reshape_matrix,X_val_reshape, y_val_reshape_matrix,vector_size,model_batch_size,no_epochs):
# We usually match up the size of the embedding layer output with the number of hidden layers in the LSTM cell.
# https://adventuresinmachinelearning.com/keras-lstm-tutorial/
# hidden_size = 4096
# hidden_size = 32
hidden_size = 512
from keras.models import Model
from keras.layers import Input, Dense
from keras.layers import TimeDistributed
from keras.layers import Conv1D, Dropout, Dense, Flatten, LSTM, MaxPooling1D, Bidirectional
from keras.optimizers import Adam
from keras.layers import Flatten
DROP_RATE_DENSE = 0.1
#perahaps i want to use a combinatinos of convolutional and lstm?
# https://github.com/jatinmandav/Neural-Networks/blob/master/Sentiment-Analysis/universal-sentence-encoder/universal_sentence_encoder_sentiment-analysis.ipynb
####################### MODEL 1: ~ 30% accuracy ####################################
# use variable length timesteps using None
# https://datascience.stackexchange.com/questions/26366/training-an-rnn-with-examples-of-different-lengths-in-keras
# model = Sequential()
# timesteps = None
# data_dim = X_train_reshape.shape[2]
#
# model = Sequential()
# input = Input(shape=(timesteps, data_dim))
# model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)),merge_mode='concat')(input) #accuracy = 0.316
# model = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(input) # accuracy = 0.279
# # model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)),merge_mode='concat')(model) # 3) added this 0.21259
# # model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)), merge_mode='concat')(model)
# model = Dropout(DROP_RATE_DENSE)(model)
#
# model = TimeDistributed(Dense(256, activation='relu'))(model) #accuracy after adding this line =0.318
# # model = Flatten(input_shape=(timesteps, data_dim))(model)
# # model = Dense(100, activation='relu')(model)
# model = Dropout(DROP_RATE_DENSE)(model)
#
# output = Dense(y_train_reshape_matrix.shape[2], activation='softmax')(model)
# model = Model(input, output)
# model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy'])
####################### MODEL 1: ####################################
####################### MODEL 2: ~ 30% accuracy ####################################
# use variable length timesteps using None
# https://datascience.stackexchange.com/questions/26366/training-an-rnn-with-examples-of-different-lengths-in-keras
model = Sequential()
timesteps = None
data_dim = X_train_reshape.shape[2]
vector_size = 512
model = Sequential()
input = Input(shape=(timesteps, data_dim))
model = Conv1D(32, kernel_size=3, activation='elu', padding='same', input_shape=(vector_size, 1))(input)
model = Conv1D(32, kernel_size=3, activation='elu', padding='same')(model)
model = Conv1D(32, kernel_size=3, activation='relu', padding='same')(model)
model = MaxPooling1D(pool_size=3)(model)
model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)),
merge_mode='concat')(model) # accuracy = 0.316
model = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(input) # accuracy = 0.279
model = Bidirectional(LSTM(hidden_size, return_sequences=True),merge_mode='concat')(model) # 3) added this 0.21259
model = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(model)
output = Dense(y_train_reshape_matrix.shape[2], activation='softmax')(model)
model = Model(input, output)
model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy'])
####################### MODEL 2: ####################################
# from keras.models import Sequential
# from keras.layers import Conv1D, Dropout, Dense, Flatten, LSTM, MaxPooling1D, Bidirectional
# from keras.optimizers import Adam
# from keras.callbacks import EarlyStopping, TensorBoard
#
# model = Sequential()
#
# vector_size = 512
# model.add(Conv1D(32, kernel_size=3, activation='elu', padding='same',
# input_shape=(vector_size,1)))
# model.add(Conv1D(32, kernel_size=3, activation='elu', padding='same'))
# model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same'))
# model.add(MaxPooling1D(pool_size=3))
#
# model.add(Bidirectional(LSTM(512, dropout=0.2, recurrent_dropout=0.3)))
#
# model.add(Dense(512, activation='sigmoid'))
# model.add(Dropout(0.2))
# model.add(Dense(512, activation='sigmoid'))
# model.add(Dropout(0.25))
# model.add(Dense(512, activation='sigmoid'))
# model.add(Dropout(0.25))
#
#
# model.add(Dense(y_train_reshape_matrix.shape[1], activation='softmax'))
# model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy'])
print(model.summary())
# Keras detects the output_shape and automatically determines which accuracy to use when accuracy is specified. For multi-class classification, categorical_accuracy will be used internally.
# https://stackoverflow.com/questions/43544358/categorical-crossentropy-need-to-use-categorical-accuracy-or-accuracy-as-the-met
tensorboard = TensorBoard(log_dir='logs/', histogram_freq=0, write_graph=True, write_images=True)
# print('data_dim shape: ' + str(data_dim))
print('X_train_reshape shape: ' + str(X_train_reshape.shape))
print('y_train_reshape_matrix shape: ' + str(y_train_reshape_matrix.shape))
# model.fit(X_train_reshape,y_train_reshape_matrix, batch_size=500, epochs=15)
model.fit(X_train_reshape, y_train_reshape_matrix, batch_size=model_batch_size, epochs=no_epochs,
validation_data=(X_val_reshape, y_val_reshape_matrix), callbacks=[tensorboard, EarlyStopping(min_delta=0.0001, patience=3)])
# prevent overfitting or over training of the network, EarlyStopping() is used in callback
print('generate final prediction accuracy metrics')
val_lost, val_acc = model.evaluate(X_val_reshape,y_val_reshape_matrix)
print('final val_lost ' + str(val_lost))
print('final val_acc ' + str(val_acc))
return val_lost, val_acc, model
# LSTM and CNN for sequence classification in the IMDB dataset
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
# load the dataset but only keep the top n words, zero the rest
top_words = 5000
(X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words)
# truncate and pad input sequences
max_review_length = 500
X_train = sequence.pad_sequences(X_train, maxlen=max_review_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_review_length)
# create the model
embedding_vecor_length = 32
model = Sequential()
model.add(Embedding(top_words, embedding_vecor_length, input_length=max_review_length))
model.add(Conv1D(32, 3, padding='same', activation='relu'))
model.add(MaxPooling1D())
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
model.fit(X_train, y_train, epochs=3, batch_size=64)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))
#Importing Keras libraries and packages from keras.models import Sequential from keras.layers import Dense from keras.layers import Flatten from keras.layers import Dropout from keras.layers.convolutional import Conv1D from keras.layers.convolutional import MaxPooling1D from keras.utils import to_categorical verbose, epochs, batch_size = 0, 100, 32 n_timesteps, n_features, n_outputs = X_train.shape[1], X_train.shape[2], 5 regressor = Sequential() regressor.add(Conv1D(filters = 32, kernel_size = 5, activation='relu', input_shape=(n_timesteps, n_features))) regressor.add(Dropout(0.1)) regressor.add(Conv1D(filters=64, kernel_size=5, activation='relu')) regressor.add(Dropout(0.2)) regressor.add(MaxPooling1D(pool_size=2)) regressor.add(Flatten()) regressor.add(Dense(100, activation='relu')) regressor.add(Dropout(0.5)) regressor.add(Dense(5, activation='softmax'))
print(len_max)
for x in tqdm(X_train[1:10]):
print(x)
tokenizer = Tokenizer(num_words=len(list(unique_words)))
tokenizer.fit_on_texts(list(X_train))
X_train = tokenizer.texts_to_sequences(X_train)
X_val = tokenizer.texts_to_sequences(X_val)
X_test = tokenizer.texts_to_sequences(test_sentences)
X_train = sequence.pad_sequences(X_train, maxlen=len_max)
X_val = sequence.pad_sequences(X_val, maxlen=len_max)
X_test = sequence.pad_sequences(X_test, maxlen=len_max)
print(X_train.shape, X_val.shape, X_test.shape)
model = Sequential()
model.add(Embedding(len(list(unique_words)), 300, input_length=len_max))
model.add(Conv1D(128, 5, activation='relu'))
model.add(MaxPooling1D(5))
model.add(Conv1D(128, 5, activation='relu'))
# model.add(MaxPooling1D(35))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=0.005),
metrics=['accuracy'])
model.summary()
history = model.fit(X_train,
y_train,
validation_data=(X_val, y_val),
epochs=4,
from keras.preprocessing import sequence
# Loading only top words from dataset
top_words_number = 6000
(X_train, y_train), (X_test,
y_test) = imdb.load_data(num_words=top_words_number)
# Pads dataset to maximum review words
max_words = 500
X_train = sequence.pad_sequences(X_train, maxlen=max_words)
X_test = sequence.pad_sequences(X_test, maxlen=max_words)
print("Building Model")
# Build Model
model = Sequential()
model.add(Embedding(top_words_number, 32, input_length=max_words))
model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(250, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
# Model fitting
model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=2,
def train_model(classifier, feature_vector_train, label, feature_vector_valid):
# fit the training dataset on the classifier
classifier.fit(feature_vector_train, label)
# predict the labels on validation dataset
predictions = classifier.predict(feature_vector_valid)
return metrics.accuracy_score(predictions, ytest), predictions
'''
lx = len(embedding_matrix)
inp = Input(shape=(maxlen, ))
x = Embedding(lx, embed_size, weights=[embedding_matrix])(inp)
x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x)
x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x)
x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
x = Bidirectional(
LSTM(300, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
x = GlobalMaxPool1D()(x)
x = Dense(300, activation="relu")(x)
x = Dense(100, activation="relu")(x)
x = Dropout(0.1)(x)
#ylayer=numpy.asarray(ylayer)
x = Dense(3, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
model.compile(loss='binary_crossentropy',
nb_tag = len(data.index_tag)
maxlen = 100
word_embedding_dim = 50
lstm_dim = 50
batch_size = 64
word_input = Input(shape=(maxlen,), dtype='int32', name='word_input')
word_emb = Embedding(nb_word, word_embedding_dim, input_length=maxlen, dropout=0.2, name='word_emb')(word_input)
bilstm = Bidirectional(LSTM(lstm_dim, dropout_W=0.1, dropout_U=0.1, return_sequences=True))(word_emb)
bilstm_d = Dropout(0.1)(bilstm)
half_window_size=2
paddinglayer=ZeroPadding1D(padding=half_window_size)(word_emb)
conv=Conv1D(nb_filter=50,filter_length=(2*half_window_size+1),border_mode='valid')(paddinglayer)
conv_d = Dropout(0.1)(conv)
dense_conv = TimeDistributed(Dense(50))(conv_d)
rnn_cnn_merge=merge([bilstm_d,dense_conv], mode='concat', concat_axis=2)
dense = TimeDistributed(Dense(nb_tag))(rnn_cnn_merge)
crf = ChainCRF()
crf_output = crf(dense)
model = Model(input=[word_input], output=[crf_output])
model.compile(loss=crf.sparse_loss,
optimizer=RMSprop(0.0001),
metrics=['sparse_categorical_accuracy'])
# create a weight matrix for words in training docs
# wiki.tl.vec = 300, glove = 200
dim_size = 300
# dim_size = 200
embedding_matrix = np.zeros((vocab_size, dim_size))
for word, i in t.word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
# Layer 1 - input layer using word embeddings
sequence_input = Input(shape=(MAX_LENGTH,), dtype='int32')
embedding_layer = Embedding(vocab_size, dim_size, weights=[embedding_matrix], input_length=MAX_LENGTH, trainable=False)
embedding_sequences = embedding_layer(sequence_input)
x = Conv1D(filters=100, kernel_size=4, activation='relu')(embedding_sequences)
# print(x)
x = Dropout(0.5)(x)
x = Flatten()(x)
outputs = Dense(units=5, activation='sigmoid')(x)
model = Model(sequence_input, outputs)
# Second input - quoted_posts
sequence_input2 = Input(shape=(MAX_LENGTH, ), dtype='int32')
embedding_layer2 = Embedding(vocab_size, dim_size, weights=[embedding_matrix], input_length=MAX_LENGTH,
trainable=False)
embedding_sequences2 = embedding_layer2(sequence_input2)
y = Conv1D(filters=100, kernel_size=4, activation='relu')(embedding_sequences2)
y = Dropout(0.5)(y)
y = Flatten()(y)
def SentenceClassificationSupervised(self):
MAX_NB_WORDS = 50000
print(time.asctime())
print('reading labeled csv and tokenizing')
#df=pd.read_csv(self.basPath+'TRAIN_1.csv')
self.sup_sentence_df=pd.read_csv(self.config['TRAIN'])
df=self.sup_sentence_df
assert len(df)>0 ,' empty TRAIN FILE'
#build a tokenizer
self.tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True)
#tokenising the sentences
self.tokenizer.fit_on_texts(df['sentence'].values)
word_index = self.tokenizer.word_index
print('read words and tokenized')
print(time.asctime())
# load the dataset but only keep the top n words, zero the rest
top_words = 50000
#convert sentence array to a sequence of integers
X = self.tokenizer.texts_to_sequences(df['sentence'].values)
Y = df['class_int'].values
X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.2, random_state = 42)
#print('Shape of data tensor:', X.shape)
print(X_train[0])
print(Y_train[0])
#maximum length of a sequence 50 words
max_length = 50
X_train = sequence.pad_sequences(X_train, maxlen=max_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_length)
# create the model
embedding_vecor_length = 100
model = Sequential()
#building layers of ANN Model
model.add(Embedding(top_words, embedding_vecor_length, input_length=max_length))
model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
model.fit(X_train, Y_train, epochs=2, batch_size=64)
# Final evaluation of the model
scores = model.evaluate(X_test, Y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))
Y_pred=model.predict_classes(X_test)
self.ML_model=model
# catch the metriccs of evaluation for Classification
tn,fp,fn,tp=sm.confusion_matrix(Y_test,Y_pred).ravel()
precision=tp/(tp+fp)
recall=tp/(tp+fn)
f1=2*precision*recall/(precision+recall)
print("PRECISION: %.2f%%" % precision)
print("RECALL: %.2f%%" % recall)
print("F1 SCORE: %.2f%%" % f1)
#df['PREDICTION']=Y_pred
pickle.dump(self.tokenizer,open(self.basPath+"tokenizer.h5","wb"))
model.save(self.basPath+'model')
self.sup_sentence_df.to_csv(self.basPath+'PREDICTION.csv')
print("DONE")
####################### for fixed size filter CNN-LSTM
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import *
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
model = Sequential()
model.add(Embedding(input_dim=len(word_index)+1, output_dim=embedding_dim, weights=[embedding_matrix], input_length=sequence_length))
model.add(Conv1D(filters=1024, kernel_size=3, padding='same', activation='tanh'))
model.add(MaxPooling1D(pool_size=4))
model.add(LSTM(100))
model.add(Dropout(0.1))
model.add(Dense(6, activation='softmax'))
checkpoint = ModelCheckpoint('./CNN-2LSTM-SWBD/weights.{epoch:03d}-{val_acc:.4f}.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')
stop = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='max')
adam = Adam(lr=0.01, decay=0.7)
model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
print(model.summary())
model.fit(X_train, y_train, epochs=100, callbacks=[checkpoint, stop], batch_size=64, validation_data=(X_test, y_test))
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))
predicted=model.predict(X_test)