selected = int(0.4 * len(out)) trainTextsSeq_Fr = out[:selected, :-1] y_train_Fr = out[:selected, -1] embedding_vector_length = 100 class_names = ['0', '1'] # Building the model on the entire dataset and testing it on the test data runs = [] for run in range(1): myInput = Input(shape=(max_rep_length, ), name='input_English') # 500 vectors of size 100 x = Embedding(output_dim=100, input_dim=vocabSize_Fr, input_length=max_rep_length)(myInput) CNN_h1 = Conv1D(200, kernel_size=4, strides=1, activation='relu')(x) h1 = MaxPooling1D(pool_size=2, strides=None, padding='same')(CNN_h1) CNN_h2 = Conv1D(200, kernel_size=3, strides=1, activation='relu')(h1) h2 = MaxPooling1D(pool_size=2, strides=None, padding='same')(CNN_h2) CNN_h3 = Conv1D(200, kernel_size=3, strides=1, activation='relu')(h2) h3 = MaxPooling1D(pool_size=2, strides=None, padding='same')(CNN_h3) CNN_h4 = Conv1D(200, kernel_size=3, strides=1, activation='relu')(h3) h5 = MaxPooling1D(pool_size=2, strides=None, padding='same')(CNN_h4) f = Flatten()(h1) predictions = Dense(2, activation='softmax', name='pred_eng')(f) model = Model(myInput, outputs=predictions) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
ytest = np.array([0 for _ in range(num if num else 12500)] + [1 for _ in range(num if num else 12500)]) ytest = to_categorical(ytest, num_classes=2) # Using pre-trained word embedding print("Loading Word Embedding") wordlist = load_emb(embedding_dir, tokenizer.word_index) embedding_layer = Embedding(vocab_size, 300, weights=[wordlist], input_length=max_length, trainable=False) # Define model model = Sequential() model.add(embedding_layer) model.add(Conv1D(filters=128, kernel_size=5, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dense(2, activation='softmax')) model.summary() # Compile network model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # Fit network model.fit(Xtrain, ytrain, epochs=20, verbose=1) # evaluate loss, acc = model.evaluate(Xtest, ytest, verbose=0) print('Test Accuracy: %f' % (acc * 100))
print(i, "th Fold *****************************************") i = i + 1 x_train_clinical, x_test_clinical = X_clinical[train_index], X_clinical[ test_index] y_train_clinical, y_test_clinical = Y_clinical[train_index], Y_clinical[ test_index] x_train_clinical = numpy.expand_dims(x_train_clinical, axis=2) x_test_clinical = numpy.expand_dims(x_test_clinical, axis=2) # first Clinical CNN Model init = initializers.glorot_normal(seed=1) bias_init = initializers.Constant(value=0.1) main_input1 = Input(shape=(25, 1), name='Input') conv1 = Conv1D(filters=25, kernel_size=15, strides=2, activation='tanh', padding='same', name='Conv1D', kernel_initializer=init, bias_initializer=bias_init)(main_input1) flat1 = Flatten(name='Flatten')(conv1) dense1 = Dense(150, activation='tanh', name='dense1', kernel_initializer=init, bias_initializer=bias_init)(flat1) output = Dense(1, activation='sigmoid', name='output', activity_regularizer=regularizers.l2(0.01), kernel_initializer=init, bias_initializer=bias_init)(dense1)
# -+-+-+-+-+-+-+- BUILDING MODEL -+-+-+-+-+-+-+- print("BUILDING MODEL") embedding_vecor_length = 32 input_layer = Embedding(len(tokenizer.word_index) + 1, global_emb_dim, weights=[emb_matrix], input_length=global_max_seq, trainable=False) branch_3 = Sequential() branch_3.add(input_layer) branch_3.add( Conv1D(filters=32, kernel_size=3, padding='same', kernel_regularizer=l2(.01))) branch_3.add(Activation('relu')) branch_3.add(MaxPooling1D(pool_size=2)) branch_3.add(Dropout(0.5)) branch_3.add(BatchNormalization()) branch_3.add(LSTM(100)) branch_4 = Sequential() branch_4.add(input_layer) branch_4.add( Conv1D(filters=32, kernel_size=4, padding='same', kernel_regularizer=l2(.01))) branch_4.add(Activation('relu'))
D = [44, 40, 45, 70, 20, 67, 36, 43, 65, 63] for NConv1, NConv2, NJanet1, NJanet2, NDense in zip(C1, C2, J1, J2, D): acc_list = [] prec_list = [] recall_list = [] f1_list = [] for i in range(repetition): early_stopping = EarlyStopping(monitor='val_acc', patience=patience) checkpoint = ModelCheckpoint('6dmg_try.h5', monitor='val_acc', verbose=0, save_weights_only=True) nadam = Nadam(lr=1e-4, beta_1=0.9, beta_2=0.9) model3 = Sequential() model3.add(Conv1D(NConv1, 5, activation='relu', input_shape=(240, 6))) model3.add(Conv1D(NConv2, 5, activation='relu')) model3.add(JANET(NJanet1, activation='relu', return_sequences=True)) #, input_shape=(None, 6))) model3.add(JANET(NJanet2, activation='relu')) model3.add(Dense(NDense, activation='relu')) model3.add(Dense(20, activation='softmax')) model3.compile(loss='categorical_crossentropy', optimizer=nadam, metrics=['accuracy']) model3.fit(xTrain, yTrain, epochs=epochs, validation_data=(xVal, yVal), callbacks=[early_stopping], verbose=1,
def train_cnn_lstm(x_train, x_test, y_train, y_test, embedding_matrix, max_length): """ Function that trains a CNN-LSTM model using the pre-trained skip-gram embedding Inputs: x_train: np array x_test: np array y_train: np array y_test: np array embedding_matrix: np array of shape num_words * dim_embedding max_length: int - maximum length of the sequence (in words) """ kfold = StratifiedKFold(n_splits=5, shuffle=False) es = EarlyStopping(monitor='val_loss', patience=5, min_delta=0.01) num_words = embedding_matrix.shape[0] dim_embedding = embedding_matrix.shape[1] accs = [] aucs = [] for train, valid in kfold.split(x_train, y_train): model = Sequential() embedding_layer = Embedding(input_dim=num_words, output_dim=dim_embedding, weights=[embedding_matrix], input_length=max_length, trainable=False) model.add(embedding_layer) model.add(Dropout(0.25)) model.add( Conv1D(filters=10, kernel_size=2, padding='same', activation='relu')) model.add( Conv1D(filters=10, kernel_size=3, padding='same', activation='relu')) model.add(Dropout(0.5)) model.add(MaxPooling1D(pool_size=4)) model.add( LSTM(units=256, recurrent_dropout=0.5, recurrent_regularizer=regularizers.l2(0.01))) model.add(Dense(1, activation='sigmoid')) # binary classification model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) history = model.fit(x_train[train], y_train[train], validation_data=(x_train[valid], y_train[valid]), callbacks=[es], epochs=10, batch_size=128, verbose=0) score = model.evaluate(x_test, y_test, verbose=0) accs.append(score[1]) y_proba = model.predict_proba( x_test) # get the classification probs on the test set # Compute ROC curve fpr, tpr, thresholds = roc_curve(y_test, y_proba, pos_label=1) # Compute ROC area roc_auc = auc(fpr, tpr) aucs.append(roc_auc) # plot training history plt.plot(history.history['loss'], label='train') plt.plot(history.history['val_loss'], label='test') plt.legend() plt.show() return history, accs, aucs, model
sequence_length = 17 max_review_length = 17 # input dim vocabulary_size = len(vocabulary_inv_train) # output dim embedding_vector_length = 220 model = Sequential() model.add( Embedding(vocabulary_size, embedding_vector_length, input_length=max_review_length)) model.add( Conv1D(filters=128, kernel_size=3, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add( Conv1D(filters=96, kernel_size=3, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add( Conv1D(filters=64, kernel_size=3, padding='same', activation='relu')) # print(model.summary()) model.add(MaxPooling1D(pool_size=1)) model.add( Bidirectional( LSTM(250, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))) model.add(
sequence_length = 5 max_review_length = 5 # input dim vocabulary_size = len(vocabulary_inv_train) # output dim embedding_vector_length = 20 model = Sequential() model.add( Embedding(vocabulary_size, embedding_vector_length, input_length=max_review_length)) model.add( Conv1D(filters=150, kernel_size=20, padding='same', activation='relu')) model.add( Bidirectional( LSTM(250, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))) model.add( Bidirectional( LSTM(350, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))) model.add(Bidirectional(LSTM(250, dropout=0.2))) model.add(Dense(50, activation='softmax'))
from scipy import fromstring, int16 import numpy as np import os.path from keras.models import Sequential, load_model from keras.layers import Dense, Flatten, Reshape from keras.layers.noise import GaussianNoise from keras.layers.convolutional import Conv1D, UpSampling1D from keras.layers.pooling import MaxPooling1D from keras.layers.normalization import BatchNormalization from keras import backend as K from keras.layers.advanced_activations import LeakyReLU if os.path.exists(par.l1_encoder_filename): model = load_model(par.l1_encoder_filename) else: model = Sequential() model.add(Conv1D(par.l1_first_filters, 21, strides=par.l1_first_strides, padding='same', input_shape=(par.l1_input_length, par.l1_channel_size))) model.add(Conv1D(par.l1_second_filters, 14, strides=par.l1_second_strides, padding='same')) model.add(UpSampling1D(par.l1_second_strides)) model.add(Conv1D(par.l1_second_filters, 14, padding='same')) model.add(UpSampling1D(par.l1_first_strides)) model.add(Conv1D(par.l1_first_filters, 21, padding='same')) model.add(Conv1D(par.l1_channel_size, 8, padding='same')) train_filename = par.l1_train_filename epochs = par.l1_epochs encoder_file = par.l1_encoder_filename encoder = K.function([model.layers[0].input], [model.layers[1].output]) decoder = K.function([model.layers[2].input], [model.layers[6].output])
y = np.load('../output/y_data.npy') pos_ix = [i for i in range(len(y)) if y[i] == 1] neg_ix = [i for i in range(len(y)) if y[i] == 0] train_ix = range(250) train_set = [pos_ix[ix] for ix in train_ix] + [neg_ix[ix] for ix in train_ix] x_train = x[train_set] y_train = y[train_set] for i in range(2): neural_net = Sequential() neural_net.add(Conv1D(nb_filter=10, filter_length=5, input_shape=(17, 4))) neural_net.add(Flatten()) neural_net.add(Dense(10)) neural_net.add(Activation('relu')) neural_net.add(Dense(1)) neural_net.add(Activation('sigmoid')) neural_net.compile(loss='mse', optimizer='sgd') neural_net.fit(x=x_train, y=y_train, batch_size=32, nb_epoch=250) # read in sequences from rap1-lieb-test.txt filepath = '../data/rap1-lieb-test.txt' file_text = open(filepath) sequences = [] for line in file_text: sequence = line.strip() sequences.append(sequence)
X_train = sequence.pad_sequences(X_train, maxlen=max_review_length) X_test = sequence.pad_sequences(X_test, maxlen=max_review_length) print("") # -+-+-+-+-+-+-+- BUILDING MODEL -+-+-+-+-+-+-+- print("BUILDING MODEL") embedding_vecor_length = 32 input_layer = Embedding(top_words, embedding_vecor_length, input_length=max_review_length) branch_3 = Sequential() branch_3.add(input_layer) branch_3.add(Conv1D(filters=32, kernel_size=3, padding='same')) branch_3.add(Activation('relu')) branch_3.add(MaxPooling1D(pool_size=2)) branch_3.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2)) branch_4 = Sequential() branch_4.add(input_layer) branch_4.add(Conv1D(filters=32, kernel_size=4, padding='same')) branch_4.add(Activation('relu')) branch_4.add(MaxPooling1D(pool_size=2)) branch_4.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2)) branch_5 = Sequential() branch_5.add(input_layer) branch_5.add(Conv1D(filters=32, kernel_size=5, padding='same')) branch_5.add(Activation('relu'))
print("y_train_ori:", y_train_ori.sum()) y_train = np_utils.to_categorical(y_train_ori, num_classes=2) y_test = np_utils.to_categorical(y_test_ori, num_classes=2) print("Building model") model_input = Input(shape=(int(SAMPLE_LENGTH / 10), 2)) # PD = 2, LCP = 3 # x= Bidirectional(LSTM(hi, return_sequences=True), merge_mode='concat')(model_input) rnn_feature = Bidirectional(LSTM(hidden_num, return_sequences=True))(model_input) pooled_outputs = [] for filter_size in FILTER_SIZES: x = Conv1D(NUM_FILTERS * 2, filter_size, padding='same')(rnn_feature) x = BatchNormalization()(x) x = Activation('relu')(x) # x = AveragePooling1D(int(x.shape[1]))(x) # x = GlobalAveragePooling1D()(x) x = AttLayer()(x) x = Reshape((1, -1))(x) pooled_outputs.append(x) merged = concatenate(pooled_outputs) x = Flatten()(merged) x = BatchNormalization()(x) x = Dense(NUM_FILTERS)(x) x = BatchNormalization()(x) # x = Activation('relu')(x) model_output = Dense(num_class, activation='softmax')(x)
def build_model(vector_size): #batch_size = 512 nb_epochs = 5 model = Sequential() # This f*****g block will eat all of your goddamn memory # Sacrifice 3 chickens to improve accuracy # sparse regularization model.add( Conv1D(32, kernel_size=3, kernel_regularizer=regularizers.l2(0.005), activation='relu', padding='same', input_shape=(vector_size, 3))) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Dropout(0.5)) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Dropout(0.5)) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Dropout(0.5)) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=1, activation='relu', padding='same')) model.add(Conv1D(32, kernel_size=1, activation='relu', padding='same')) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(64, activation='tanh')) model.add(Dense(64, activation='tanh')) model.add(Dense(1, activation='sigmoid')) # Compile the model model.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.001, decay=1e-6), metrics=['accuracy']) return model
half_train_length = int(round(len(train) / 2)) half_test_length = int(round(len(test) / 2)) # define train and test labels y_train = array([0 for _ in range(half_train_length)] + [1 for _ in range(half_train_length)]) y_test = array([0 for _ in range(half_test_length)] + [1 for _ in range(half_test_length)]) # define vocab size (+1 for unknown words) vocab_size = len(tokenizer.word_index) + 1 # define model model = Sequential() model.add(Embedding(vocab_size, 100, input_length=max_length)) model.add(Conv1D(filters=32, kernel_size=8, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dense(10, activation='relu')) model.add(Dense(1, activation='sigmoid')) print(model.summary()) # save model model.save('models/embedded_conv_1d') # create dict of train and test data data_dict = {'x_train': x_train, 'y_train': y_train, 'x_test': x_test, 'y_test': y_test} # serialize data dict print('saving data dict to pickle...')
def create_model(): """Creates the neural network. Returns: neural network model, optimizer """ dense_regularizer = L1L2(l2=0.0005) audio_input = Input(shape=(1, 128, 301), dtype="float64") audio = Conv1D(32, kernel_size=(128, 8), padding="same", activation="relu", name="Conv1D_32/128/8_relu")(audio_input) audio = MaxPooling1D(pool_size=(1, 4), name="MaxPooling1D_1/4_1")(audio) audio = Conv1D(32, kernel_size=(1, 8), padding="same", activation="relu", name="Conv1D_32/1/8_relu")(audio) audio = MaxPooling1D(pool_size=(1, 2), name="MaxPooling1D_1/2_2")(audio) audio = Conv1D(32, kernel_size=(1, 4), padding="same", activation="relu", name="Conv1D_32/1/4_relu")(audio) audio = MaxPooling1D(pool_size=(1, 2), name="MaxPooling1D_1/2_3")(audio) audio = Dense(1024, activation="relu", name="Dense_1024", kernel_regularizer=dense_regularizer)(audio) audio = Dense(512, activation=" relu", name=" Dense_512", kernel_regularizer=dense_regularizer)(audio) audio_output = audio cnn_audio_model = Model(audio_input, audio_output) input_audio = Input(shape=(128, 301)) input_image = Input(shape=(1, 2048)) input_time = Input(shape=(1)) list_output_audio = K.map_fn(lambda feature: cnn_audio_model, input_audio) #element wise output_audio = np.maximum.reduce(list_output_audio) merged = Concatenate([input_image, output_audio, input_time]) X_input = Input(merged) X = Dense(100, activation="relu", name="Dense_100_1", kernel_regularizer=dense_regularizer)(X_input) X = Dense(100, activation=" relu", name=" Dense_100_2", kernel_regularizer=dense_regularizer)(X) X_output = X return Model([input_audio, input_image, input_time], X_output)
#getting the vocabulary of data sentences = tokenizer.texts_to_sequences(sentences) padded_docs= pad_sequences(sentences,maxlen=max_review_len) le = preprocessing.LabelEncoder() y = le.fit_transform(y) X_train, X_test, y_train, y_test = train_test_split(padded_docs, y, test_size=0.25, random_state=1000) vocab_size= len(tokenizer.word_index)+1 num_classes = y_test.shape[0] model = Sequential() model.add(layers.Embedding(vocab_size, 50, input_length=max_review_len)) #model.add(layers.Flatten()) model.add(Conv1D(32, 3, activation='relu', padding='same')) model.add(Dropout(0.2)) model.add(Conv1D(64, 3, activation='relu', padding='same')) model.add(MaxPooling1D(pool_size=1)) model.add(Conv1D(128, 3, activation='relu', padding='same')) model.add(Dropout(0.2)) model.add(Conv1D(128, 3, activation='relu', padding='same')) model.add(MaxPooling1D(pool_size=1)) # flattening the matrix into vector form model.add(Flatten()) model.add(Dropout(0.2)) model.add(Dense(1024, activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.2))
from keras.layers.convolutional import MaxPooling1D x = array([[10, 20, 30], [20, 30, 40], [30, 40, 50], [40, 50, 60]]) y = array([40, 50, 60, 70]) x = x.reshape((x.shape[0], x.shape[1], 1)) print("x.shape[0]", x.shape[0]) print("x.shape[1]", x.shape[1]) print("x.shape", x.shape) print("y.shape", y.shape) # define model model = Sequential() model.add( Conv1D(filters=64, kernel_size=2, activation='relu', input_shape=(3, 1))) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dense(50, activation='relu')) model.add(Dense(1)) model.compile(optimizer='adam', loss='mse') model.summary() #fit model model.fit(x, y, epochs=1000, verbose=0) #demonstrate prediction x_input = array([60, 70, 90]) x_input = x_input.reshape(1, 3, 1)
# univariate cnn-lstm example from numpy import array from keras.models import Sequential from keras.layers import LSTM from keras.layers import Dense from keras.layers import Flatten from keras.layers import TimeDistributed from keras.layers.convolutional import Conv1D from keras.layers.convolutional import MaxPooling1D # define dataset X = array([[10, 20, 30, 40], [20, 30, 40, 50], [30, 40, 50, 60], [40, 50, 60, 70]]) y = array([50, 60, 70, 80]) # reshape from [samples, timesteps] into [samples, subsequences, timesteps, features] X = X.reshape((X.shape[0], 2, 2, 1)) # define model model = Sequential() model.add(TimeDistributed(Conv1D(filters=64, kernel_size=1, activation='relu'), input_shape=(None, 2, 1))) model.add(TimeDistributed(MaxPooling1D(pool_size=2))) model.add(TimeDistributed(Flatten())) model.add(LSTM(50, activation='relu')) model.add(Dense(1)) model.compile(optimizer='adam', loss='mse') # fit model model.fit(X, y, epochs=500, verbose=0) # demonstrate prediction x_input = array([50, 60, 70, 80]) x_input = x_input.reshape((1, 2, 2, 1)) yhat = model.predict(x_input, verbose=0) print(yhat)
def build_decoder(self, latent, name='conv', n_layers=1): decoder = latent if name == 'conv': if n_layers == 3: decoder = Dense(self.window_size // 64 * 256, activation='relu', name='decoder_dense1')(decoder) decoder = Reshape((self.window_size // 64, 256), name='decoder_reshape1')(decoder) decoder = UpSampling1D(4, name='decoder_upsample1')(decoder) decoder = Conv1D(128, 3, padding='same', activation='relu', name='decoder_conv1')(decoder) decoder = UpSampling1D(4, name='decocer_upsample2')(decoder) decoder = Conv1D(64, 3, padding='same', activation='relu', name='decoder_conv2')(decoder) if n_layers == 2: decoder = Dense(self.window_size // 16 * 128, activation='relu', name='decoder_dense1')(decoder) decoder = Reshape((self.window_size // 16, 128), name='decoder_reshape1')(decoder) decoder = UpSampling1D(4, name='decocer_upsample2')(decoder) decoder = Conv1D(64, 3, padding='same', activation='relu', name='decoder_conv2')(decoder) if n_layers == 1: decoder = Dense(self.window_size // 4 * 64, activation='relu', name='decoder_dense1')(decoder) decoder = Reshape((self.window_size // 4, 64), name='decoder_reshape1')(decoder) decoder = UpSampling1D(4, name='decoder_upsample3')(decoder) decoder = Conv1D(4, 1, padding='same', name='decoder_conv3')(decoder) decoder = Lambda(lambda x: K.softmax(x, axis=-1), name='output_softmax')(decoder) decoder = Lambda(lambda x: K.mean(K.reshape( x, (-1, self.n_sampler, self.window_size, self.n_channels)), axis=1), name='output_mean')(decoder) elif name == 'mlp': if n_layers >= 2: decoder = Dense(128, activation='relu', name='decoder_dense2')(decoder) decoder = Dense(self.window_size * self.n_channels, name='decoder_dense3')(decoder) decoder = Lambda(lambda x: K.softmax(x, axis=-1), name='output_softmax')(decoder) decoder = Lambda(lambda x: K.mean(K.reshape( x, (-1, self.n_sampler, self.window_size, self.n_channels)), axis=1), name='output_mean')(decoder) elif name == 'lstm': decoder = LSTM(64, name='encoder_lstm1', return_sequences=True)(decoder) return decoder
label, sent = line.strip().split('\t') ys.append(label) words = [w.lower() for w in nltk.word_tokenize(sent)] wids = [word2index[w] for w in words] xs.append(wids) fin.close() X = pad_sequences(xs, maxlen=maxlen) Y = np_utils.to_categorical(ys) Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.3, random_state=42) print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) model = Sequential() model.add(Embedding(input_dim=vocab_sz, output_dim=EMBED_SIZE, input_length=maxlen)) model.add(SpatialDropout1D(0.2)) model.add(Conv1D(filters=NUM_FILTERS, kernel_size=NUM_WORDS, activation='relu')) model.add(GlobalMaxPooling1D()) model.add(Dense(2, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, verbose=VERBOSE, validation_data=[Xtest, Ytest]) plt.subplot(211) plt.title("accuracy") plt.plot(history.history['acc'], color='r', label='train') plt.plot(history.history['val_acc'], color='b', label='validation') plt.legend(loc='best') plt.subplot(212) plt.title('loss') plt.plot(history.history['loss'], color='r', label='train')
def simple_LSTM(X_train_reshape, y_train_reshape_matrix,X_val_reshape, y_val_reshape_matrix,vector_size,model_batch_size,no_epochs): # We usually match up the size of the embedding layer output with the number of hidden layers in the LSTM cell. # https://adventuresinmachinelearning.com/keras-lstm-tutorial/ # hidden_size = 4096 # hidden_size = 32 hidden_size = 512 from keras.models import Model from keras.layers import Input, Dense from keras.layers import TimeDistributed from keras.layers import Conv1D, Dropout, Dense, Flatten, LSTM, MaxPooling1D, Bidirectional from keras.optimizers import Adam from keras.layers import Flatten DROP_RATE_DENSE = 0.1 #perahaps i want to use a combinatinos of convolutional and lstm? # https://github.com/jatinmandav/Neural-Networks/blob/master/Sentiment-Analysis/universal-sentence-encoder/universal_sentence_encoder_sentiment-analysis.ipynb ####################### MODEL 1: ~ 30% accuracy #################################### # use variable length timesteps using None # https://datascience.stackexchange.com/questions/26366/training-an-rnn-with-examples-of-different-lengths-in-keras # model = Sequential() # timesteps = None # data_dim = X_train_reshape.shape[2] # # model = Sequential() # input = Input(shape=(timesteps, data_dim)) # model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)),merge_mode='concat')(input) #accuracy = 0.316 # model = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(input) # accuracy = 0.279 # # model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)),merge_mode='concat')(model) # 3) added this 0.21259 # # model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)), merge_mode='concat')(model) # model = Dropout(DROP_RATE_DENSE)(model) # # model = TimeDistributed(Dense(256, activation='relu'))(model) #accuracy after adding this line =0.318 # # model = Flatten(input_shape=(timesteps, data_dim))(model) # # model = Dense(100, activation='relu')(model) # model = Dropout(DROP_RATE_DENSE)(model) # # output = Dense(y_train_reshape_matrix.shape[2], activation='softmax')(model) # model = Model(input, output) # model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy']) ####################### MODEL 1: #################################### ####################### MODEL 2: ~ 30% accuracy #################################### # use variable length timesteps using None # https://datascience.stackexchange.com/questions/26366/training-an-rnn-with-examples-of-different-lengths-in-keras model = Sequential() timesteps = None data_dim = X_train_reshape.shape[2] vector_size = 512 model = Sequential() input = Input(shape=(timesteps, data_dim)) model = Conv1D(32, kernel_size=3, activation='elu', padding='same', input_shape=(vector_size, 1))(input) model = Conv1D(32, kernel_size=3, activation='elu', padding='same')(model) model = Conv1D(32, kernel_size=3, activation='relu', padding='same')(model) model = MaxPooling1D(pool_size=3)(model) model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)), merge_mode='concat')(model) # accuracy = 0.316 model = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(input) # accuracy = 0.279 model = Bidirectional(LSTM(hidden_size, return_sequences=True),merge_mode='concat')(model) # 3) added this 0.21259 model = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(model) output = Dense(y_train_reshape_matrix.shape[2], activation='softmax')(model) model = Model(input, output) model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy']) ####################### MODEL 2: #################################### # from keras.models import Sequential # from keras.layers import Conv1D, Dropout, Dense, Flatten, LSTM, MaxPooling1D, Bidirectional # from keras.optimizers import Adam # from keras.callbacks import EarlyStopping, TensorBoard # # model = Sequential() # # vector_size = 512 # model.add(Conv1D(32, kernel_size=3, activation='elu', padding='same', # input_shape=(vector_size,1))) # model.add(Conv1D(32, kernel_size=3, activation='elu', padding='same')) # model.add(Conv1D(32, kernel_size=3, activation='relu', padding='same')) # model.add(MaxPooling1D(pool_size=3)) # # model.add(Bidirectional(LSTM(512, dropout=0.2, recurrent_dropout=0.3))) # # model.add(Dense(512, activation='sigmoid')) # model.add(Dropout(0.2)) # model.add(Dense(512, activation='sigmoid')) # model.add(Dropout(0.25)) # model.add(Dense(512, activation='sigmoid')) # model.add(Dropout(0.25)) # # # model.add(Dense(y_train_reshape_matrix.shape[1], activation='softmax')) # model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy']) print(model.summary()) # Keras detects the output_shape and automatically determines which accuracy to use when accuracy is specified. For multi-class classification, categorical_accuracy will be used internally. # https://stackoverflow.com/questions/43544358/categorical-crossentropy-need-to-use-categorical-accuracy-or-accuracy-as-the-met tensorboard = TensorBoard(log_dir='logs/', histogram_freq=0, write_graph=True, write_images=True) # print('data_dim shape: ' + str(data_dim)) print('X_train_reshape shape: ' + str(X_train_reshape.shape)) print('y_train_reshape_matrix shape: ' + str(y_train_reshape_matrix.shape)) # model.fit(X_train_reshape,y_train_reshape_matrix, batch_size=500, epochs=15) model.fit(X_train_reshape, y_train_reshape_matrix, batch_size=model_batch_size, epochs=no_epochs, validation_data=(X_val_reshape, y_val_reshape_matrix), callbacks=[tensorboard, EarlyStopping(min_delta=0.0001, patience=3)]) # prevent overfitting or over training of the network, EarlyStopping() is used in callback print('generate final prediction accuracy metrics') val_lost, val_acc = model.evaluate(X_val_reshape,y_val_reshape_matrix) print('final val_lost ' + str(val_lost)) print('final val_acc ' + str(val_acc)) return val_lost, val_acc, model
# LSTM and CNN for sequence classification in the IMDB dataset from keras.datasets import imdb from keras.models import Sequential from keras.layers import Dense from keras.layers import LSTM from keras.layers.convolutional import Conv1D from keras.layers.convolutional import MaxPooling1D from keras.layers.embeddings import Embedding from keras.preprocessing import sequence # load the dataset but only keep the top n words, zero the rest top_words = 5000 (X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words) # truncate and pad input sequences max_review_length = 500 X_train = sequence.pad_sequences(X_train, maxlen=max_review_length) X_test = sequence.pad_sequences(X_test, maxlen=max_review_length) # create the model embedding_vecor_length = 32 model = Sequential() model.add(Embedding(top_words, embedding_vecor_length, input_length=max_review_length)) model.add(Conv1D(32, 3, padding='same', activation='relu')) model.add(MaxPooling1D()) model.add(LSTM(100)) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() model.fit(X_train, y_train, epochs=3, batch_size=64) # Final evaluation of the model scores = model.evaluate(X_test, y_test, verbose=0) print("Accuracy: %.2f%%" % (scores[1]*100))
#Importing Keras libraries and packages from keras.models import Sequential from keras.layers import Dense from keras.layers import Flatten from keras.layers import Dropout from keras.layers.convolutional import Conv1D from keras.layers.convolutional import MaxPooling1D from keras.utils import to_categorical verbose, epochs, batch_size = 0, 100, 32 n_timesteps, n_features, n_outputs = X_train.shape[1], X_train.shape[2], 5 regressor = Sequential() regressor.add(Conv1D(filters = 32, kernel_size = 5, activation='relu', input_shape=(n_timesteps, n_features))) regressor.add(Dropout(0.1)) regressor.add(Conv1D(filters=64, kernel_size=5, activation='relu')) regressor.add(Dropout(0.2)) regressor.add(MaxPooling1D(pool_size=2)) regressor.add(Flatten()) regressor.add(Dense(100, activation='relu')) regressor.add(Dropout(0.5)) regressor.add(Dense(5, activation='softmax'))
print(len_max) for x in tqdm(X_train[1:10]): print(x) tokenizer = Tokenizer(num_words=len(list(unique_words))) tokenizer.fit_on_texts(list(X_train)) X_train = tokenizer.texts_to_sequences(X_train) X_val = tokenizer.texts_to_sequences(X_val) X_test = tokenizer.texts_to_sequences(test_sentences) X_train = sequence.pad_sequences(X_train, maxlen=len_max) X_val = sequence.pad_sequences(X_val, maxlen=len_max) X_test = sequence.pad_sequences(X_test, maxlen=len_max) print(X_train.shape, X_val.shape, X_test.shape) model = Sequential() model.add(Embedding(len(list(unique_words)), 300, input_length=len_max)) model.add(Conv1D(128, 5, activation='relu')) model.add(MaxPooling1D(5)) model.add(Conv1D(128, 5, activation='relu')) # model.add(MaxPooling1D(35)) model.add(Flatten()) model.add(Dense(100, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.005), metrics=['accuracy']) model.summary() history = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=4,
from keras.preprocessing import sequence # Loading only top words from dataset top_words_number = 6000 (X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words_number) # Pads dataset to maximum review words max_words = 500 X_train = sequence.pad_sequences(X_train, maxlen=max_words) X_test = sequence.pad_sequences(X_test, maxlen=max_words) print("Building Model") # Build Model model = Sequential() model.add(Embedding(top_words_number, 32, input_length=max_words)) model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dense(250, activation='relu')) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() # Model fitting model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=2,
def train_model(classifier, feature_vector_train, label, feature_vector_valid): # fit the training dataset on the classifier classifier.fit(feature_vector_train, label) # predict the labels on validation dataset predictions = classifier.predict(feature_vector_valid) return metrics.accuracy_score(predictions, ytest), predictions ''' lx = len(embedding_matrix) inp = Input(shape=(maxlen, )) x = Embedding(lx, embed_size, weights=[embedding_matrix])(inp) x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x) x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x) x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) x = Bidirectional( LSTM(300, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x) x = GlobalMaxPool1D()(x) x = Dense(300, activation="relu")(x) x = Dense(100, activation="relu")(x) x = Dropout(0.1)(x) #ylayer=numpy.asarray(ylayer) x = Dense(3, activation="sigmoid")(x) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy',
nb_tag = len(data.index_tag) maxlen = 100 word_embedding_dim = 50 lstm_dim = 50 batch_size = 64 word_input = Input(shape=(maxlen,), dtype='int32', name='word_input') word_emb = Embedding(nb_word, word_embedding_dim, input_length=maxlen, dropout=0.2, name='word_emb')(word_input) bilstm = Bidirectional(LSTM(lstm_dim, dropout_W=0.1, dropout_U=0.1, return_sequences=True))(word_emb) bilstm_d = Dropout(0.1)(bilstm) half_window_size=2 paddinglayer=ZeroPadding1D(padding=half_window_size)(word_emb) conv=Conv1D(nb_filter=50,filter_length=(2*half_window_size+1),border_mode='valid')(paddinglayer) conv_d = Dropout(0.1)(conv) dense_conv = TimeDistributed(Dense(50))(conv_d) rnn_cnn_merge=merge([bilstm_d,dense_conv], mode='concat', concat_axis=2) dense = TimeDistributed(Dense(nb_tag))(rnn_cnn_merge) crf = ChainCRF() crf_output = crf(dense) model = Model(input=[word_input], output=[crf_output]) model.compile(loss=crf.sparse_loss, optimizer=RMSprop(0.0001), metrics=['sparse_categorical_accuracy'])
# create a weight matrix for words in training docs # wiki.tl.vec = 300, glove = 200 dim_size = 300 # dim_size = 200 embedding_matrix = np.zeros((vocab_size, dim_size)) for word, i in t.word_index.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: embedding_matrix[i] = embedding_vector # Layer 1 - input layer using word embeddings sequence_input = Input(shape=(MAX_LENGTH,), dtype='int32') embedding_layer = Embedding(vocab_size, dim_size, weights=[embedding_matrix], input_length=MAX_LENGTH, trainable=False) embedding_sequences = embedding_layer(sequence_input) x = Conv1D(filters=100, kernel_size=4, activation='relu')(embedding_sequences) # print(x) x = Dropout(0.5)(x) x = Flatten()(x) outputs = Dense(units=5, activation='sigmoid')(x) model = Model(sequence_input, outputs) # Second input - quoted_posts sequence_input2 = Input(shape=(MAX_LENGTH, ), dtype='int32') embedding_layer2 = Embedding(vocab_size, dim_size, weights=[embedding_matrix], input_length=MAX_LENGTH, trainable=False) embedding_sequences2 = embedding_layer2(sequence_input2) y = Conv1D(filters=100, kernel_size=4, activation='relu')(embedding_sequences2) y = Dropout(0.5)(y) y = Flatten()(y)
def SentenceClassificationSupervised(self): MAX_NB_WORDS = 50000 print(time.asctime()) print('reading labeled csv and tokenizing') #df=pd.read_csv(self.basPath+'TRAIN_1.csv') self.sup_sentence_df=pd.read_csv(self.config['TRAIN']) df=self.sup_sentence_df assert len(df)>0 ,' empty TRAIN FILE' #build a tokenizer self.tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True) #tokenising the sentences self.tokenizer.fit_on_texts(df['sentence'].values) word_index = self.tokenizer.word_index print('read words and tokenized') print(time.asctime()) # load the dataset but only keep the top n words, zero the rest top_words = 50000 #convert sentence array to a sequence of integers X = self.tokenizer.texts_to_sequences(df['sentence'].values) Y = df['class_int'].values X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.2, random_state = 42) #print('Shape of data tensor:', X.shape) print(X_train[0]) print(Y_train[0]) #maximum length of a sequence 50 words max_length = 50 X_train = sequence.pad_sequences(X_train, maxlen=max_length) X_test = sequence.pad_sequences(X_test, maxlen=max_length) # create the model embedding_vecor_length = 100 model = Sequential() #building layers of ANN Model model.add(Embedding(top_words, embedding_vecor_length, input_length=max_length)) model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(LSTM(100)) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) model.fit(X_train, Y_train, epochs=2, batch_size=64) # Final evaluation of the model scores = model.evaluate(X_test, Y_test, verbose=0) print("Accuracy: %.2f%%" % (scores[1]*100)) Y_pred=model.predict_classes(X_test) self.ML_model=model # catch the metriccs of evaluation for Classification tn,fp,fn,tp=sm.confusion_matrix(Y_test,Y_pred).ravel() precision=tp/(tp+fp) recall=tp/(tp+fn) f1=2*precision*recall/(precision+recall) print("PRECISION: %.2f%%" % precision) print("RECALL: %.2f%%" % recall) print("F1 SCORE: %.2f%%" % f1) #df['PREDICTION']=Y_pred pickle.dump(self.tokenizer,open(self.basPath+"tokenizer.h5","wb")) model.save(self.basPath+'model') self.sup_sentence_df.to_csv(self.basPath+'PREDICTION.csv')
print("DONE") ####################### for fixed size filter CNN-LSTM from keras.models import Sequential from keras.layers import Dense from keras.layers import * from keras.layers.convolutional import Conv1D from keras.layers.convolutional import MaxPooling1D from keras.layers.embeddings import Embedding from keras.preprocessing import sequence model = Sequential() model.add(Embedding(input_dim=len(word_index)+1, output_dim=embedding_dim, weights=[embedding_matrix], input_length=sequence_length)) model.add(Conv1D(filters=1024, kernel_size=3, padding='same', activation='tanh')) model.add(MaxPooling1D(pool_size=4)) model.add(LSTM(100)) model.add(Dropout(0.1)) model.add(Dense(6, activation='softmax')) checkpoint = ModelCheckpoint('./CNN-2LSTM-SWBD/weights.{epoch:03d}-{val_acc:.4f}.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max') stop = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='max') adam = Adam(lr=0.01, decay=0.7) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) print(model.summary()) model.fit(X_train, y_train, epochs=100, callbacks=[checkpoint, stop], batch_size=64, validation_data=(X_test, y_test)) # Final evaluation of the model scores = model.evaluate(X_test, y_test, verbose=0) print("Accuracy: %.2f%%" % (scores[1]*100)) predicted=model.predict(X_test)