def _train(self): data, label = self._load_data() train_data, train_label, validate_data, validate_label, test_data, test_label = split_data(data, label, to_categorical=True) network_input = Input(shape=(100, 3)) network = LSTM(32, return_sequences=True)(network_input) network = LSTM(32)(network) network = Dense(5, activation=softmax)(network) network = Model(inputs=[network_input], outputs=[network]) network.compile(optimizer=RMSprop(lr=0.01), loss=categorical_crossentropy, metrics=[categorical_accuracy]) network.summary() callback = [ callbacks.ReduceLROnPlateau(monitor="categorical_accuracy", factor=0.1, patience=3) ] self.train_history = network.fit(train_data, train_label, validation_data=(validate_data, validate_label), batch_size=self.BATCH_SIZE, epochs=self.EPOCHS, callbacks=callback) self.evaluate_history = network.evaluate(test_data, test_label, batch_size=self.BATCH_SIZE) return network
def run_LSTM(X_train, X_test, y_train, y_test, symbol, price): y_train = to_categorical(y_train) y_test = to_categorical(y_test) LSTM = create_LSTM(X_train) history = LSTM.fit(X_train, y_train, epochs=100, batch_size=32) print(LSTM.summary()) results = LSTM.evaluate(X_test, y_test, batch_size=32) print("test loss, test acc:", results) # summarize history for accuracy plt.plot(history.history['accuracy'], label='Train') plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() prediction = LSTM.predict(X_test) y_prediction = prediction[:, -1, :] y_prediction = np.argmax(prediction, axis=1) print(y_prediction) NAV_history = Generate_nav(10000, symbol, price, y_prediction)
conv5 = Conv1D(32, 5, activation='relu', padding='same')(input1) conv5 = Dropout(DropoutRate)(conv5) conv5 = Conv1D(32, 3, activation='relu', padding='same')(conv5) conv5 = Dropout(DropoutRate)(conv5) conv5 = Conv1D(32, 3, activation='relu', padding='same')(conv5) # model model = concatenate([conv1, conv3, conv5], axis=-1) # LSTM Layer model = LSTM(32, return_sequences=True)(model) model = LSTM(32, return_sequences=False)(model) output = Dense(3, activation='softmax')(model) model = Model(input1, output) model.summary() # Optimizer = optimizers.Adam(lr = 0.00001) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) callback_list = [ ReduceLROnPlateau( # Model의 Val_loss를 Monitoring monitor='val_loss', # Callback 호출시 Learning rate를 1/10으로 줄임 factor=0.1, # Val_loss가 5 Epoch동안 개선되지 않을 경우 CallBack 호출 patience=5),
def CNN_model(filename, train, X_train, X_test, word2ind, maxWords, y_train, y_test, ind2label, maxChar, char2ind, validation=False, X_valid=None, y_valid=None, pretrained_embedding="", word_embedding_size=100, char_embedding_size=50, lstm_hidden=32, nbr_epochs=25, batch_size=128, dropout=0.5, optimizer='rmsprop', early_stopping_patience=-1, folder_path="CNN_results", print_to_file = True, gen_confusion_matrix=False, return_model = False ): """ Build, train and test the CNN-CNN-LSTM Keras model. Works for multi-tasking learning. The model architecture looks like: - CNN character-level representation - CNN word-level + character representation - LSTM - Softmax for prediction :param filename: File to redirect the printing :param train: Boolean if the model must be trained or not. If False, the model's wieght are expected to be stored in "folder_path/filename/filename.h5" :param X_train: Data to train the model. It must be a list of word and character indices. :param X_test: Data to test the model. It must be a list of word and character indices. :param word2ind: Dictionary containing all words in the training data and a unique integer per word :param maxWords: Maximum number of words in a sequence :param y_train: Labels to train the model for the prediction task :param y_test: Labels to test the model for the prediction task :param ind2label: Dictionary where all labels are mapped into a unique integer :param maxChar: The maximum numbers of characters in a word. If set to 0, the model will not use character-level representations of the words :param char2ind: A dictionary where each character is mapped into a unique integer :param validation: Boolean. If true, the validation score will be computed from 'X_valid' and 'y_valid' :param X_valid: Optional. Validation dataset :param y_valid: Optional. Validation dataset labels :param pretrained_embedding: Use the pretrained word embeddings. Pretrained vectors must be located here: "dataset/pretrained_vectors" Three values: - "": Do not use pre-trained word embeddings (Default) - False: Use the pre-trained embedding vectors as the weights in the Embedding layer - True: Use the pre-trained embedding vectors as weight initialiers. The Embedding layer will still be trained. :param word_embedding_size: Size of the pre-trained word embedding to use (100 or 300) :param char_embedding_size: size of the character-level word representations :param lstm_hidden: Dimentionality of the LSTM output space :param nbr_epochs: Number of epochs to train the model :param batch_size: Size of batches while training the model :param dropout: Rate to apply for each Dropout layer in the model :param optimizer: Optimizer to use while compiling the model :param early_stopping_patience: Number of continuous tolerated epochs without improvement during training. :param folder_path: Path to the directory storing all to-be-generated files :param print_to_file: if True redirects the printings to a file (given in filename), if False std_out is kept :param gen_confusion_matrix: Boolean value. Generated confusion matrices or not. :parm return_model: if True returns the Keras model object, otherwise return best results :return: The classification scores for both tasks (default for compatibility). If returnModel = True, returns model object for further computation """ print("====== {0} start ======".format(filename)) end_string = "====== {0} end ======".format(filename) os.makedirs(folder_path+"/"+filename, exist_ok=True) filepath = folder_path+"/"+filename+"/"+filename # Set print outputs file if print_to_file: file, stdout_original = setPrintToFile("{0}.txt".format(filepath)) nbr_words = len(word2ind)+1 out_size = len(ind2label)+1 nbr_chars = len(char2ind)+1 best_results = None # Embedding - Characters character_input = Input((maxWords,maxChar,)) embed_char_out = TimeDistributed(Embedding(nbr_chars, char_embedding_size))(character_input) conv1d_out = TimeDistributed(Convolution1D(filters = 50, kernel_size = 3, strides=1, activation="relu", padding='same'))(embed_char_out) pool_out = TimeDistributed(MaxPooling1D(pool_size=2, strides=1, padding='same'))(conv1d_out) char_enc = TimeDistributed(Flatten())(pool_out) # Embedding - Words word_input = Input((maxWords,)) if pretrained_embedding=="": word_emb = Embedding(nbr_words, word_embedding_size)(word_input) else: # Retrieve embeddings from word2vec embedding_matrix = word2VecEmbeddings(word2ind, word_embedding_size) word_emb = Embedding(nbr_words, word_embedding_size, weights=[embedding_matrix], trainable=pretrained_embedding, mask_zero=False)(word_input) # Model inputs inputs = [word_input, character_input] # Full word representation word_full = concatenate([char_enc, word_emb]) # encode words w_full = w_char + w_embed conv1d_w1_out = Convolution1D(filters = 800, kernel_size = 5, strides=1, activation="relu", padding='same')(word_full) drop_w1_layer = Dropout(dropout, noise_shape=None, seed=None)(conv1d_w1_out) conv1d_w2_out = Convolution1D(filters = 800, kernel_size = 5, strides=1, activation="relu", padding='same')(drop_w1_layer) # pool size out? word_enc = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv1d_w2_out) # LSTM layer model = LSTM(lstm_hidden, return_sequences=True, dropout=dropout)(word_enc) # Output - Softmax outputs = [Dense(out_size, activation='softmax')(model) for out_size in [len(x)+1 for x in ind2label]] model_loss = ['categorical_crossentropy' for x in outputs] model_metrics = None # Model model = Model(inputs=inputs, outputs=outputs) model.compile(loss=model_loss, metrics=model_metrics, optimizer=get_optimizer(optimizer)) print(model.summary(line_length=150),"\n\n\n\n") # Training Callbacks: callbacks = [] value_to_monitor = 'val_f1' best_model_weights_path = "{0}.h5".format(filepath) # 1) Classifition scores classification_scores = Classification_Scores([X_train, y_train], ind2label, best_model_weights_path) callbacks.append(classification_scores) # 2) EarlyStopping if early_stopping_patience != -1: early_stopping = EarlyStopping(monitor=value_to_monitor, patience=early_stopping_patience, mode='max') callbacks.append(early_stopping) else: early_stopping = EarlyStopping(monitor=value_to_monitor, patience=nbr_epochs, mode='max') callbacks.append(early_stopping) # Train if train: # Train the model. Keras's method argument 'validation_data' is referred as 'testing data' in this code. hist = model.fit(X_train, y_train, validation_data=[X_test, y_test], epochs=nbr_epochs, batch_size=batch_size, callbacks=callbacks, verbose=2) print() print('-------------------------------------------') print("Best F1 score:", early_stopping.best, " (epoch number {0})".format(1+np.argmax(hist.history[value_to_monitor]))) # Save Training scores save_model_training_scores("{0}".format(filepath), hist, classification_scores) # Print best testing classification report best_epoch = np.argmax(hist.history[value_to_monitor]) print(classification_scores.test_report[best_epoch]) # Best epoch results best_results = model_best_scores(classification_scores, best_epoch) # HACK: optmizer weight length issue # https://github.com/keras-team/keras/issues/4044 import h5py with h5py.File(best_model_weights_path, 'a') as f: if 'optimizer_weights' in f.keys(): del f['optimizer_weights'] # Load weigths from best training epoch into model save_load_utils.load_all_weights(model, best_model_weights_path) # Create confusion matrices if gen_confusion_matrix: for i, y_target in enumerate(y_test): # Compute predictions, flatten predictions, target = compute_predictions(model, X_test, y_target, ind2label[i]) # Generate confusion matrices save_confusion_matrix(target, predictions, list(ind2label[i].values()), "{0}_task_{1}_confusion_matrix_test".format(filepath,str(i+1))) # Validation dataset if validation: print() print("Validation dataset") print("======================") # Compute classification report for i, y_target in enumerate(y_valid): # Compute predictions, flatten predictions, target = compute_predictions(model, X_valid, y_target, ind2label[i], nbrTask=i) # Only for multi-task if len(y_train) > 1: print("For task "+str(i+1)+"\n") print("====================================================================================") print("") print("With padding into account") print(metrics.flat_classification_report([target], [predictions], digits=4)) print("") print('----------------------------------------------') print("") print("Without the padding:") print(metrics.flat_classification_report([target], [predictions], digits=4, labels=list(ind2label[i].values()))) # Generate confusion matrices save_confusion_matrix(target, predictions, list(ind2label[i].values()), "{0}_task_{1}_confusion_matrix_validation".format(filepath,str(i+1))) # Close file if print_to_file: closePrintToFile(file, stdout_original) print(end_string) # Returns model itself for further computation, otherwise best results if return_model: return model else: return best_results