def run_image_cnn_model(model, train_data, train_labels, test_data, test_labels): print('### CNN ###') # compile the model model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy', 'mse']) train_data, val_data, train_labels, val_labels = ch.get_train_test_from_data( train_data, train_labels) # TODO: # test if it is multi label or not # train the model model.fit(train_data, train_labels, epochs=5) # , # batch_size=512, # validation_data=(val_data, val_labels), # verbose=1) # check on the test dataset test_loss, test_acc, test_mse = model.evaluate(test_data, test_labels) # make predictions predictions = model.predict(test_data) return [test_loss, test_acc, test_mse], predictions
def run_text_cnn_model(model, train_data, train_labels, test_data, test_labels): print('### CNN ###') # compile the model model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', 'mse']) train_data, val_data, train_labels, val_labels = ch.get_train_test_from_data( train_data, train_labels) # TODO: convert to multi label classification # train_labels = train_labels[:, 0] # val_labels = val_labels[:, 0] # train the model history = model.fit(train_data, train_labels, epochs=40, batch_size=512, validation_data=(val_data, val_labels), verbose=1) # check on the test dataset test_loss, test_acc, test_mse = model.evaluate(test_data, test_labels) predictions = model.predict(test_data) return [test_loss, test_acc, test_mse], predictions
def apply_word2vec_extratrees(data_frame, classif_level, classif_type, source_path): data_frame['text'] = data_frame.apply( lambda row: th.tokenize_complex_text(row['text']), axis=1) data_frame['classification'] = data_frame.apply( lambda row: th.tokenize_complex_text(row['classification']), axis=1) df_single_classification = ch.get_list_each_text_a_different_classification( data_frame) x = df_single_classification['text'] y = df_single_classification['classification'] X_train, X_test, y_train, y_test = ch.get_train_test_from_data(x, y) model_w2v = wmh.get_word2vec_model(X_train) etree_w2v = Pipeline([("word2vec vectorizer", wmh.MeanEmbeddingVectorizer(model_w2v)), ("extra trees", pmh.get_extra_tree())]) etree_w2v_tfidf = Pipeline([("word2vec vectorizer", wmh.TfidfEmbeddingVectorizer(model_w2v)), ("extra trees", pmh.get_extra_tree())]) # NB!!!: the model does not support multi targets, so i duplicate the sources and give them different targets y_pred = pmh.fit_predict_functions(etree_w2v_tfidf, X_train, y_train, X_test) classifier_name_0 = 'Word2Vec/MeanEmbeddingVectorizer' classifier_name_1, parameters_1 = ch.get_extratree_classifier_information( str(etree_w2v)) model_name = '[all classes predictions]' + classifier_name_0 + '/' + classifier_name_1 # this should be changed by comparing all the possibilities for specified text (i can use the original dataframe!) list_metrics = mh.calculate_metrics(model_name, y_test, y_pred) none_average, binary_average, micro_average, macro_average = list_metrics ch.save_results(classifier_name_1, list_metrics, parameters_1, model_name, classif_level, classif_type, source_path) # NB!!!: the model does not support multi targets, so i duplicate the sources and give them different targets y_pred = pmh.fit_predict_functions(etree_w2v, X_train, y_train, X_test) classifier_name_2 = 'Word2Vec/TfidfEmbeddingVectorizer' model_name = '[all classes predictions]' + classifier_name_2 + '/' + classifier_name_1 # this should be changed by comparing all the possibilities for specified text (i can use the original dataframe!) list_metrics = mh.calculate_metrics(model_name, y_test, y_pred) none_average, binary_average, micro_average, macro_average = list_metrics ch.save_results(classifier_name_1, list_metrics, parameters_1, model_name, classif_level, classif_type, source_path)
def second_attempt_from_web(data_frame, text_vectorizer, class_vectorizer, classif_level, classif_type, dataset_location): root_location = fh.get_root_location('data/convolutional_outcome/') # imdb = keras.datasets.imdb # (train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000) # explore data # print("Training entries: {}, labels: {}".format(len(train_data), len(train_labels))) # print("how data looks like: ", train_data[0]) # [1, 14, 34...] # print("how labels looks like: ", train_labels[0]) # 0 # preprocess data # A dictionary mapping words to an integer index # word_index = imdb.get_word_index() # The first indices are reserved, thus it increases all the indices by 3 # word_index = {k : (v + 3) for k, v in word_index.items()} # word_index["<PAD>"] = 0 # word_index["<START>"] = 1 # word_index["<UNK>"] = 2 # unknown # word_index["<UNUSED>"] = 3 # make both the train and the test dataset the same length # train_data = keras.preprocessing.sequence.pad_sequences(train_data, # value=word_index["<PAD>"], # padding='post', # maxlen=256) # test_data = keras.preprocessing.sequence.pad_sequences(test_data, # value=word_index["<PAD>"], # padding='post', # maxlen=256) # print(train_data[0]) # [1 14 34 0 0 0] - with little difference: PAD,START,UNK,UNUSED model_name = text_vectorizer+'/'+class_vectorizer+'/NN' standard_results = ch.apply_df_vectorizer(data_frame, text_vectorizer, class_vectorizer, model_name) train_data, test_data, train_labels, test_labels, classes, n_classes, vocab_processor, len_vocabulary = standard_results train_data, val_data, train_labels, val_labels = ch.get_train_test_from_data(train_data, train_labels) # print("Training entries: {}, labels: {}".format(len(train_data), len(train_labels))) # print("how data looks like: ", train_data[0]) # [1 14 34 0 0 0] # print("how labels looks like: ", train_labels[0:5]) # list of lists [[0 1 0 0 0 1 0 1], ...] model = pmh.get_text_convolutional_from_web(len_vocabulary, n_classes) metrics, predictions = pmh.run_text_cnn_model(model, train_data, train_labels, test_data, test_labels) classifier_name, layers = ch.get_sequential_classifier_information(model) mh.display_convolutional_metrics(classifier_name, metrics[0], metrics[1], metrics[2], test_labels, predictions) ch.save_results(classifier_name, metrics, layers, model_name, classif_level, classif_type, dataset_location)
def apply_multilabel_label_encoder_tfidf_classification( data_frame, classif_level, classif_type, source_path): data_frame = ch.get_list_each_text_a_different_classification(data_frame) temp_text, patent_ids, vectorizer = ch.apply_tfidf_vectorizer_fit_transform( data_frame) X_train_tfidf, X_test_tfidf, y_train, y_test = ch.get_train_test_from_data( temp_text, data_frame['classification']) y_train = ch.apply_label_encoder(y_train) y_test = ch.apply_label_encoder(y_test) apply_naive_bayes(X_train_tfidf, y_train, X_test_tfidf, y_test, classif_level, classif_type, source_path) apply_svm(X_train_tfidf, y_train, X_test_tfidf, y_test, classif_level, classif_type, source_path)
def train_testing_convolution(data_frame, text_vectorizer, class_vectorizer): save_standard_sets = True root_location = fh.get_root_location('data/convolutional_outcome/') sets_location = fh.join_paths(root_location, "model_sets") checkpoint_path = fh.join_paths(root_location, "model_checkpoints") model_path = fh.link_paths(checkpoint_path, 'convolution_model') weights_path = fh.link_paths(checkpoint_path, 'convolution_weights') # get sets model_name = text_vectorizer+'/'+class_vectorizer+'/NN' standard_results = ch.apply_df_vectorizer(data_frame, text_vectorizer, class_vectorizer, model_name) train_data, test_data, train_labels, test_labels, classes, n_classes, vocab_processor, len_vocabulary = standard_results train_data, val_data, train_labels, val_labels = ch.get_train_test_from_data(train_data, train_labels) # save sets # ch.save_sets(sets_location, train_data, test_data, val_data, train_labels, test_labels, val_labels, # [classes, n_classes, vocab_processor, len_vocabulary]) # this is for test train_data, test_data, val_data, train_labels, test_labels, val_labels, _ = ch.load_sets(sets_location) # it could be that a label is only in the test/data data, might be a problem sequence_length = train_data.shape[1] # define the model model = pmh.get_cnn_test(len_vocabulary, n_classes, sequence_length) # calculates metrics with validating data model, val_predictions = pmh.run_cnn_test(model, train_data, train_labels, val_data, val_labels, val_data, val_labels, model_path, weights_path, True) binary_val_predictions = mh.get_binary_0_5(val_predictions) print(val_labels.shape) print(val_predictions.shape) # display validation metrics metrics = mh.get_sequential_metrics(val_labels, val_predictions, binary_predictions) mh.display_sequential_metrics('validation convolution sequence', metrics)
def test_LSTM_from_web(data_frame, text_vectorizer, class_vectorizer, classif_level, classif_type, dataset_location): print('### LSTM Doing Testing ###') root_location = fh.get_root_location('data/lstm_outcome/') nn_parameter_search_location = fh.join_paths(root_location, "nn_fhv_parameter_search") doc2vec_model_location = fh.link_paths(fh.join_paths(root_location, "doc2vec_model/vocab_model/"), "doc2vec_model") save_results = True sequence_size = 1 EMBEDDING_SIZE = 150 model_name = text_vectorizer+'/'+class_vectorizer+'/LSTM' results = ch.apply_df_vectorizer(data_frame, text_vectorizer, class_vectorizer, model_name) X_train, X_test, y_train, y_test, classes, n_classes, vocab_processor, len_vocabulary = results X_train, X_val, y_train, y_val = ch.get_train_test_from_data(X_train, y_train) training_docs_list = X_train['patent_id'] test_docs_list = X_test['patent_id'] val_docs_list = X_val['patent_id'] X_data, Xv_data, Xt_data = ch.get_df_data(3, training_docs_list, val_docs_list, test_docs_list, sequence_size, EMBEDDING_SIZE, doc2vec_model_location) GLOBAL_VARS.DOC2VEC_MODEL_NAME, GLOBAL_VARS.MODEL_NAME = wmh.set_parameters_lstm_doc2vec(nn_parameter_search_location, classif_level, classif_type) NN_INPUT_NEURONS, NN_SEQUENCE_SIZE, NN_OUTPUT_NEURONS = pmh.get_lstm_shapes(X_data, n_classes) NN_BATCH_SIZE, PARTS_LEVEL, NN_MAX_EPOCHS, QUEUE_SIZE = pmh.get_lstm_basic_parameters() params = pmh.get_lstm_testing_parameters() lstm_output_size,w_dropout_do,u_dropout_do, stack_layers, conv_size, conv_filter_length, conv_max_pooling_length = params EARLY_STOPPER_MIN_DELTA, EARLY_STOPPER_PATIENCE = pmh.get_early_stopping_parameters() TEST_METRICS_FILENAME = '{}_level_{}_standard_nn_test_metrics_dict.pkl' test_metrics_dict = dict() test_metrics_path = fh.link_paths(fh.link_paths(nn_parameter_search_location, GLOBAL_VARS.MODEL_NAME), TEST_METRICS_FILENAME.format(classif_type, PARTS_LEVEL)) param_results_path = fh.link_paths(fh.link_paths(nn_parameter_search_location, GLOBAL_VARS.MODEL_NAME), NN_PARAMETER_SEARCH_PREFIX.format(classif_type, classif_level, NN_BATCH_SIZE)) param_results_dict = pickle.load(open(param_results_path, 'rb')) GLOBAL_VARS.NN_MODEL_NAME = 'lstm_size_{}_w-drop_{}_u-drop_{}_stack_{}_conv_{}'.format(lstm_output_size, w_dropout_do, u_dropout_do, stack_layers, str(conv_size) ) if conv_size: GLOBAL_VARS.NN_MODEL_NAME += '_conv-filter-length_{}_max-pooling-size_{}'.format(conv_filter_length, conv_max_pooling_length) if GLOBAL_VARS.NN_MODEL_NAME not in param_results_dict.keys(): print("Can't find model: {}".format(GLOBAL_VARS.NN_MODEL_NAME)) raise Exception() if fh.ensure_exists_path_location(test_metrics_path): test_metrics_dict = pickle.load(open(fh.link_paths(fh.link_paths(nn_parameter_search_location, GLOBAL_VARS.MODEL_NAME), TEST_METRICS_FILENAME.format(classif_type,PARTS_LEVEL)), 'rb')) if GLOBAL_VARS.NN_MODEL_NAME in test_metrics_dict.keys(): print("Test metrics already exist for: {}".format(GLOBAL_VARS.NN_MODEL_NAME)) test_metrics = test_metrics_dict[GLOBAL_VARS.NN_MODEL_NAME] print("** Test Metrics: Cov Err: {:.3f}, Avg Labels: {:.3f}, \n\t\t Top 1: {:.3f}, Top 3: {:.3f}, Top 5: {:.3f}, \n\t\t F1 Micro: {:.3f}, F1 Macro: {:.3f}".format( test_metrics['coverage_error'], test_metrics['average_num_of_labels'], test_metrics['top_1'], test_metrics['top_3'], test_metrics['top_5'], test_metrics['f1_micro'], test_metrics['f1_macro'])) raise Exception() print('***************************************************************************************') print(GLOBAL_VARS.NN_MODEL_NAME) model = pmh.get_keras_rnn_model(NN_INPUT_NEURONS, NN_SEQUENCE_SIZE, NN_OUTPUT_NEURONS, lstm_output_size, w_dropout_do, u_dropout_do, stack_layers, conv_size, conv_filter_length, conv_max_pooling_length) # get model best weights weights = param_results_dict[GLOBAL_VARS.NN_MODEL_NAME]['best_weights'] model.set_weights(weights) print('Evaluating on Test Data using best weights') _, ytp, ytp_binary = pmh.predict_generator(None, model, Xt_data, y_test, NN_BATCH_SIZE, QUEUE_SIZE, test_docs_list) print('Generating Test Metrics') test_metrics = mh.get_sequential_metrics(y_test, ytp, ytp_binary) mh.display_sequential_metrics(test_metrics) if save_results: classifier_name, parameters = ch.get_sequential_classifier_information(model) ch.save_results(classifier_name+'_LSTM', test_metrics, parameters, model_name, classif_level, classif_type, dataset_location) test_metrics_dict[GLOBAL_VARS.NN_MODEL_NAME] = test_metrics pickle.dump(test_metrics_dict, open(test_metrics_path, 'wb'))
def preprocessing_data_for_fasttext(data_frame, text_vectorizer, class_vectorizer): root_location = fh.get_root_location('data/fasttext_outcome/') data_frame['text'] = data_frame['text'].replace( '\n', ' ', regex=True).replace('\t', ' ', regex=True) model_name = text_vectorizer + '/' + class_vectorizer + '/FastText' try: X_train, X_test, Y_train, Y_test, _, _, _, _ = ch.apply_df_vectorizer( data_frame, text_vectorizer, class_vectorizer, model_name) X_train, X_val, Y_train, Y_val = ch.get_train_test_from_data( X_train, Y_train) # self.data_vectors = pd.DataFrame(columns=range(vectors_size), index=range(corpus_size)) if not isinstance(X_train, pd.DataFrame): train = pd.DataFrame(data=X_train) test = pd.DataFrame(data=X_test) val = pd.DataFrame(data=X_val) # test_labels = pd.DataFrame(columns=['']) else: train = X_train test = X_test val = X_val train.loc[:, 1] = Y_train test.loc[:, 1] = Y_test val.loc[:, 1] = Y_val train.drop(columns=['patent_id'], inplace=True) test.drop(columns=['patent_id'], inplace=True) val.drop(columns=['patent_id'], inplace=True) data_frame.to_csv(fh.link_paths(root_location, 'dataframe.csv'), index=False, sep=' ', header=False, quoting=csv.QUOTE_NONE, quotechar="", escapechar=" ") train.to_csv(fh.link_paths(root_location, 'training set.csv'), index=False, sep=' ', header=False, quoting=csv.QUOTE_NONE, quotechar="", escapechar=" ") test.to_csv(fh.link_paths(root_location, 'testing set.csv'), index=False, sep=',', header=False, quoting=csv.QUOTE_NONE, quotechar="", escapechar=" ") except: print('a problem occurred while trying to store the dataframes') X_train, X_test, Y_train, Y_test, _, _, _, _ = ch.apply_df_vectorizer( data_frame, text_vectorizer, class_vectorizer, model_name) X_train, X_val, Y_train, Y_val = ch.get_train_test_from_data( X_train, Y_train) val = pd.DataFrame({'text': X_val, 'classification': Y_val}) train = pd.DataFrame({'text': X_train, 'classification': Y_train}) test = pd.DataFrame({'text': X_test, 'classification': Y_test}) data_frame.to_csv(fh.link_paths(root_location, 'dataframe.csv'), index=False, sep=' ', header=False, quoting=csv.QUOTE_NONE, quotechar="", escapechar=" ") val.to_csv(fh.link_paths(root_location, 'validating set.csv'), index=False, sep=' ', header=False, quoting=csv.QUOTE_NONE, quotechar="", escapechar=" ") train.to_csv(fh.link_paths(root_location, 'training set.csv'), index=False, sep=' ', header=False, quoting=csv.QUOTE_NONE, quotechar="", escapechar=" ") test.to_csv(fh.link_paths(root_location, 'testing set.csv'), index=False, sep=',', header=False, quoting=csv.QUOTE_NONE, quotechar="", escapechar=" ")