def _get_train_test_and_embedding(train_csv, test_csv, sentence_length, embedding_file=None): UNKNOWN_WORD = "_UNK_" END_WORD = "_END_" NAN_WORD = "_NAN_" CLASSES = [ "toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate" ] train_data = pd.read_csv(train_csv) test_data = pd.read_csv(test_csv) list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values y_train = train_data[CLASSES].values print("Tokenizing sentences in train set...") tokenized_sentences_train, words_dict = tokenize_sentences( list_sentences_train, {}) print("Tokenizing sentences in test set...") tokenized_sentences_test, words_dict = tokenize_sentences( list_sentences_test, words_dict) words_dict[UNKNOWN_WORD] = len(words_dict) print("Loading embeddings...") embedding_list, embedding_word_dict = read_embedding_list(embedding_file) embedding_size = len(embedding_list[0]) print("Preparing data...") embedding_list, embedding_word_dict = clear_embedding_list( embedding_list, embedding_word_dict, words_dict) embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict) embedding_list.append([0.] * embedding_size) embedding_word_dict[END_WORD] = len(embedding_word_dict) embedding_list.append([-1.] * embedding_size) embedding_matrix = np.array(embedding_list) id_to_word = dict((id, word) for word, id in words_dict.items()) train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train, id_to_word, embedding_word_dict, sentence_length) test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test, id_to_word, embedding_word_dict, sentence_length) X_train = np.array(train_list_of_token_ids) X_test = np.array(test_list_of_token_ids) return X_train, y_train, X_test, embedding_matrix
def main(): parser = argparse.ArgumentParser( description="Recurrent neural network for identifying and classifying toxic online comments") parser.add_argument("train_file_path") parser.add_argument("test_file_path") parser.add_argument("embedding_path") parser.add_argument("--result-path", default="toxic_results") parser.add_argument("--batch-size", type=int, default=256) parser.add_argument("--sentences-length", type=int, default=500) parser.add_argument("--recurrent-units", type=int, default=64) parser.add_argument("--dropout-rate", type=float, default=0.3) parser.add_argument("--dense-size", type=int, default=32) parser.add_argument("--fold-count", type=int, default=10) args = parser.parse_args() if args.fold_count <= 1: raise ValueError("fold-count should be more than 1") print("Loading data...") train_data = pd.read_csv(args.train_file_path) test_data = pd.read_csv(args.test_file_path) list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values y_train = train_data[CLASSES].values n_classes = y_train.nunique() y_train = keras.utils.to_categorical(y_train, num_classes=n_classes) print("Tokenizing sentences in train set...") tokenized_sentences_train, words_dict = tokenize_sentences(list_sentences_train, {}) print("Tokenizing sentences in test set...") tokenized_sentences_test, words_dict = tokenize_sentences(list_sentences_test, words_dict) words_dict[UNKNOWN_WORD] = len(words_dict) print("Loading embeddings...") embedding_list, embedding_word_dict = read_embedding_list(args.embedding_path) embedding_size = len(embedding_list[0]) print("Preparing data...") embedding_list, embedding_word_dict = clear_embedding_list(embedding_list, embedding_word_dict, words_dict) embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict) embedding_list.append([0.] * embedding_size) embedding_word_dict[END_WORD] = len(embedding_word_dict) embedding_list.append([-1.] * embedding_size) embedding_matrix = np.array(embedding_list) id_to_word = dict((id, word) for word, id in words_dict.items()) train_list_of_token_ids = convert_tokens_to_ids( tokenized_sentences_train, id_to_word, embedding_word_dict, args.sentences_length) test_list_of_token_ids = convert_tokens_to_ids( tokenized_sentences_test, id_to_word, embedding_word_dict, args.sentences_length) X_train = np.array(train_list_of_token_ids) X_test = np.array(test_list_of_token_ids) get_model_func = lambda: get_model( embedding_matrix, args.sentences_length, args.dropout_rate, args.recurrent_units, args.dense_size) print("Starting to train models...") models = train_folds(X_train, y_train, args.fold_count, args.batch_size, get_model_func) if not os.path.exists(args.result_path): os.mkdir(args.result_path) print("Predicting results...") test_predicts_list = [] for fold_id, model in enumerate(models): model_path = os.path.join(args.result_path, "model{0}_weights.npy".format(fold_id)) np.save(model_path, model.get_weights()) test_predicts_path = os.path.join(args.result_path, "test_predicts{0}.npy".format(fold_id)) test_predicts = model.predict(X_test, batch_size=args.batch_size) test_predicts_list.append(test_predicts) np.save(test_predicts_path, test_predicts) test_predicts = np.ones(test_predicts_list[0].shape) for fold_predict in test_predicts_list: test_predicts *= fold_predict test_predicts **= (1. / len(test_predicts_list)) test_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT #test_ids = test_data["id"].values #test_ids = test_ids.reshape((len(test_ids), 1)) y_pred = test_predicts.argmax(axis=-1) #test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES) #test_predicts["id"] = test_ids #test_predicts = test_predicts[["id"] + CLASSES] test_data['emotion_pred'] = y_pred submit_path = os.path.join(args.result_path, "submit") test_predicts.to_csv(submit_path, index=False)
def main(): parser = argparse.ArgumentParser( description= "Recurrent neural network for identifying and classifying toxic online comments" ) parser.add_argument("train_file_path") parser.add_argument("test_file_path") parser.add_argument("embedding_path") parser.add_argument("--result-path", default="toxic_results") parser.add_argument("--batch-size", type=int, default=256) parser.add_argument("--sentences-length", type=int, default=500) parser.add_argument("--recurrent-units", type=int, default=64) parser.add_argument("--dropout-rate", type=float, default=0.3) parser.add_argument("--dense-size", type=int, default=32) parser.add_argument("--fold-count", type=int, default=10) # parser.add_argument("train_file_path") # parser.add_argument("test_file_path") # parser.add_argument("embedding_path") # parser.add_argument("--result-path", default="toxic_results") # parser.add_argument("--batch-size", type=int, default=32) # parser.add_argument("--sentences-length", type=int, default=200) # parser.add_argument("--recurrent-units", type=int, default=80) # parser.add_argument("--dropout-rate", type=float, default=0.3) # parser.add_argument("--dense-size", type=int, default=32) # parser.add_argument("--fold-count", type=int, default=10) args = parser.parse_args() if args.fold_count <= 1: raise ValueError("fold-count should be more than 1") print("Loading data...") train_data = pd.read_csv(args.train_file_path) test_data = pd.read_csv(args.test_file_path) list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values y_train = train_data[CLASSES].values print("Tokenizing sentences in train set...") tokenized_sentences_train, words_dict = tokenize_sentences( list_sentences_train, {}) print("Tokenizing sentences in test set...") tokenized_sentences_test, words_dict = tokenize_sentences( list_sentences_test, words_dict) words_dict[UNKNOWN_WORD] = len(words_dict) print("Loading embeddings...") embedding_list, embedding_word_dict = read_embedding_list( args.embedding_path) embedding_size = len(embedding_list[0]) print("Preparing data...") embedding_list, embedding_word_dict = clear_embedding_list( embedding_list, embedding_word_dict, words_dict) embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict) embedding_list.append([0.] * embedding_size) embedding_word_dict[END_WORD] = len(embedding_word_dict) embedding_list.append([-1.] * embedding_size) embedding_matrix = np.array(embedding_list) id_to_word = dict((id, word) for word, id in words_dict.items()) train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train, id_to_word, embedding_word_dict, args.sentences_length) test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test, id_to_word, embedding_word_dict, args.sentences_length) X_train = np.array(train_list_of_token_ids) X_test = np.array(test_list_of_token_ids) get_model_func = lambda: get_model(embedding_matrix, args.sentences_length, args.dropout_rate, args.recurrent_units, args.dense_size) import gc del train_data, test_data, list_sentences_train, list_sentences_test del tokenized_sentences_train, tokenized_sentences_test, words_dict del embedding_list, embedding_word_dict del train_list_of_token_ids, test_list_of_token_ids gc.collect() print("Starting to train models...") models = train_folds(X_train, y_train, X_test, args.fold_count, args.batch_size, get_model_func)
list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values y_train = train_data[CLASSES].values #marking comments without any tags as "clean" rowsums = train_data.iloc[:, 2:8].sum(axis=1) train_data['clean'] = (rowsums == 0) #count number of clean entries train_data['clean'].sum() print("Total comments = ", len(train_data)) print("Total clean comments = ", train_data['clean'].sum()) print("Total tags =", rowsums.sum()) print("Tokenizing sentences in train set...") tokenized_sentences_train, words_dict = tokenize_sentences( list_sentences_train, {}) print("Tokenizing sentences in test set...") tokenized_sentences_test, words_dict = tokenize_sentences( list_sentences_test, words_dict) print(embedding_path) words_dict[UNKNOWN_WORD] = len(words_dict) print("Loading embeddings...") embedding_list, embedding_word_dict = read_embedding_list( file_path=embedding_path) embedding_size = len(embedding_list[0]) import numpy as np
def main(): parser = argparse.ArgumentParser( description= "Recurrent neural network for identifying and classifying toxic online comments" ) parser.add_argument("train_file_path") parser.add_argument("test_file_path") parser.add_argument("embedding_path") parser.add_argument("--result-path", default="toxic_results") parser.add_argument("--batch-size", type=int, default=256) parser.add_argument("--sentences-length", type=int, default=500) parser.add_argument("--recurrent-units", type=int, default=64) parser.add_argument("--dropout-rate", type=float, default=0.3) parser.add_argument("--dense-size", type=int, default=32) parser.add_argument("--fold-count", type=int, default=10) args = parser.parse_args() if args.fold_count <= 1: raise ValueError("fold-count should be more than 1") print("Loading data...") train_data = pd.read_csv(args.train_file_path) test_data = pd.read_csv(args.test_file_path) # Identify language #train_data['language'] = train_data['comment_text'].apply(detect_language) #test_data['language'] = test_data['comment_text'].apply(detect_language) # Translate the non-english to the english. #train_data['comment_text'] = train_data.apply(lambda x: translate(x.comment_text, x.language),axis=1) #test_data['comment_text'] = test_data.apply(lambda x: translate(x.comment_text, x.language),axis=1) #train_data.to_csv("train_data_translated.csv") #test_data.to_csv("test_data_translated.csv") #train_data['comment_text'] = train_data.apply(lambda x: clean(x.comment_text), axis=1) #train_data['comment_text'] = train_data.apply(lambda x: clean(x.comment_text), axis=1) #train_data.to_csv("train_data_cleaned_after_translate.csv") #test_data.to_csv("test_data_cleaned_after_translate.csv") list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values y_train = train_data[CLASSES].values print("Tokenizing sentences in train set...") tokenized_sentences_train, words_dict = tokenize_sentences( list_sentences_train, {}) print("Tokenizing sentences in test set...") tokenized_sentences_test, words_dict = tokenize_sentences( list_sentences_test, words_dict) words_dict[UNKNOWN_WORD] = len(words_dict) print("Loading embeddings...") embedding_list, embedding_word_dict = read_embedding_list( args.embedding_path) embedding_size = len(embedding_list[0]) print("Preparing data...") embedding_list, embedding_word_dict = clear_embedding_list( embedding_list, embedding_word_dict, words_dict) embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict) embedding_list.append([0.] * embedding_size) embedding_word_dict[END_WORD] = len(embedding_word_dict) embedding_list.append([-1.] * embedding_size) embedding_matrix = np.array(embedding_list) embedding_matrix_path = os.path.join(args.result_path, "embedding_matrix.npy") np.save(embedding_matrix_path, embedding_matrix) words_dict_path = os.path.join(args.result_path, "words_dict.npy") np.save(words_dict_path, words_dict) id_to_word = dict((id, word) for word, id in words_dict.items()) train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train, id_to_word, embedding_word_dict, args.sentences_length) test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test, id_to_word, embedding_word_dict, args.sentences_length) X_train = np.array(train_list_of_token_ids) X_test = np.array(test_list_of_token_ids) print(embedding_matrix.shape) print(embedding_matrix.shape[0]) print(embedding_matrix.shape[1]) get_model_func = lambda: get_model(embedding_matrix, args.sentences_length, args.dropout_rate, args.recurrent_units, args.dense_size) print("Starting to train models...") models = train_folds(X_train, y_train, args.fold_count, args.batch_size, get_model_func) if not os.path.exists(args.result_path): os.mkdir(args.result_path) print("Predicting results...") test_predicts_list = [] for fold_id, model in enumerate(models): model_path = os.path.join(args.result_path, "model{0}_weights.npy".format(fold_id)) np.save(model_path, model.get_weights()) model.save_weights("model{0}_weights.h5".format(fold_id)) test_predicts_path = os.path.join( args.result_path, "test_predicts{0}.npy".format(fold_id)) test_predicts = model.predict(X_test, batch_size=args.batch_size) test_predicts_list.append(test_predicts) np.save(test_predicts_path, test_predicts) test_predicts = np.ones(test_predicts_list[0].shape) for fold_predict in test_predicts_list: test_predicts *= fold_predict test_predicts **= (1. / len(test_predicts_list)) test_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT test_ids = test_data["id"].values test_ids = test_ids.reshape((len(test_ids), 1)) test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES) test_predicts["id"] = test_ids test_predicts = test_predicts[["id"] + CLASSES] submit_path = os.path.join(args.result_path, "submit") test_predicts.to_csv(submit_path, index=False) print("Predicting Discussion posts...") posts = pd.read_csv("posts_cleaned.csv") posts = posts.dropna() discussion_posts = posts['MSG_TEXT'].tolist() tokenized_discussion_posts, words_dict = tokenize_sentences( discussion_posts, words_dict) #id_to_word = dict((id, word) for word, id in words_dict.items()) discussion_list_of_token_ids = convert_tokens_to_ids( tokenized_discussion_posts, id_to_word, embedding_word_dict, args.sentences_length) X_test = np.array(discussion_list_of_token_ids) discussion_predict_list = [] for fold_id, model in enumerate(models): discussion_predicts = model.predict(X_test, batch_size=args.batch_size) discussion_predict_list.append(discussion_predicts) discussion_predicts = np.ones(discussion_predict_list[0].shape) for fold_predict in discussion_predict_list: discussion_predicts *= fold_predict discussion_predicts **= (1. / len(discussion_predict_list)) discussion_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT discussion_predicts = pd.DataFrame(data=discussion_predicts, columns=CLASSES) discussion_predicts['MSG_TEXT'] = discussion_posts discussion_predicts = discussion_predicts[["MSG_TEXT"] + CLASSES] discussion_predicts_path = os.path.join(args.result_path, "discussion_predicts.csv") discussion_predicts.to_csv(discussion_predicts_path, index=False)
def main(): parser = argparse.ArgumentParser( description= "Recurrent neural network for identifying and classifying toxic online comments" ) parser.add_argument("train_file_path") parser.add_argument("test_file_path") parser.add_argument("embedding_path") parser.add_argument("--result-path", default="toxic_results") parser.add_argument("--batch-size", type=int, default=256) parser.add_argument("--sentences-length", type=int, default=500) parser.add_argument("--recurrent-units", type=int, default=64) parser.add_argument("--dropout-rate", type=float, default=0.3) parser.add_argument("--dense-size", type=int, default=32) parser.add_argument("--fold-count", type=int, default=10) parser.add_argument("--modelname-prefix", type=str, default="") parser.add_argument("--cv", type=str, default="True") parser.add_argument("--use-roc", type=str, default="False") args = parser.parse_args() if args.fold_count <= 1: raise ValueError("fold-count should be more than 1") print('Input params') print(args) start = time.time() print('#' * 50) print("Loading data...") print('#' * 50) if os.path.exists( os.path.join( args.result_path, 'tokenized_sentences_train.pkl')) and os.path.exists( os.path.join( args.result_path, 'tokenized_sentences_train.pkl')) and os.path.exists( os.path.join(args.result_path, 'tokenized_sentences_train.pkl')): print('Preprocessed files found. Reading preprocess files') train_data = pd.read_csv(args.train_file_path) test_data = pd.read_csv(args.test_file_path) y_train = train_data[CLASSES].values with open( os.path.join(args.result_path, 'tokenized_sentences_train.pkl'), 'rb') as f: tokenized_sentences_train = pickle.load(f) with open( os.path.join(args.result_path, 'tokenized_sentences_test.pkl'), 'rb') as f: tokenized_sentences_test = pickle.load(f) with open(os.path.join(args.result_path, 'words_dict.pkl'), 'rb') as f: words_dict = pickle.load(f) else: print('Preprocessed files not found.') train_data = pd.read_csv(args.train_file_path) test_data = pd.read_csv(args.test_file_path) list_sentences_train = train_data["comment_text"].fillna( NAN_WORD).values list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values y_train = train_data[CLASSES].values print('#' * 50) print("Tokenizing sentences in train set...") print('#' * 50) tokenized_sentences_train, words_dict = tokenize_sentences( list_sentences_train, {}) print('#' * 50) print("Tokenizing sentences in test set...") print('#' * 50) tokenized_sentences_test, words_dict = tokenize_sentences( list_sentences_test, words_dict) print('Saving preprocess files...') with open( os.path.join(args.result_path, 'tokenized_sentences_train.pkl'), 'wb') as f: pickle.dump(tokenized_sentences_train, f) with open( os.path.join(args.result_path, 'tokenized_sentences_test.pkl'), 'wb') as f: pickle.dump(tokenized_sentences_test, f) with open(os.path.join(args.result_path, 'words_dict.pkl'), 'wb') as f: pickle.dump(words_dict, f) print('total words', len(words_dict)) words_dict[UNKNOWN_WORD] = len(words_dict) print('#' * 50) print("Loading embeddings...") print('#' * 50) if 'glove' in args.embedding_path: print('Reading Glove embedding') embedding_list, embedding_word_dict = read_embedding_list_glove( args.embedding_path) else: print('Reading Fasttext embedding') embedding_list, embedding_word_dict = read_embedding_list( args.embedding_path) embedding_size = len(embedding_list[0]) print('Embedding size', embedding_size) print('#' * 50) print("Preparing data...") print('#' * 50) embedding_list, embedding_word_dict = clear_embedding_list( embedding_list, embedding_word_dict, words_dict) embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict) embedding_list.append([0.] * embedding_size) embedding_word_dict[END_WORD] = len(embedding_word_dict) embedding_list.append([-1.] * embedding_size) embedding_matrix = np.array(embedding_list) print('Embedding matrix shape:', embedding_matrix.shape) id_to_word = dict((id, word) for word, id in words_dict.items()) train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train, id_to_word, embedding_word_dict, args.sentences_length) test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test, id_to_word, embedding_word_dict, args.sentences_length) X_train = np.array(train_list_of_token_ids) X_test = np.array(test_list_of_token_ids) # GRU cross validation # get_model_func = lambda: get_model( # embedding_matrix, # args.sentences_length, # args.dropout_rate, # args.recurrent_units, # args.dense_size) # GRU maxpool, avgpool # get_model_func = lambda: get_model_pool( # embedding_matrix, # args.sentences_length, # args.dropout_rate, # args.recurrent_units, # args.dense_size) # deepmoji style # get_model_func = lambda: get_model_deepmoji_style( # embedding_matrix, # args.sentences_length, # args.dropout_rate, # args.recurrent_units, # args.dense_size) # GRU maxpool, avgpool validation # get_model_func = lambda: get_gru_model( # embedding_matrix, # args.sentences_length, # args.dropout_rate, # args.recurrent_units, # args.dense_size) # GRU maxpool, avgpool + cnn validation # get_model_func = lambda: get_model_pool_gru_cnn( # embedding_matrix, # args.sentences_length, # args.dropout_rate, # args.recurrent_units, # args.dense_size) # dpcnn validation get_model_func = lambda: get_dpcnn_model( embedding_matrix, args.sentences_length, args.dropout_rate, args. dense_size) # lstm with attention cross val # get_model_func = lambda: get_model_att_lstm( # embedding_matrix, # args.sentences_length, # args.dropout_rate, # args.recurrent_units, # args.dense_size) # capsule net # get_model_func = lambda: get_capsnet_model( # embedding_matrix, # args.sentences_length, # args.dropout_rate, # args.recurrent_units, # args.dense_size) print('#' * 50) print("Starting to train models...") print('#' * 50) models = train_folds(X_train, y_train, args.fold_count, args.batch_size, get_model_func, args.cv, args.use_roc) if not os.path.exists(args.result_path): os.mkdir(args.result_path) print('#' * 50) print("Predicting results...") print('#' * 50) if args.cv == "True": test_predicts_list = [] for fold_id, model in enumerate(models): model_path = os.path.join( args.result_path, "{0}_model{1}_weights.npy".format(args.modelname_prefix, fold_id)) np.save(model_path, model.get_weights()) test_predicts_path = os.path.join( args.result_path, "{0}_test_predicts{1}.npy".format(args.modelname_prefix, fold_id)) test_predicts = model.predict(X_test, batch_size=args.batch_size * 2) test_predicts_list.append(test_predicts) np.save(test_predicts_path, test_predicts) test_predicts = np.ones(test_predicts_list[0].shape) for fold_predict in test_predicts_list: test_predicts *= fold_predict test_predicts **= (1. / len(test_predicts_list)) # test_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT test_ids = test_data["id"].values test_ids = test_ids.reshape((len(test_ids), 1)) test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES) test_predicts["id"] = test_ids test_predicts = test_predicts[["id"] + CLASSES] submit_path = os.path.join(args.result_path, "{}_submit".format(args.modelname_prefix)) test_predicts.to_csv(submit_path, index=False) print('#' * 50) print('Prediction Completed...') print('#' * 50) total_time = time.time() - start mins, sec = divmod(total_time, 60) hrs, mins = divmod(mins, 60) print('Total time taken : {:.0f}h {:.0f}m {:.0f}s'.format( hrs, mins, sec)) else: print('No Cross Validation') test_predicts = models.predict(X_test, batch_size=args.batch_size * 2) test_ids = test_data["id"].values test_ids = test_ids.reshape((len(test_ids), 1)) test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES) test_predicts["id"] = test_ids test_predicts = test_predicts[["id"] + CLASSES] submit_path = os.path.join( args.result_path, "{}_submit_nocv".format(args.modelname_prefix)) test_predicts.to_csv(submit_path, index=False) total_time = time.time() - start mins, sec = divmod(total_time, 60) hrs, mins = divmod(mins, 60) print('Total time taken : {:.0f}h {:.0f}m {:.0f}s'.format( hrs, mins, sec))
def main(): parser = argparse.ArgumentParser( description= "Recurrent neural network for identifying and classifying toxic online comments" ) parser.add_argument("train_file_path") parser.add_argument("test_file_path") parser.add_argument("embedding_path") parser.add_argument("--result-path", default="predict_results") parser.add_argument("--batch-size", type=int, default=256) parser.add_argument("--sentences-length", type=int, default=500) parser.add_argument("--recurrent-units", type=int, default=64) parser.add_argument("--dropout-rate", type=float, default=0.3) parser.add_argument("--dense-size", type=int, default=32) parser.add_argument("--fold-count", type=int, default=10) args = parser.parse_args() if args.fold_count <= 1: raise ValueError("fold-count should be more than 1") start = time.time() print("Loading data...") train_data = pd.read_csv(args.train_file_path) test_data = pd.read_csv(args.test_file_path) list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values y_train = train_data[CLASSES].values # print("list_sentences_test = ", list_sentences_test) print("Tokenizing sentences in train set...") tokenized_sentences_train, words_dict = tokenize_sentences( list_sentences_train, {}) # print("words_dict before = ", words_dict) print("Tokenizing sentences in test set...") tokenized_sentences_test, words_dict = tokenize_sentences( list_sentences_test, words_dict) # print("words_dict after = ", words_dict) words_dict[UNKNOWN_WORD] = len(words_dict) print("Loading embeddings...") embedding_list, embedding_word_dict = read_embedding_list( args.embedding_path) embedding_size = len(embedding_list[0]) """ with open('embedding_list.txt', 'wb') as file: pickle.dump(embedding_list, file) with open('embedding_word_dict.txt', 'wb') as file: pickle.dump(embedding_word_dict, file) with open('embedding_list.txt', 'rb') as file: embedding_list = pickle.load(file) with open('embedding_word_dict.txt', 'rb') as file: embedding_word_dict = pickle.load(file) print(embedding_list) print(embedding_word_dict) """ end = time.time() print("Time elapsed: " + end - start) print("Preparing data...") embedding_list, embedding_word_dict = clear_embedding_list( embedding_list, embedding_word_dict, words_dict) embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict) embedding_list.append([0.] * embedding_size) embedding_word_dict[END_WORD] = len(embedding_word_dict) embedding_list.append([-1.] * embedding_size) embedding_matrix = np.array(embedding_list) id_to_word = dict((id, word) for word, id in words_dict.items()) train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train, id_to_word, embedding_word_dict, args.sentences_length) test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test, id_to_word, embedding_word_dict, args.sentences_length) X_train = np.array(train_list_of_token_ids) X_test = np.array(test_list_of_token_ids) get_model_func = lambda: get_model(embedding_matrix, args.sentences_length, args.dropout_rate, args.recurrent_units, args.dense_size) print("Starting to train models...") models = train_folds(X_train, y_train, args.fold_count, args.batch_size, get_model_func) if not os.path.exists(args.result_path): os.mkdir(args.result_path) print("Predicting results...") test_predicts_list = [] for fold_id, model in enumerate(models): model_path = os.path.join(args.result_path, "model{0}_weights.npy".format(fold_id)) np.save(model_path, model.get_weights()) test_predicts_path = os.path.join( args.result_path, "test_predicts{0}.npy".format(fold_id)) test_predicts = model.predict(X_test, batch_size=args.batch_size) test_predicts_list.append(test_predicts) np.save(test_predicts_path, test_predicts) test_predicts = np.ones(test_predicts_list[0].shape) for fold_predict in test_predicts_list: print("fold_predict = ", fold_predict) test_predicts *= fold_predict test_predicts **= (1. / len(test_predicts_list)) test_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT test_ids = test_data["id"].values test_ids = test_ids.reshape((len(test_ids), 1)) print("test_predicts = ", test_predicts) test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES) test_predicts["id"] = test_ids test_predicts = test_predicts[["id"] + CLASSES] # Route results to a CSV file: "finished.csv" with path "predict_results/finished" submit_path = os.path.join(args.result_path, "finished") test_predicts.to_csv(submit_path, index=False)