def _get_train_test_and_embedding(train_csv,
test_csv,
sentence_length,
embedding_file=None):
UNKNOWN_WORD = "_UNK_"
END_WORD = "_END_"
NAN_WORD = "_NAN_"
CLASSES = [
"toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"
]
train_data = pd.read_csv(train_csv)
test_data = pd.read_csv(test_csv)
list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values
list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values
y_train = train_data[CLASSES].values
print("Tokenizing sentences in train set...")
tokenized_sentences_train, words_dict = tokenize_sentences(
list_sentences_train, {})
print("Tokenizing sentences in test set...")
tokenized_sentences_test, words_dict = tokenize_sentences(
list_sentences_test, words_dict)
words_dict[UNKNOWN_WORD] = len(words_dict)
print("Loading embeddings...")
embedding_list, embedding_word_dict = read_embedding_list(embedding_file)
embedding_size = len(embedding_list[0])
print("Preparing data...")
embedding_list, embedding_word_dict = clear_embedding_list(
embedding_list, embedding_word_dict, words_dict)
embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
embedding_list.append([0.] * embedding_size)
embedding_word_dict[END_WORD] = len(embedding_word_dict)
embedding_list.append([-1.] * embedding_size)
embedding_matrix = np.array(embedding_list)
id_to_word = dict((id, word) for word, id in words_dict.items())
train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train,
id_to_word,
embedding_word_dict,
sentence_length)
test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test,
id_to_word,
embedding_word_dict,
sentence_length)
X_train = np.array(train_list_of_token_ids)
X_test = np.array(test_list_of_token_ids)
return X_train, y_train, X_test, embedding_matrix
def main():
parser = argparse.ArgumentParser(
description="Recurrent neural network for identifying and classifying toxic online comments")
parser.add_argument("train_file_path")
parser.add_argument("test_file_path")
parser.add_argument("embedding_path")
parser.add_argument("--result-path", default="toxic_results")
parser.add_argument("--batch-size", type=int, default=256)
parser.add_argument("--sentences-length", type=int, default=500)
parser.add_argument("--recurrent-units", type=int, default=64)
parser.add_argument("--dropout-rate", type=float, default=0.3)
parser.add_argument("--dense-size", type=int, default=32)
parser.add_argument("--fold-count", type=int, default=10)
args = parser.parse_args()
if args.fold_count <= 1:
raise ValueError("fold-count should be more than 1")
print("Loading data...")
train_data = pd.read_csv(args.train_file_path)
test_data = pd.read_csv(args.test_file_path)
list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values
list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values
y_train = train_data[CLASSES].values
n_classes = y_train.nunique()
y_train = keras.utils.to_categorical(y_train, num_classes=n_classes)
print("Tokenizing sentences in train set...")
tokenized_sentences_train, words_dict = tokenize_sentences(list_sentences_train, {})
print("Tokenizing sentences in test set...")
tokenized_sentences_test, words_dict = tokenize_sentences(list_sentences_test, words_dict)
words_dict[UNKNOWN_WORD] = len(words_dict)
print("Loading embeddings...")
embedding_list, embedding_word_dict = read_embedding_list(args.embedding_path)
embedding_size = len(embedding_list[0])
print("Preparing data...")
embedding_list, embedding_word_dict = clear_embedding_list(embedding_list, embedding_word_dict, words_dict)
embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
embedding_list.append([0.] * embedding_size)
embedding_word_dict[END_WORD] = len(embedding_word_dict)
embedding_list.append([-1.] * embedding_size)
embedding_matrix = np.array(embedding_list)
id_to_word = dict((id, word) for word, id in words_dict.items())
train_list_of_token_ids = convert_tokens_to_ids(
tokenized_sentences_train,
id_to_word,
embedding_word_dict,
args.sentences_length)
test_list_of_token_ids = convert_tokens_to_ids(
tokenized_sentences_test,
id_to_word,
embedding_word_dict,
args.sentences_length)
X_train = np.array(train_list_of_token_ids)
X_test = np.array(test_list_of_token_ids)
get_model_func = lambda: get_model(
embedding_matrix,
args.sentences_length,
args.dropout_rate,
args.recurrent_units,
args.dense_size)
print("Starting to train models...")
models = train_folds(X_train, y_train, args.fold_count, args.batch_size, get_model_func)
if not os.path.exists(args.result_path):
os.mkdir(args.result_path)
print("Predicting results...")
test_predicts_list = []
for fold_id, model in enumerate(models):
model_path = os.path.join(args.result_path, "model{0}_weights.npy".format(fold_id))
np.save(model_path, model.get_weights())
test_predicts_path = os.path.join(args.result_path, "test_predicts{0}.npy".format(fold_id))
test_predicts = model.predict(X_test, batch_size=args.batch_size)
test_predicts_list.append(test_predicts)
np.save(test_predicts_path, test_predicts)
test_predicts = np.ones(test_predicts_list[0].shape)
for fold_predict in test_predicts_list:
test_predicts *= fold_predict
test_predicts **= (1. / len(test_predicts_list))
test_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT
#test_ids = test_data["id"].values
#test_ids = test_ids.reshape((len(test_ids), 1))
y_pred = test_predicts.argmax(axis=-1)
#test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES)
#test_predicts["id"] = test_ids
#test_predicts = test_predicts[["id"] + CLASSES]
test_data['emotion_pred'] = y_pred
submit_path = os.path.join(args.result_path, "submit")
test_predicts.to_csv(submit_path, index=False)
def main():
parser = argparse.ArgumentParser(
description=
"Recurrent neural network for identifying and classifying toxic online comments"
)
parser.add_argument("train_file_path")
parser.add_argument("test_file_path")
parser.add_argument("embedding_path")
parser.add_argument("--result-path", default="toxic_results")
parser.add_argument("--batch-size", type=int, default=256)
parser.add_argument("--sentences-length", type=int, default=500)
parser.add_argument("--recurrent-units", type=int, default=64)
parser.add_argument("--dropout-rate", type=float, default=0.3)
parser.add_argument("--dense-size", type=int, default=32)
parser.add_argument("--fold-count", type=int, default=10)
# parser.add_argument("train_file_path")
# parser.add_argument("test_file_path")
# parser.add_argument("embedding_path")
# parser.add_argument("--result-path", default="toxic_results")
# parser.add_argument("--batch-size", type=int, default=32)
# parser.add_argument("--sentences-length", type=int, default=200)
# parser.add_argument("--recurrent-units", type=int, default=80)
# parser.add_argument("--dropout-rate", type=float, default=0.3)
# parser.add_argument("--dense-size", type=int, default=32)
# parser.add_argument("--fold-count", type=int, default=10)
args = parser.parse_args()
if args.fold_count <= 1:
raise ValueError("fold-count should be more than 1")
print("Loading data...")
train_data = pd.read_csv(args.train_file_path)
test_data = pd.read_csv(args.test_file_path)
list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values
list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values
y_train = train_data[CLASSES].values
print("Tokenizing sentences in train set...")
tokenized_sentences_train, words_dict = tokenize_sentences(
list_sentences_train, {})
print("Tokenizing sentences in test set...")
tokenized_sentences_test, words_dict = tokenize_sentences(
list_sentences_test, words_dict)
words_dict[UNKNOWN_WORD] = len(words_dict)
print("Loading embeddings...")
embedding_list, embedding_word_dict = read_embedding_list(
args.embedding_path)
embedding_size = len(embedding_list[0])
print("Preparing data...")
embedding_list, embedding_word_dict = clear_embedding_list(
embedding_list, embedding_word_dict, words_dict)
embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
embedding_list.append([0.] * embedding_size)
embedding_word_dict[END_WORD] = len(embedding_word_dict)
embedding_list.append([-1.] * embedding_size)
embedding_matrix = np.array(embedding_list)
id_to_word = dict((id, word) for word, id in words_dict.items())
train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train,
id_to_word,
embedding_word_dict,
args.sentences_length)
test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test,
id_to_word,
embedding_word_dict,
args.sentences_length)
X_train = np.array(train_list_of_token_ids)
X_test = np.array(test_list_of_token_ids)
get_model_func = lambda: get_model(embedding_matrix, args.sentences_length,
args.dropout_rate, args.recurrent_units,
args.dense_size)
import gc
del train_data, test_data, list_sentences_train, list_sentences_test
del tokenized_sentences_train, tokenized_sentences_test, words_dict
del embedding_list, embedding_word_dict
del train_list_of_token_ids, test_list_of_token_ids
gc.collect()
print("Starting to train models...")
models = train_folds(X_train, y_train, X_test, args.fold_count,
args.batch_size, get_model_func)
list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values
list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values
y_train = train_data[CLASSES].values
#marking comments without any tags as "clean"
rowsums = train_data.iloc[:, 2:8].sum(axis=1)
train_data['clean'] = (rowsums == 0)
#count number of clean entries
train_data['clean'].sum()
print("Total comments = ", len(train_data))
print("Total clean comments = ", train_data['clean'].sum())
print("Total tags =", rowsums.sum())
print("Tokenizing sentences in train set...")
tokenized_sentences_train, words_dict = tokenize_sentences(
list_sentences_train, {})
print("Tokenizing sentences in test set...")
tokenized_sentences_test, words_dict = tokenize_sentences(
list_sentences_test, words_dict)
print(embedding_path)
words_dict[UNKNOWN_WORD] = len(words_dict)
print("Loading embeddings...")
embedding_list, embedding_word_dict = read_embedding_list(
file_path=embedding_path)
embedding_size = len(embedding_list[0])
import numpy as np
def main():
parser = argparse.ArgumentParser(
description=
"Recurrent neural network for identifying and classifying toxic online comments"
)
parser.add_argument("train_file_path")
parser.add_argument("test_file_path")
parser.add_argument("embedding_path")
parser.add_argument("--result-path", default="toxic_results")
parser.add_argument("--batch-size", type=int, default=256)
parser.add_argument("--sentences-length", type=int, default=500)
parser.add_argument("--recurrent-units", type=int, default=64)
parser.add_argument("--dropout-rate", type=float, default=0.3)
parser.add_argument("--dense-size", type=int, default=32)
parser.add_argument("--fold-count", type=int, default=10)
args = parser.parse_args()
if args.fold_count <= 1:
raise ValueError("fold-count should be more than 1")
print("Loading data...")
train_data = pd.read_csv(args.train_file_path)
test_data = pd.read_csv(args.test_file_path)
# Identify language
#train_data['language'] = train_data['comment_text'].apply(detect_language)
#test_data['language'] = test_data['comment_text'].apply(detect_language)
# Translate the non-english to the english.
#train_data['comment_text'] = train_data.apply(lambda x: translate(x.comment_text, x.language),axis=1)
#test_data['comment_text'] = test_data.apply(lambda x: translate(x.comment_text, x.language),axis=1)
#train_data.to_csv("train_data_translated.csv")
#test_data.to_csv("test_data_translated.csv")
#train_data['comment_text'] = train_data.apply(lambda x: clean(x.comment_text), axis=1)
#train_data['comment_text'] = train_data.apply(lambda x: clean(x.comment_text), axis=1)
#train_data.to_csv("train_data_cleaned_after_translate.csv")
#test_data.to_csv("test_data_cleaned_after_translate.csv")
list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values
list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values
y_train = train_data[CLASSES].values
print("Tokenizing sentences in train set...")
tokenized_sentences_train, words_dict = tokenize_sentences(
list_sentences_train, {})
print("Tokenizing sentences in test set...")
tokenized_sentences_test, words_dict = tokenize_sentences(
list_sentences_test, words_dict)
words_dict[UNKNOWN_WORD] = len(words_dict)
print("Loading embeddings...")
embedding_list, embedding_word_dict = read_embedding_list(
args.embedding_path)
embedding_size = len(embedding_list[0])
print("Preparing data...")
embedding_list, embedding_word_dict = clear_embedding_list(
embedding_list, embedding_word_dict, words_dict)
embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
embedding_list.append([0.] * embedding_size)
embedding_word_dict[END_WORD] = len(embedding_word_dict)
embedding_list.append([-1.] * embedding_size)
embedding_matrix = np.array(embedding_list)
embedding_matrix_path = os.path.join(args.result_path,
"embedding_matrix.npy")
np.save(embedding_matrix_path, embedding_matrix)
words_dict_path = os.path.join(args.result_path, "words_dict.npy")
np.save(words_dict_path, words_dict)
id_to_word = dict((id, word) for word, id in words_dict.items())
train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train,
id_to_word,
embedding_word_dict,
args.sentences_length)
test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test,
id_to_word,
embedding_word_dict,
args.sentences_length)
X_train = np.array(train_list_of_token_ids)
X_test = np.array(test_list_of_token_ids)
print(embedding_matrix.shape)
print(embedding_matrix.shape[0])
print(embedding_matrix.shape[1])
get_model_func = lambda: get_model(embedding_matrix, args.sentences_length,
args.dropout_rate, args.recurrent_units,
args.dense_size)
print("Starting to train models...")
models = train_folds(X_train, y_train, args.fold_count, args.batch_size,
get_model_func)
if not os.path.exists(args.result_path):
os.mkdir(args.result_path)
print("Predicting results...")
test_predicts_list = []
for fold_id, model in enumerate(models):
model_path = os.path.join(args.result_path,
"model{0}_weights.npy".format(fold_id))
np.save(model_path, model.get_weights())
model.save_weights("model{0}_weights.h5".format(fold_id))
test_predicts_path = os.path.join(
args.result_path, "test_predicts{0}.npy".format(fold_id))
test_predicts = model.predict(X_test, batch_size=args.batch_size)
test_predicts_list.append(test_predicts)
np.save(test_predicts_path, test_predicts)
test_predicts = np.ones(test_predicts_list[0].shape)
for fold_predict in test_predicts_list:
test_predicts *= fold_predict
test_predicts **= (1. / len(test_predicts_list))
test_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT
test_ids = test_data["id"].values
test_ids = test_ids.reshape((len(test_ids), 1))
test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES)
test_predicts["id"] = test_ids
test_predicts = test_predicts[["id"] + CLASSES]
submit_path = os.path.join(args.result_path, "submit")
test_predicts.to_csv(submit_path, index=False)
print("Predicting Discussion posts...")
posts = pd.read_csv("posts_cleaned.csv")
posts = posts.dropna()
discussion_posts = posts['MSG_TEXT'].tolist()
tokenized_discussion_posts, words_dict = tokenize_sentences(
discussion_posts, words_dict)
#id_to_word = dict((id, word) for word, id in words_dict.items())
discussion_list_of_token_ids = convert_tokens_to_ids(
tokenized_discussion_posts, id_to_word, embedding_word_dict,
args.sentences_length)
X_test = np.array(discussion_list_of_token_ids)
discussion_predict_list = []
for fold_id, model in enumerate(models):
discussion_predicts = model.predict(X_test, batch_size=args.batch_size)
discussion_predict_list.append(discussion_predicts)
discussion_predicts = np.ones(discussion_predict_list[0].shape)
for fold_predict in discussion_predict_list:
discussion_predicts *= fold_predict
discussion_predicts **= (1. / len(discussion_predict_list))
discussion_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT
discussion_predicts = pd.DataFrame(data=discussion_predicts,
columns=CLASSES)
discussion_predicts['MSG_TEXT'] = discussion_posts
discussion_predicts = discussion_predicts[["MSG_TEXT"] + CLASSES]
discussion_predicts_path = os.path.join(args.result_path,
"discussion_predicts.csv")
discussion_predicts.to_csv(discussion_predicts_path, index=False)
def main():
parser = argparse.ArgumentParser(
description=
"Recurrent neural network for identifying and classifying toxic online comments"
)
parser.add_argument("train_file_path")
parser.add_argument("test_file_path")
parser.add_argument("embedding_path")
parser.add_argument("--result-path", default="toxic_results")
parser.add_argument("--batch-size", type=int, default=256)
parser.add_argument("--sentences-length", type=int, default=500)
parser.add_argument("--recurrent-units", type=int, default=64)
parser.add_argument("--dropout-rate", type=float, default=0.3)
parser.add_argument("--dense-size", type=int, default=32)
parser.add_argument("--fold-count", type=int, default=10)
parser.add_argument("--modelname-prefix", type=str, default="")
parser.add_argument("--cv", type=str, default="True")
parser.add_argument("--use-roc", type=str, default="False")
args = parser.parse_args()
if args.fold_count <= 1:
raise ValueError("fold-count should be more than 1")
print('Input params')
print(args)
start = time.time()
print('#' * 50)
print("Loading data...")
print('#' * 50)
if os.path.exists(
os.path.join(
args.result_path,
'tokenized_sentences_train.pkl')) and os.path.exists(
os.path.join(
args.result_path,
'tokenized_sentences_train.pkl')) and os.path.exists(
os.path.join(args.result_path,
'tokenized_sentences_train.pkl')):
print('Preprocessed files found. Reading preprocess files')
train_data = pd.read_csv(args.train_file_path)
test_data = pd.read_csv(args.test_file_path)
y_train = train_data[CLASSES].values
with open(
os.path.join(args.result_path,
'tokenized_sentences_train.pkl'), 'rb') as f:
tokenized_sentences_train = pickle.load(f)
with open(
os.path.join(args.result_path, 'tokenized_sentences_test.pkl'),
'rb') as f:
tokenized_sentences_test = pickle.load(f)
with open(os.path.join(args.result_path, 'words_dict.pkl'), 'rb') as f:
words_dict = pickle.load(f)
else:
print('Preprocessed files not found.')
train_data = pd.read_csv(args.train_file_path)
test_data = pd.read_csv(args.test_file_path)
list_sentences_train = train_data["comment_text"].fillna(
NAN_WORD).values
list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values
y_train = train_data[CLASSES].values
print('#' * 50)
print("Tokenizing sentences in train set...")
print('#' * 50)
tokenized_sentences_train, words_dict = tokenize_sentences(
list_sentences_train, {})
print('#' * 50)
print("Tokenizing sentences in test set...")
print('#' * 50)
tokenized_sentences_test, words_dict = tokenize_sentences(
list_sentences_test, words_dict)
print('Saving preprocess files...')
with open(
os.path.join(args.result_path,
'tokenized_sentences_train.pkl'), 'wb') as f:
pickle.dump(tokenized_sentences_train, f)
with open(
os.path.join(args.result_path, 'tokenized_sentences_test.pkl'),
'wb') as f:
pickle.dump(tokenized_sentences_test, f)
with open(os.path.join(args.result_path, 'words_dict.pkl'), 'wb') as f:
pickle.dump(words_dict, f)
print('total words', len(words_dict))
words_dict[UNKNOWN_WORD] = len(words_dict)
print('#' * 50)
print("Loading embeddings...")
print('#' * 50)
if 'glove' in args.embedding_path:
print('Reading Glove embedding')
embedding_list, embedding_word_dict = read_embedding_list_glove(
args.embedding_path)
else:
print('Reading Fasttext embedding')
embedding_list, embedding_word_dict = read_embedding_list(
args.embedding_path)
embedding_size = len(embedding_list[0])
print('Embedding size', embedding_size)
print('#' * 50)
print("Preparing data...")
print('#' * 50)
embedding_list, embedding_word_dict = clear_embedding_list(
embedding_list, embedding_word_dict, words_dict)
embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
embedding_list.append([0.] * embedding_size)
embedding_word_dict[END_WORD] = len(embedding_word_dict)
embedding_list.append([-1.] * embedding_size)
embedding_matrix = np.array(embedding_list)
print('Embedding matrix shape:', embedding_matrix.shape)
id_to_word = dict((id, word) for word, id in words_dict.items())
train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train,
id_to_word,
embedding_word_dict,
args.sentences_length)
test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test,
id_to_word,
embedding_word_dict,
args.sentences_length)
X_train = np.array(train_list_of_token_ids)
X_test = np.array(test_list_of_token_ids)
# GRU cross validation
# get_model_func = lambda: get_model(
# embedding_matrix,
# args.sentences_length,
# args.dropout_rate,
# args.recurrent_units,
# args.dense_size)
# GRU maxpool, avgpool
# get_model_func = lambda: get_model_pool(
# embedding_matrix,
# args.sentences_length,
# args.dropout_rate,
# args.recurrent_units,
# args.dense_size)
# deepmoji style
# get_model_func = lambda: get_model_deepmoji_style(
# embedding_matrix,
# args.sentences_length,
# args.dropout_rate,
# args.recurrent_units,
# args.dense_size)
# GRU maxpool, avgpool validation
# get_model_func = lambda: get_gru_model(
# embedding_matrix,
# args.sentences_length,
# args.dropout_rate,
# args.recurrent_units,
# args.dense_size)
# GRU maxpool, avgpool + cnn validation
# get_model_func = lambda: get_model_pool_gru_cnn(
# embedding_matrix,
# args.sentences_length,
# args.dropout_rate,
# args.recurrent_units,
# args.dense_size)
# dpcnn validation
get_model_func = lambda: get_dpcnn_model(
embedding_matrix, args.sentences_length, args.dropout_rate, args.
dense_size)
# lstm with attention cross val
# get_model_func = lambda: get_model_att_lstm(
# embedding_matrix,
# args.sentences_length,
# args.dropout_rate,
# args.recurrent_units,
# args.dense_size)
# capsule net
# get_model_func = lambda: get_capsnet_model(
# embedding_matrix,
# args.sentences_length,
# args.dropout_rate,
# args.recurrent_units,
# args.dense_size)
print('#' * 50)
print("Starting to train models...")
print('#' * 50)
models = train_folds(X_train, y_train, args.fold_count, args.batch_size,
get_model_func, args.cv, args.use_roc)
if not os.path.exists(args.result_path):
os.mkdir(args.result_path)
print('#' * 50)
print("Predicting results...")
print('#' * 50)
if args.cv == "True":
test_predicts_list = []
for fold_id, model in enumerate(models):
model_path = os.path.join(
args.result_path,
"{0}_model{1}_weights.npy".format(args.modelname_prefix,
fold_id))
np.save(model_path, model.get_weights())
test_predicts_path = os.path.join(
args.result_path,
"{0}_test_predicts{1}.npy".format(args.modelname_prefix,
fold_id))
test_predicts = model.predict(X_test,
batch_size=args.batch_size * 2)
test_predicts_list.append(test_predicts)
np.save(test_predicts_path, test_predicts)
test_predicts = np.ones(test_predicts_list[0].shape)
for fold_predict in test_predicts_list:
test_predicts *= fold_predict
test_predicts **= (1. / len(test_predicts_list))
# test_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT
test_ids = test_data["id"].values
test_ids = test_ids.reshape((len(test_ids), 1))
test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES)
test_predicts["id"] = test_ids
test_predicts = test_predicts[["id"] + CLASSES]
submit_path = os.path.join(args.result_path,
"{}_submit".format(args.modelname_prefix))
test_predicts.to_csv(submit_path, index=False)
print('#' * 50)
print('Prediction Completed...')
print('#' * 50)
total_time = time.time() - start
mins, sec = divmod(total_time, 60)
hrs, mins = divmod(mins, 60)
print('Total time taken : {:.0f}h {:.0f}m {:.0f}s'.format(
hrs, mins, sec))
else:
print('No Cross Validation')
test_predicts = models.predict(X_test, batch_size=args.batch_size * 2)
test_ids = test_data["id"].values
test_ids = test_ids.reshape((len(test_ids), 1))
test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES)
test_predicts["id"] = test_ids
test_predicts = test_predicts[["id"] + CLASSES]
submit_path = os.path.join(
args.result_path, "{}_submit_nocv".format(args.modelname_prefix))
test_predicts.to_csv(submit_path, index=False)
total_time = time.time() - start
mins, sec = divmod(total_time, 60)
hrs, mins = divmod(mins, 60)
print('Total time taken : {:.0f}h {:.0f}m {:.0f}s'.format(
hrs, mins, sec))
def main():
parser = argparse.ArgumentParser(
description=
"Recurrent neural network for identifying and classifying toxic online comments"
)
parser.add_argument("train_file_path")
parser.add_argument("test_file_path")
parser.add_argument("embedding_path")
parser.add_argument("--result-path", default="predict_results")
parser.add_argument("--batch-size", type=int, default=256)
parser.add_argument("--sentences-length", type=int, default=500)
parser.add_argument("--recurrent-units", type=int, default=64)
parser.add_argument("--dropout-rate", type=float, default=0.3)
parser.add_argument("--dense-size", type=int, default=32)
parser.add_argument("--fold-count", type=int, default=10)
args = parser.parse_args()
if args.fold_count <= 1:
raise ValueError("fold-count should be more than 1")
start = time.time()
print("Loading data...")
train_data = pd.read_csv(args.train_file_path)
test_data = pd.read_csv(args.test_file_path)
list_sentences_train = train_data["comment_text"].fillna(NAN_WORD).values
list_sentences_test = test_data["comment_text"].fillna(NAN_WORD).values
y_train = train_data[CLASSES].values
# print("list_sentences_test = ", list_sentences_test)
print("Tokenizing sentences in train set...")
tokenized_sentences_train, words_dict = tokenize_sentences(
list_sentences_train, {})
# print("words_dict before = ", words_dict)
print("Tokenizing sentences in test set...")
tokenized_sentences_test, words_dict = tokenize_sentences(
list_sentences_test, words_dict)
# print("words_dict after = ", words_dict)
words_dict[UNKNOWN_WORD] = len(words_dict)
print("Loading embeddings...")
embedding_list, embedding_word_dict = read_embedding_list(
args.embedding_path)
embedding_size = len(embedding_list[0])
"""
with open('embedding_list.txt', 'wb') as file:
pickle.dump(embedding_list, file)
with open('embedding_word_dict.txt', 'wb') as file:
pickle.dump(embedding_word_dict, file)
with open('embedding_list.txt', 'rb') as file:
embedding_list = pickle.load(file)
with open('embedding_word_dict.txt', 'rb') as file:
embedding_word_dict = pickle.load(file)
print(embedding_list)
print(embedding_word_dict)
"""
end = time.time()
print("Time elapsed: " + end - start)
print("Preparing data...")
embedding_list, embedding_word_dict = clear_embedding_list(
embedding_list, embedding_word_dict, words_dict)
embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
embedding_list.append([0.] * embedding_size)
embedding_word_dict[END_WORD] = len(embedding_word_dict)
embedding_list.append([-1.] * embedding_size)
embedding_matrix = np.array(embedding_list)
id_to_word = dict((id, word) for word, id in words_dict.items())
train_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_train,
id_to_word,
embedding_word_dict,
args.sentences_length)
test_list_of_token_ids = convert_tokens_to_ids(tokenized_sentences_test,
id_to_word,
embedding_word_dict,
args.sentences_length)
X_train = np.array(train_list_of_token_ids)
X_test = np.array(test_list_of_token_ids)
get_model_func = lambda: get_model(embedding_matrix, args.sentences_length,
args.dropout_rate, args.recurrent_units,
args.dense_size)
print("Starting to train models...")
models = train_folds(X_train, y_train, args.fold_count, args.batch_size,
get_model_func)
if not os.path.exists(args.result_path):
os.mkdir(args.result_path)
print("Predicting results...")
test_predicts_list = []
for fold_id, model in enumerate(models):
model_path = os.path.join(args.result_path,
"model{0}_weights.npy".format(fold_id))
np.save(model_path, model.get_weights())
test_predicts_path = os.path.join(
args.result_path, "test_predicts{0}.npy".format(fold_id))
test_predicts = model.predict(X_test, batch_size=args.batch_size)
test_predicts_list.append(test_predicts)
np.save(test_predicts_path, test_predicts)
test_predicts = np.ones(test_predicts_list[0].shape)
for fold_predict in test_predicts_list:
print("fold_predict = ", fold_predict)
test_predicts *= fold_predict
test_predicts **= (1. / len(test_predicts_list))
test_predicts **= PROBABILITIES_NORMALIZE_COEFFICIENT
test_ids = test_data["id"].values
test_ids = test_ids.reshape((len(test_ids), 1))
print("test_predicts = ", test_predicts)
test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES)
test_predicts["id"] = test_ids
test_predicts = test_predicts[["id"] + CLASSES]
# Route results to a CSV file: "finished.csv" with path "predict_results/finished"
submit_path = os.path.join(args.result_path, "finished")
test_predicts.to_csv(submit_path, index=False)