def gather_sentiment_data_set():
"""
labelled_data.to_csv('all_labelled_data_before_processing.csv', encoding="utf-8", index=False)
unlabelled_data.to_csv('all_unlabelled_data_before_processing.csv', encoding="utf-8", index=False)
:return: labelled_data, unlabelled_data
"""
mc.move_to_main_location()
path_to_train_pos = os.path.abspath(os.path.join(os.path.curdir, "SentimentDataSet", "train", "pos"))
path_to_train_neg = os.path.abspath(os.path.join(os.path.curdir, "SentimentDataSet", "train", "neg"))
path_to_test_pos = os.path.abspath(os.path.join(os.path.curdir, "SentimentDataSet", "test", "pos"))
path_to_test_neg = os.path.abspath(os.path.join(os.path.curdir, "SentimentDataSet", "test", "neg"))
train_pos_df = review2pddataframe(path_to_train_pos, 1)
train_neg_df = review2pddataframe(path_to_train_neg, 0)
test_pos_df = review2pddataframe(path_to_test_pos, 1)
test_neg_df = review2pddataframe(path_to_test_neg, 0)
labelled_data = [test_neg_df, train_neg_df, test_pos_df, train_pos_df]
labelled_data = pd.concat(labelled_data)
labelled_data = labelled_data.sample(frac=1).reset_index(drop=True)
path_to_unlabelled_data = os.path.abspath(os.path.join(os.path.curdir, "SentimentDataSet", "train", "unsup"))
unlabelled_data = review2pddataframe(path_to_unlabelled_data, -1)
unlabelled_data = unlabelled_data.sample(frac=1).reset_index(drop=True)
mc.move_to_data_location()
labelled_data.to_csv('all_labelled_data_before_processing.csv', encoding="utf-8", index=False)
unlabelled_data.to_csv('all_unlabelled_data_before_processing.csv', encoding="utf-8", index=False)
return labelled_data, unlabelled_data
def preprocessing_reviews_in_df_to_words(data_before_processing, filename):
"""
mc.move_to_data_location()
write_list_of_lists_to_csv(processed_reviews, filename)
:param data_before_processing:
:param filename:
:return: processed_reviews
"""
reviews = data_before_processing['review'].as_matrix().tolist()
clean = lambda x: re.sub('[^a-zA-Z1-9\s]', '', x)
processed_reviews = [(''.join(list(map(clean, reviews[i]))).lower()).split() for i in range(len(reviews))]
mc.move_to_data_location()
write_list_of_lists_to_csv(processed_reviews, filename)
return processed_reviews
def preprocessing_reviews_in_df_to_sentences(data_before_processing, filename):
"""
write_list_of_lists_to_csv(sentences, filename)
:param data_before_processing:
:param filename:
:return: sentences (python list)
"""
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
reviews = data_before_processing['review'].as_matrix().tolist()
reviews_sentences =[tokenizer.tokenize(review.strip()) for review in reviews]
clean = lambda x: re.sub('[^a-zA-Z1-9\s]', '', x)
lower_split = lambda x: x.lower().split()
clean_sentences = [list(map(clean, reviews_sentences[i])) for i in range(len(reviews_sentences))]
to_low_split_sentences = [list(map(lower_split, clean_sentences[i])) for i in range(len(clean_sentences))]
sentences = [sentence for review in to_low_split_sentences for sentence in review]
mc.move_to_data_location()
write_list_of_lists_to_csv(sentences, filename)
return sentences
def preprocessing_reviews_in_df(set_df, max_seq_length, filename):
"""
mc.move_to_data_location()
set_df.to_csv(filename + '.csv', index=False)
:param set_df:
:param max_seq_length:
:param filename:
:return: set_df
"""
# BUG when reading from csv -- quotes
for i in tnrange(1, desc='processing'):
set_df = set_df[['review', 'sentiment']]
set_df.info()
set_df['review'] = set_df['review'].apply(lambda x: re.sub('[^a-zA-Z1-9\s]', '', x))
set_df['review'] = set_df['review'].apply(lambda x: x.lower().split()[:max_seq_length])
set_df['sentiment'] = set_df['sentiment'].apply(lambda x: [1, 0] if x == 1 else [0, 1])
mc.move_to_data_location()
set_df.to_(filename + '.csv', index=False)
return set_df
def convert_pd_words_reviews_to_np_ids_matrix(reviews_df, maxSeqLength,
wordsList, set_name):
"""
mc.move_to_data_location()
np.save(set_name + '_ids_matrix', ids_matrix)
np.save(set_name + '_sentiment_labels', sentiment_set)
:param reviews_df:
:param maxSeqLength:
:param wordsList:
:param set_name:
:return: ids_matrix, sentiment_set
"""
number_of_rev = len(reviews_df)
id_of_unknown = len(wordsList)
ids_matrix = np.full((number_of_rev, maxSeqLength),
id_of_unknown,
dtype='int32')
rev_set = reviews_df['review'].as_matrix()
sentiment_set = reviews_df['sentiment'].as_matrix()
rev_counter = 0
word_counter = 0
for i in tnrange(len(rev_set), desc='processing review'):
for word in rev_set[i]:
try:
ids_matrix[rev_counter][word_counter] = wordsList.index(word)
except ValueError:
ids_matrix[rev_counter][word_counter] = id_of_unknown
word_counter += 1
rev_counter += 1
word_counter = 0
mc.move_to_data_location()
np.save(set_name + '_ids_matrix', ids_matrix)
np.save(set_name + '_sentiment_labels', sentiment_set)
mc.move_to_main_location()
return ids_matrix, sentiment_set
batch_predictions = sess.run(predictions, {
X: x_test_batch,
dropout_keep_prob: 1.0
})
all_predictions = np.concatenate(
[all_predictions, batch_predictions])
counter += 1
# Print accuracy if y_test is defined
correct_predictions = float(sum(all_predictions == y_test))
print("Total number of test examples: {}".format(len(y_test)))
print('Accuracy: {:g}'.format(correct_predictions /
float(len(y_test))))
# Save the evaluation to a csv
mc.move_to_data_location()
# TODO: change when learn new model and prepere new data
test_set = pd.read_csv('test_processed_set.csv')
x_raw = test_set['review']
predictions_human_readable = np.column_stack(
(np.array(x_raw), all_predictions))
# without dots
out_path = os.path.join(FLAGS.checkpoint_dir, "prediction.csv")
print("Saving evaluation to {0}".format(out_path))
with open(out_path, 'w') as f:
csv.writer(f).writerows(predictions_human_readable)
acc_path = os.path.join(FLAGS.checkpoint_dir, "accuracy.csv")
print("Saving accurancy to {0}".format(acc_path))
with open(acc_path, 'w') as f:
f.write('{}'.format(correct_predictions / float(len(y_test))))
def load_ids_matrix_and_sentiment(x_filename, y_filename):
mc.move_to_data_location()
X = np.load(x_filename)
Y = np.load(y_filename)
Y = np.array(Y.tolist())
return X, Y