def get_scores(sent_list):
l = len(sent_list)
X = []
total_n_grams = ws.get_total_ngrams(sent_list)
for sent in sent_list:
sent_scores = []
words_list = dissect.remove_stopwords(sent)
word_freq_score, tf_df_score = ws.word_frequency_and_tf_df(
l, words_list, sent_list)
sent_scores.append(word_freq_score)
sent_scores.append(tf_df_score)
sent_scores.append(ws.upper_case(words_list))
sent_scores.append(ws.proper_noun(words_list))
sent_scores.append(ws.lexical_similarity(words_list))
sent_scores.append(ws.n_gram(words_list, total_n_grams))
#Normalization
sent_scores_sum = sum(sent_scores)
sent_scores = [ele / sent_scores_sum for ele in sent_scores]
X.append(sent_scores)
#print(X, "\n")
return X
def word_frequency_and_tf_df(l, words_list, sentences_list):
"""
computes word_frequency and tf_idf of each word w.r.t to the entire words in the corpus(sentences_list)
:param l : total number of sentences in the entire sentences_list
:param words_list : list of all the words in the current sentence
:param sentences_list : total sentences in the entire corpus
:return word_freq_score : score obtained from word_frequency
:return tf_df_score : score obtained from tf-idf
"""
tf_df_score = 0
word_freq_score = 0
df = 0
for word in words_list:
tf = get_word_frequency(words_list, word)
for sentence in sentences_list:
sent_words_list = dissect.remove_stopwords(sentence)
# document frequency
df = df + get_word_frequency(sent_words_list, word)
# tf_df score
temp1 = log(1 + tf) / log(1 + df)
# word_frequency score
temp2 = (1 + tf) / (1 + df)
# add the scores of each word to get sentence score
tf_df_score = tf_df_score + (l * temp1)
word_freq_score = word_freq_score + temp2
return word_freq_score, tf_df_score
def get_total_ngrams(sent_list):
total_words_list = []
for sent in sent_list:
total_words_list.append(dissect.remove_stopwords(sent))
single_total_words_list = list(
itertools.chain.from_iterable(total_words_list))
return get_n_grams(single_total_words_list)
def get_total_ngrams(sent_list):
"""
finds n_grams of all the words in the corpus(total sentences)
:param sent_list : total sentences list of all the documents
:return : n_grams for the sent_list(by calling get_n_grams)
"""
total_words_list = []
for sent in sent_list:
total_words_list.append(dissect.remove_stopwords(sent))
single_total_words_list = list(
itertools.chain.from_iterable(total_words_list))
return get_n_grams(single_total_words_list)
def word_frequency_and_tf_df(l, words_list, sentences_list):
tf_df_score = 0
word_freq_score = 0
df = 0
for word in words_list:
tf = get_word_frequency(words_list, word)
for sentence in sentences_list:
sent_words_list = dissect.remove_stopwords(sentence)
df = df + get_word_frequency(sent_words_list, word)
temp1 = log(1 + tf) / log(1 + df)
temp2 = (1 + tf) / (1 + df)
tf_df_score = tf_df_score + (l * temp1)
word_freq_score = word_freq_score + (l * temp2)
return word_freq_score, tf_df_score
def get_scores(sent_list):
"""
computes scores for each sentence in the sentence_list by extracting scores from each word-scoring algo
:param sent_list : complete sentences list
:return : list of scores list(normalized list for each sentence) of all sentences
"""
l = len(sent_list)
X = []
# Compute total n_grams in the entire corpus(sentences list)
total_n_grams = ws.get_total_ngrams(sent_list)
for sent in sent_list:
sent_scores = []
words_list = dissect.remove_stopwords(sent)
# Perform each of the specified word-scoring algotithm and append all the scores to get feature vector
word_freq_score, tf_df_score = ws.word_frequency_and_tf_df(
l, words_list, sent_list)
sent_scores.append(word_freq_score)
sent_scores.append(tf_df_score)
sent_scores.append(ws.upper_case(words_list))
sent_scores.append(ws.proper_noun(words_list))
#sent_scores.append(ws.lexical_similarity(words_list))
sent_scores.append(ws.n_gram(words_list, total_n_grams))
#Normalize the scores
sent_scores_sum = sum(sent_scores)
sent_scores = [ele / sent_scores_sum for ele in sent_scores]
#feature vector for linear regression
X.append(sent_scores)
#print(X, "\n")
return X