def getSentics(self, word):
senticsAndItensity = []
sn = SenticNet('en')
try:
sentics = sn.sentics(word)
polarity_intensity = sn.polarity_intense(word)
# print(sentics)
# print(sentics['pleasantness'])
# print(sentics['attention'])
# print(sentics['sensitivity'])
# print(sentics['aptitude'])
# print(polarity_intensity)
senticsAndItensity.append(float(sentics['pleasantness']))
senticsAndItensity.append(float(sentics['attention']))
senticsAndItensity.append(float(sentics['sensitivity']))
senticsAndItensity.append(float(sentics['aptitude']))
senticsAndItensity.append(float(polarity_intensity))
return senticsAndItensity
except Exception as e:
defaultsentics = [0.0, 0.0, 0.0, 0.0, 0.0]
return defaultsentics
# ##TESTING AREA
# yas = SenticValuer()
# print(yas.getSentics("awkward"))
def senticnet(text):
"""
Returns a list obtained from SenticNet with the following four features normalized: [pleasantness_value, attention_value, sensiivity_value, aptitude_value]
:param text: input text pre-processed by Spacy
:return: a list with the SenticNet features averaged for all the words in text
"""
list_features = [0] * 4
sn = SenticNet()
count_words = 0
for token in text:
try:
concept_info = sn.concept(token)
list_features[0] += float(concept_info['sentics']['pleasantness'])
list_features[1] += float(concept_info['sentics']['attention'])
list_features[2] += float(concept_info['sentics']['sensitivity'])
list_features[3] += float(concept_info['sentics']['aptitude'])
count_words += 1
except KeyError:
pass
if count_words != 0:
list_features = [feature / count_words for feature in list_features]
return list_features
def getMaxSum_senti(text):
wnl = WordNetLemmatizer()
sn = SenticNet()
sentences = nltk.sent_tokenize(text)
text_sentiAvg = 0
sentence_maxSenti = 0
for index in range(len(sentences)):
sentence = sentences[index].strip()
sentence = sentence[0:-1]
assert '.' not in sentence
words = nltk.word_tokenize(sentence.lower())
pos_tags = nltk.pos_tag(words)
sentence_sentiSum = getSentenceSentiSum(pos_tags, wnl, sn)
# print sentence_sentiSum,
if sentence_sentiSum > sentence_maxSenti:
sentence_maxSenti = sentence_sentiSum
text_sentiAvg += sentence_sentiSum
text_sentiAvg = text_sentiAvg / len(sentences)
text_sentiAvg = round(text_sentiAvg, 6)
sentence_maxSenti = round(sentence_maxSenti, 6)
return text_sentiAvg, sentence_maxSenti
def doc_sentiment(data):
"""
"""
#Call SenticNet module
sn = SenticNet()
#Create positive and negative variables
total_sentiment = 0
#Calculate sentiment for all words in document
for i in range(len(data)):
#If words don't exist in SenticNet vocabulary they will return an error
#We treat these words as if they have a sentiment of 0
try:
#Calculate sentiment of word
sentiment = sn.polarity_value(data[i])
#Update total sentiment
total_sentiment += float(sentiment)
except:
None
try:
#If total sentiment = 0 division errors will occur
#Calculate average sentiment for the document
avg_sentiment = total_sentiment/len(data)
except:
avg_sentiment = 0
if avg_sentiment >= 0:
output = 1
else:
output = 0
return output
def sem(d):
try:
sn = SenticNet()
sn.semantics(d)
return True
except KeyError:
return False
def Terms_Chooser(data, n_of_words, polarity_threshold):
sn = SenticNet()
#choosing words
data["Content"] = data["Content"].apply(lambda row: nltk.word_tokenize(row))
lista = np.array(data["Content"].values.tolist())
lista = list(itertools.chain.from_iterable(lista))
FD = FreqDist(lista)
MC = FD.most_common(n_of_words)
common_words = []
for i in range (0,n_of_words):
common_words.append(MC[i][0])
polarity = list()
words = list()
for x in common_words:
try:
temp = sn.polarity_intense(x)
if (float(temp) > polarity_threshold or float(temp) < -(polarity_threshold)):
polarity.append(temp)
words.append(x)
except Exception:
continue
return words
def fun1(d):
try:
from senticnet.senticnet import SenticNet
sn = SenticNet()
sn.semantics(d)
return True
except KeyError as error:
return False
def get_emotions(tokens):
from senticnet.senticnet import SenticNet
result = {}
sn = SenticNet()
for token in tokens:
moodtags = ""
if token in sn.data:
moodtags = sn.moodtags(token)
print(token, moodtags)
#TODO
return result
def __init__(self):
print("Start SenticNet - Sentiment Analysis")
self.sp = Support()
self.sn = SenticNet()
self.corpus = self.sp.import_corpus_bank()
self.terminology = self.sp.import_bank_terminology(filename='bank_terminology')
self.data, self.label = self.sp.process_data(filename='bank_message',
size_msg=3,
clean=True,
replace_text=True,
stemmed=None,
lemmatize=None,
spelling=None)
def __init__(self, *args, **kwargs):
# loading necessaries
self.nlp = spacy.load("en_core_web_sm")
self.senti = SenticNet()
self.sid = SentimentIntensityAnalyzer()
#loading dataset
self.df = pd.read_json("./Sarcasm_Headlines_Dataset.json", lines=True)
self.df = self.df[:15000]
self.df.drop(columns="article_link",
inplace=True) #dropping unnessary attribute
#storing nlp data in headlines variable
self.headlines = []
self.uni_gram = set()
self.uni_feature = []
self.y_ = []
for i in self.df['headline']:
self.headlines.append(self.nlp(i))
def sentiment_avg(self, text):
sn = SenticNet('pt')
list_polarity = []
qtd_words = len(text)
temp = text.split()
avg_n = 0
for i in range(len(temp)):
try:
polarity_value = sn.polarity_value(
self.treatment_string(temp[i]))
list_polarity.append(polarity_value)
except:
qtd_words -= 1
i += 1
avg_n = self.avg(list_polarity, qtd_words)
if avg_n > 0.003 or avg_n < -0.003:
return True
else:
return False
def __init__(self, vocab):
self.vocab = vocab
self.X_width = len(vocab)
self.sn = SenticNet()
def get_clues(text):
text = text
print("*--------(%s)-------------*" % (text))
print(type(text))
nlp = StanfordCoreNLP('http://localhost:9001')
stop_words = set(stopwords.words('english'))
'''
Method to remove numbers appended at last
'''
dep_parse = nlp.annotate(text,
properties={
'annotators': 'depparse',
'outputFormat': 'json',
'timeout': 10000,
})
pos = nlp.annotate(text,
properties={
'annotators': 'lemma',
'outputFormat': 'json',
'timeout': 10000,
})
sn = SenticNet()
word_to_dep = [{} for i in range(len(dep_parse['sentences']))]
word_to_par = [{} for i in range(len(dep_parse['sentences']))]
word_to_pos = [{} for i in range(len(dep_parse['sentences']))]
word_to_lemma = [{} for i in range(len(dep_parse['sentences']))]
word_to_child = [{} for i in range(len(dep_parse['sentences']))]
sents = [[] for i in range(len(dep_parse['sentences']))]
index_to_word = {}
'''
Constructing dicts for maintaining the dependencies among words.
'''
'''
Appending each word by occurence number to maintain distinct word count
'''
#print(dep_parse['sentences'])
print("********")
for i, sent in enumerate(dep_parse['sentences']):
for dep in sent['basicDependencies']:
word_to_dep[i][dep['dependentGloss'] +
str(dep['dependent'])] = dep['dep']
word_to_par[i][dep['dependentGloss'] +
str(dep['dependent'])] = dep['governorGloss'] + str(
dep['governor'])
index_to_word[dep['dependentGloss'] +
str(dep['dependent'])] = dep['dependentGloss']
if (dep['governorGloss'] + str(dep['governor'])
not in word_to_child[i]):
word_to_child[i][dep['governorGloss'] +
str(dep['governor'])] = []
if (dep['dependentGloss'] + str(dep['dependent'])
not in word_to_child[i]):
word_to_child[i][dep['dependentGloss'] +
str(dep['dependent'])] = []
word_to_child[i][dep['governorGloss'] +
str(dep['governor'])].append(
dep['dependentGloss'] + str(dep['dependent']))
sents[i].append(dep['dependentGloss'] + str(dep['dependent']))
word_to_dep[i]['ROOT0'] = 'root'
word_to_par[i]['ROOT0'] = 'root'
for i, sent in enumerate(pos['sentences']):
for pos_tagger in sent['tokens']:
word_to_pos[i][pos_tagger['word']] = pos_tagger['pos']
word_to_lemma[i][pos_tagger['word']] = pos_tagger['lemma']
word_to_pos[i]['ROOT'] = 'root'
word_to_lemma[i]['ROOT'] = 'root'
'''
Displaying the deps
'''
##Implemeting rules to extract aspects
for i, sent in enumerate(sents):
if (__name__ == '__main__'):
print(word_to_dep[i], word_to_par[i], word_to_pos[i])
print("Children==>")
print(word_to_child[i])
aspects = []
for i, sent in enumerate(sents):
for word in sent:
'''
Rule 0
'''
if ('subj' in word_to_dep[i][word]):
for child in word_to_child[i][word_to_par[i][word]]:
if ('amod' in word_to_dep[i][child]
or 'advmod' in word_to_dep[i][child]):
aspects.append(word_to_par[i][word])
if (__name__ == '__main__'):
print("Rule 0 triggered.")
'''
Rule 1 (without sub): Very big to hold.
'''
if (word_to_dep[i][word] == 'xcomp' and
('JJ' in word_to_pos[i][index_to_word[word_to_par[i][word]]] or
'RB' in word_to_pos[i][index_to_word[word_to_par[i][word]]])):
if (__name__ == '__main__'):
print("Rule 1 triggered")
aspects.append(word_to_par[i][word])
'''
Rule 2 (without subj): Not to mention the price of the phone
'''
if (word_to_dep[i][word] == 'dobj' and 'VB'
in word_to_pos[i][index_to_word[(word_to_par[i][word])]]
and ('NN' in word_to_pos[i][index_to_word[(word)]]
or 'JJ' in word_to_pos[i][index_to_word[(word)]])):
aspects.append(word)
if (__name__ == '__main__'):
print("Rule 2 triggered")
print(word)
'''
Rule 3 (without subj): Love the sleekness of the player
'''
if ('NN' in word_to_pos[i][index_to_word[(word)]]
and word_to_dep[i][word] == 'nmod'):
aspects.append(word_to_par[i][word])
if (__name__ == '__main__'):
print("Rule 3 triggered")
print(word_to_par[i][word])
'''
Rule 4 (with sub): The battery lasts little
two aspects
'''
if (word_to_dep[i][word] == 'advmod'
or word_to_dep[i][word] == 'amod' or word_to_dep[i][word]
== 'advcl') and ('VB' in word_to_pos[i][index_to_word[(
word_to_par[i][word])]]):
aspects.append(word_to_par[i][word])
for word2 in sent:
if (word2 != word and word_to_dep[i][word2] == 'nsubj'
and word_to_par[i][word2] == word_to_par[i][word]
and
('NN' in word_to_pos[i][index_to_word[word2]]
or 'JJ' in word_to_pos[i][index_to_word[word2]])):
aspects.append(word2)
if (__name__ == '__main__'):
print("Rule 4 triggered")
print(word2)
'''
Rule 5 (with sub): I like the lens of this camera
'''
if ('NN' in word_to_pos[i][index_to_word[(word)]]
and word_to_dep[i][word] == 'dobj'):
if (__name__ == '__main__'):
print("Rule 5 triggered")
print(word)
try:
concept_info = sn.concept((word))
print("present in senticnet")
except KeyError:
print("Yay")
aspects.append(word)
'''
Rule 6 : I like the beauty of the screen.
Check if senticnet condition should be added
'''
if ('NN' in word_to_pos[i][index_to_word[(word)]]
and word_to_dep[i][word] == 'dobj'):
try:
concept_info = sn.concept((word))
aspects.append(word)
print("yay!")
except KeyError:
print("oops, not there in SenticNet")
for word2 in sent:
if (word2 != word and word_to_par[i][word2] == word and
'NN' in word_to_pos[i][index_to_word[(word2)]]):
aspects.append(word2)
if (__name__ == '__main__'):
print("Rule 6 triggered.")
print(word2)
'''
Rule 7 : I would like to comment on the camera of this phone.
'''
if (word_to_dep[i][word] == 'xcomp'):
try:
concept_info = sn.concept((word))
aspects.append(word)
print("yay!")
except KeyError:
print("oops, not there in SenticNet")
for child in word_to_child[i][word]:
if ('NN' in word_to_pos[i][index_to_word[child]]):
aspects.append(child)
if (__name__ == '__main__'):
print("Rule 7 triggered.")
print(word)
print(child)
'''
Rule 8 : The car is expensive.
'''
if (word_to_dep[i][word] == 'nsubj'):
for word2 in sent:
if (word2 != word and word_to_dep[i][word2] == 'cop'
and word_to_par[i][word2] == word_to_par[i][word]):
aspects.append(word_to_par[i][word])
if (__name__ == '__main__'):
print("Rule 8 triggered")
print(word_to_par[i][word])
'''
Rule 9 : The camera is nice.
'''
if (word_to_dep[i][word] == 'nsubj'
and 'NN' in word_to_pos[i][index_to_word[(word)]]):
for word2 in sent:
if (word2 != word and word_to_dep[i][word2] == 'cop'
and word_to_par[i][word2] == word_to_par[i][word]):
aspects.append(word)
if (__name__ == '__main__'):
print("Rule 9 triggered")
print(word)
'''
Rule 10 : The phone is very lightweight to carry.
'''
if (word_to_dep[i][word] == 'cop'):
for word2 in sent:
if (word2 != word
and 'VB' in word_to_pos[i][index_to_word[(word2)]]
and word_to_par[i][word] == word_to_par[i][word2]):
aspects.append(word2)
if (__name__ == '__main__'):
print("Rule 10 triggered.")
print(word2)
'''
Extracting mods of dobjs
'''
if (word_to_dep[i][word] == 'dobj'):
for child in word_to_child[i][word]:
if ('mod' in word_to_dep[i][child] and 'JJ'
in word_to_pos[i][index_to_word[(child)]]):
aspects.append(child)
'''
Rule 11 : Checking for conjuctions
'''
for asp in aspects:
for word in sent:
if (word_to_dep[i][word] == 'conj'
and word_to_par[i][word] == asp):
aspects.append(word)
if (__name__ == '__main__'):
print("Rule conj triggered.")
print(word)
finalIAC = set(aspects)
finalIAC = [index_to_word[f] for f in finalIAC]
finalIAC = [w for w in finalIAC if not w in stop_words]
finalSenti = []
for iac in finalIAC:
try:
concept_info = sn.concept((iac))
finalSenti.append(iac)
except KeyError:
print("No word available for " + iac)
return finalIAC, finalSenti
# Классификация текстов по 5 шкалам согласно базе SenticNet, оценка XGBoost
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from senticnet.senticnet import SenticNet
from nltk.stem import WordNetLemmatizer
from sklearn import metrics
import xgboost
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
df = pd.read_csv('preprocessed.csv')
lemmatizer = WordNetLemmatizer() # преобразование слов в исходную форму
sn = SenticNet() # база данных, содержащая классификацию слов и выражений по значению и настроению
# Функция, возвращающая оценки введенного текста по 5 шкалам базы SenticNet: polarity intensity,
# pleasantness, attention, sensitivity и aptitude. Оценка формируется как сумма оценок
# всех включенных в базу слов и выражений из текста и нормируется на кол-во слов в тексте
def SN(data):
# Преобразование текста в вектор, формирование словаря слов и словосочетаний длинной до 3 слов включительно
vectorizer = CountVectorizer(analyzer='word', ngram_range=(1,3))
vec = vectorizer.fit_transform([data]).todense()
k = 0
polarity_intense = sentics_pleasant = sentics_attention = sentics_sense = sentics_aptitude = 0
for i in vectorizer.vocabulary_.keys():
try: # Попытка найти i-ое слово/выражение в базе
num_repetitions = vec[0, vectorizer.vocabulary_[i]]
polarity_intense += (float(sn.polarity_intense(i)) * num_repetitions)
sentics_pleasant += (float(sn.sentics(i)['pleasantness']) * num_repetitions)
def __init__(self):
self.sn = SenticNet()
def get_clues(text):
text = text
print("*--------(%s)-------------*" % (text))
print(type(text))
nlp = StanfordCoreNLP('http://localhost:9001')
stop_words = set(stopwords.words('english'))
'''
Method to remove numbers appended at last
'''
dep_parse = nlp.annotate(text,
properties={
'annotators': 'depparse',
'outputFormat': 'json',
'timeout': 10000,
})
pos = nlp.annotate(text,
properties={
'annotators': 'lemma',
'outputFormat': 'json',
'timeout': 10000,
})
sn = SenticNet()
word_to_dep = [{} for i in range(len(dep_parse['sentences']))]
word_to_par = [{} for i in range(len(dep_parse['sentences']))]
word_to_pos = [{} for i in range(len(dep_parse['sentences']))]
word_to_lemma = [{} for i in range(len(dep_parse['sentences']))]
word_to_child = [{} for i in range(len(dep_parse['sentences']))]
sents = [[] for i in range(len(dep_parse['sentences']))]
index_to_word = {}
aspect_result = [[] for i in range(len(dep_parse['sentences']))]
'''
Constructing dicts for maintaining the dependencies among words.
'''
'''
Appending each word by occurence number to maintain distinct word count
'''
print("********")
for i, sent in enumerate(dep_parse['sentences']):
for dep in sent['basicDependencies']:
word_to_dep[i][dep['dependentGloss'] +
str(dep['dependent'])] = dep['dep']
word_to_par[i][dep['dependentGloss'] +
str(dep['dependent'])] = dep['governorGloss'] + str(
dep['governor'])
index_to_word[dep['dependentGloss'] +
str(dep['dependent'])] = dep['dependentGloss']
if (dep['governorGloss'] + str(dep['governor'])
not in word_to_child[i]):
word_to_child[i][dep['governorGloss'] +
str(dep['governor'])] = []
if (dep['dependentGloss'] + str(dep['dependent'])
not in word_to_child[i]):
word_to_child[i][dep['dependentGloss'] +
str(dep['dependent'])] = []
word_to_child[i][dep['governorGloss'] +
str(dep['governor'])].append(
dep['dependentGloss'] + str(dep['dependent']))
sents[i].append(dep['dependentGloss'] + str(dep['dependent']))
word_to_dep[i]['ROOT0'] = 'root'
word_to_par[i]['ROOT0'] = 'root'
for i, sent in enumerate(pos['sentences']):
for pos_tagger in sent['tokens']:
word_to_pos[i][pos_tagger['word']] = pos_tagger['pos']
word_to_lemma[i][pos_tagger['word']] = pos_tagger['lemma']
word_to_pos[i]['ROOT'] = 'root'
word_to_lemma[i]['ROOT'] = 'root'
'''
Displaying the deps
'''
##Implemeting rules to extract aspects
for i, sent in enumerate(sents):
if (__name__ == '__main__'):
print(word_to_dep[i], word_to_par[i], word_to_pos[i],
word_to_lemma[i])
print("Children==>")
print(word_to_child[i])
for i, sent in enumerate(sents):
token_t = word_to_child[i]['ROOT0'][0]
is_sub = False
token_h = None
for child in word_to_child[i][token_t]:
if 'subj' in word_to_dep[i][child]:
is_sub = True
token_h = child
#If subject noun relationship present
if is_sub:
"""
Rule 0: if any adv or adj modifies the token t.
"""
for child in word_to_child[i][token_t]:
if ('amod' in word_to_dep[i][child]
or 'advmod' in word_to_dep[i][child]):
try:
concept_info = sn.concept(index_to_word[child])
aspect_result[i].append(token_t)
if __name__ == '__main__':
print("Rule 1 triggered.")
print("present in senticnet")
except KeyError:
print("OOps")
"""
Rule 1: The battery lasts little.
"""
for child in word_to_child[i][token_t]:
if (word_to_dep[i][child] == 'advmod' or word_to_dep[i][child]
== 'amod' or word_to_dep[i][child] == 'advcl') and (
'VB' in word_to_pos[i][index_to_word[token_t]]):
aspect_result[i].append(token_t)
aspect_result[i].append(token_h)
if __name__ == '__main__':
print("Rule 1 triggered.")
print(token_t)
print(token_h)
"""
Rule 2: I like the beauty of the screen (and I like the lens of this camera).
"""
for child in word_to_child[i][token_t]:
if (word_to_dep[i][child] == 'dobj'
and 'NN' in word_to_pos[i][index_to_word[child]]):
aspect_result[i].append(child)
if __name__ == '__main__':
print(child)
try:
concept_info = sn.concept(index_to_word[child])
if __name__ == '__main__':
print("Rule 2 triggered")
for grandchild in word_to_child[i][child]:
if ('NN' in word_to_pos[i][
index_to_word[grandchild]]):
aspect_result[i].append(grandchild)
print(grandchild)
except KeyError:
print("OOps")
"""
Rule 3: I would like to comment on the camera of this phone.
"""
for child in word_to_child[i][token_t]:
if (word_to_dep[i][child] == 'xcomp'):
try:
sn.concept(index_to_word[child])
aspect_result[i].append(child)
if __name__ == '__main__':
print(child)
except KeyError:
print("OOps")
for grandchild in word_to_child[i][child]:
if ('NN' in word_to_pos[i][index_to_word[grandchild]]):
aspect_result[i].append(grandchild)
if __name__ == '__main__':
print(grandchild)
print("Rule 3 triggered.")
"""
Rule 4: The car is expensive.
"""
for child in word_to_child[i][token_t]:
if (word_to_dep[i][child] == 'cop'):
try:
sn.concept(word_to_lemma[i][index_to_word[token_t]])
aspect_result[i].append(token_t)
if __name__ == '__main__':
print("Rule 4 triggered")
print(token_t)
except KeyError:
pass
"""
Rule 5: The camera is nice
"""
for child in word_to_child[i][token_t]:
if (word_to_dep[i][child] == 'cop'
and 'NN' in word_to_pos[i][index_to_word[token_h]]):
aspect_result[i].append(token_h)
if __name__ == '__main__':
print("Rule 5 triggered.")
print(token_h)
"""
Rule 6:
"""
for child in word_to_child[i][token_t]:
if (word_to_dep[i][child] == 'cop'):
for child2 in word_to_child[i][token_t]:
if (child != child2 and 'VB'
in word_to_pos[i][index_to_word[child2]]):
try:
sn.concept(index_to_word[token_t])
sn.concept(index_to_word[child2])
aspect_result[i].append(token_t)
aspect_result[i].append(child2)
if __name__ == '__main__':
print("rule 6 trigg")
print(token_t)
print(child2)
except KeyError:
pass
else:
"""
Rule 7:Very big to hold.
"""
for word in sent:
if ('RB' in word_to_pos[i][index_to_word[word]]
or 'JJ' in word_to_pos[i][index_to_word[word]]):
for child in word_to_child[i][word]:
if (word_to_dep[i][child] == 'xcomp'
or word_to_dep[i][child] == 'ccomp'):
aspect_result[i].append(word)
if __name__ == '__main__':
print("Rule 7 triggered")
print(word)
"""
Rule 8: Love the sleekness of the player.
"""
for word in sent:
for child in word_to_child[i][word]:
if ('NN' in word_to_pos[i][index_to_word[child]]
and word_to_dep[i][child] == 'nmod'):
for grandchild in word_to_child[i][child]:
if ('IN' in word_to_pos[i][
index_to_word[grandchild]]):
aspect_result[i].append(word)
aspect_result[i].append(child)
if __name__ == '__main__':
print(word)
print(child)
print("Rule 8 triggered.")
"""
Rule 9: Not to mention the price of the phone.
"""
for word in sent:
for child in word_to_child[i][word]:
if (word_to_dep[i][child] == 'dobj'):
aspect_result[i].append(child)
if __name__ == '__main__':
print(child)
print("Rule 9 triggered")
'''
Rule 11 : Checking for conjuctions
'''
for asp in aspect_result[i]:
for word in sent:
if (word_to_dep[i][word] == 'conj'
and word_to_par[i][word] == asp):
aspect_result[i].append(word)
if (__name__ == '__main__'):
print("Rule conj triggered.")
print(word)
finalIAC = [set(aspect_result[i]) for i in range(len(sents))]
finalIAC = [[index_to_word[w] for w in finalIAC[i]]
for i in range(len(sents))]
print(finalIAC)
singleFinalIAC = []
for i in range(len(sents)):
for w in finalIAC[i]:
if w not in stop_words:
singleFinalIAC.append(w)
print(singleFinalIAC)
finalSenti = []
for iac in singleFinalIAC:
try:
concept_info = sn.concept((iac))
finalSenti.append(iac)
except KeyError:
print("No word available for " + iac)
return singleFinalIAC, finalSenti
from senticnet.senticnet import SenticNet
sn = SenticNet('ru')
word = input('Введите ваш комментарий(например "как дела"): ')
lst = word.split()
#concept_info = sn.concept(word)
#polarity_value = sn.polarity_value(word)
#polarity_intense = sn.polarity_intense(word)
#moodtags = sn.moodtags(word)
#semantics = sn.semantics(word)
print(list(map(lambda x: sn.sentics(x), lst)))
pop = input(" ")
from senticnet.senticnet import SenticNet
sn = SenticNet()
print("polarity value:", sn.polarity_value("love"))
print("polarity intense:", sn.polarity_intense("love"))
print("moodtags:", ", ".join(sn.moodtags("love")))
print("semantics:", ", ".join(sn.semantics("love")))
print("\n".join([key + ": " + str(value) for key, value in sn.sentics("love").items()]))
def data_Preprocessing(data, data_test, n_of_words, polarity_threshold):
Reviews = data["Content"]
#Check if all char are ASCII
# If we need another method for Encode/Decode there is string.printable method
for i in range(0, len(Reviews)):
x = Reviews.iloc[i].encode('ascii', errors='ignore').decode()
# Set all the content to lower case
Reviews = Reviews.apply(lambda row: row.lower())
# Add to the follow variable the characters that you want to delete
chars_to_del = "[" + string.punctuation + string.digits + "]"
# Delete all the chars in "chars_to_del" from each row of the dataframe
Reviews = Reviews.apply(lambda row: re.sub(chars_to_del, '', row))
# Tokenize every single words of the data content
Token_Reviews = Reviews.apply(lambda row: nltk.word_tokenize(row))
# Generating the list "stop" of element TO BE REMOVED from the sentences (stopwords, numbers and punctuations)
stop = stopwords.words("english")
# Remove all the words in the variable "stop"
Filtered_Review = Token_Reviews.apply(
lambda row: [w for w in row if not w in stop])
# Stemming the data's content
# Stemming the Filtered sentence, some stemmed words:
# http://snowball.tartarus.org/algorithms/english/stemmer.html
ps = PorterStemmer()
for idx in range(0, len(Filtered_Review)):
Stemmed_Review_temp = []
for word in Filtered_Review.iloc[i]:
Stemmed_Review_temp.append(ps.stem(word))
Filtered_Review.iloc[i] = Stemmed_Review_temp
# Terms choosing: most common word
sn = SenticNet()
Filtered_Review_List = list(itertools.chain.from_iterable(Filtered_Review))
Words_Frquency = FreqDist(Filtered_Review_List)
Most_Common_Words_Frequency = Words_Frquency.most_common(n_of_words)
Most_Common_Words = []
for i in range(0, n_of_words):
Most_Common_Words.append(Most_Common_Words_Frequency[i][0])
index = 1
words_and_polarity = pd.DataFrame(columns=["Word", "Polarity"])
Selected_Words = []
# Terms polarity
for word in Most_Common_Words:
try:
temp = sn.polarity_intense(word)
if (float(temp) > polarity_threshold
or float(temp) < -(polarity_threshold)):
words_and_polarity.loc[index] = [word, float(temp)]
index = index + 1
Selected_Words.append(word)
except Exception:
continue
# Decomment if you want to recomputer the selected words and their polarity
#words_and_polarity.to_csv("Words_and_Polarity.csv", sep=",")
return data, data_test
def pre_process_and_predict(sentence):
wordnet_lemmatizer = WordNetLemmatizer()
# # Replacing double quotes with single, within a string
sentence = sentence.replace("\"", "\'")
# # Removing unnecessary special characters, keeping only , ! ?
sentence = re.sub(r"[^!?,a-zA-Z0-9\ ]+", '', sentence)
# # Lemmatization on verbs
sentence = ' '.join([
wordnet_lemmatizer.lemmatize(word, pos='v')
for word in word_tokenize(sentence)
])
sn = SenticNet()
senti = PySentiStr()
senti.setSentiStrengthPath(CODE_PATH + '/sentistrength/SentiStrength.jar')
senti.setSentiStrengthLanguageFolderPath(
CODE_PATH + '/sentistrength/SentStrength_Data/')
sentiment_score = []
for sen in sent_tokenize(sentence):
senti_pos, senti_neg = senti.getSentiment(sen, score='dual')[0]
senti_pos -= 1
if senti_neg == -1:
senti_neg = 0
sum_pos_score = 0
sum_neg_score = 0
for word in word_tokenize(sen):
try:
w_score = float(sn.polarity_intense(word)) * 5
except KeyError:
w_score = 0
if w_score > 0:
sum_pos_score = sum_pos_score + w_score
elif w_score < 0:
sum_neg_score = sum_neg_score + w_score
sum_pos_score = (sum_pos_score + senti_pos) / 2
sum_neg_score = (sum_neg_score + senti_neg) / 2
sentiment_score.append((sum_pos_score, sum_neg_score))
additional_features_s = []
additional_features_ns = []
contra = []
pos_low = []
pos_medium = []
pos_high = []
neg_low = []
neg_medium = []
neg_high = []
for sum_pos_score, sum_neg_score in sentiment_score:
contra.append(int(sum_pos_score > 0 and abs(sum_neg_score) > 0))
pos_low.append(int(sum_pos_score < 0))
pos_medium.append(int(sum_pos_score >= 0 and sum_pos_score <= 1))
pos_high.append(int(sum_pos_score >= 2))
neg_low.append(int(sum_neg_score < 0))
neg_medium.append(int(sum_neg_score >= 0 and sum_neg_score <= 1))
neg_high.append(int(sum_neg_score >= 2))
additional_features_s = additional_features_s + [
max(pos_medium),
max(pos_high),
max(neg_medium),
max(neg_high)
]
additional_features_ns = additional_features_ns + [
max(pos_low), max(neg_low)
]
tweet = sentence
punctuation_count = SequencePunctuationCount(tweet)
character_count = SequenceCharacterCount(tweet)
capitalized_count = CapitalizedCount(tweet)
exclamation_count = ExclamationCount(tweet)
# emoji_count = EmojiCount(tweet)
f_count = [
punctuation_count, character_count, capitalized_count,
exclamation_count
]
for count in f_count:
f_low = int(count == 0)
f_medium = int(count >= 1 and count <= 3)
f_high = int(count >= 4)
additional_features_s = additional_features_s + [f_medium, f_high]
additional_features_ns = additional_features_ns + [f_low]
X = [sentence]
in_file = open(os.path.join(PICKLES_PATH, "vocab.pickle"), "rb")
vocab = pickle.load(in_file)
in_file.close()
in_file = open(os.path.join(PICKLES_PATH, "model.pickle"), "rb")
model = pickle.load(in_file)
in_file.close()
vectorizer = TfidfVectorizer(vocabulary=vocab)
X = vectorizer.fit_transform(X)
ans = int(sum(model.predict(X)))
print('Sentence : ', sentence)
print('Sarcastic features : ', additional_features_s)
print('Not Sarcastic features : ', additional_features_ns)
print('Contradict : ', max(contra))
print('Model Predict : ', ans)
print(
'My obs : ',
int((sum(additional_features_s) >= sum(additional_features_ns))
and max(contra) == 1))
print('Final Prd : ', end='')
if ans == 1 or ((sum(additional_features_s) >= sum(additional_features_ns))
and max(contra) == 1):
return True
else:
return False
def __init__(self):
self.col = ['Name', 'Brand', 'Price', 'Title', 'Score', 'Time', 'Text']
self.sn = SenticNet('en')
self.wordnet_lemmatizer = WordNetLemmatizer()
from senticnet.senticnet import SenticNet
teste = []
sn = SenticNet('pt')
concept_info = sn.concept('amor')
polarity_value = sn.polarity_value('amor')
polarity_intense = sn.polarity_intense('amor')
moodtags = sn.moodtags('amor')
semantics = sn.semantics('amor')
sentics = sn.sentics('amor')
teste.append(concept_info)
print(teste)