def filterAdj(phrasesDict,filename):
phrasesDict = OrderedDict(sorted(phrasesDict.items(), key=operator.itemgetter(1), reverse=True))
newPhrases = dict()
exclude = set(string.punctuation)
exclude.remove("_")
for line_words, count in phrasesDict.items():
#Preprocessing text
line_words = ' '.join([apostropheList[word] if word in apostropheList else word for word in line_words.split()])
line_words = ''.join(ch for ch in line_words if ch not in exclude)
line_words = re.sub(r' [a-z][$]? ', ' ', line_words)
line_words = [Word(word).lemmatize() for word in line_words.split() if(word not in stopwords.words("english") and not word.isdigit()) and len(word) > 2]
line_words = ' '.join(line_words)
if(len(line_words.strip(" ").split()) == 2):
if(line_words in newPhrases):
newPhrases[line_words] += count
else:
newPhrases[line_words] = count
#Bigrams from the file
newPhrases = OrderedDict(sorted(newPhrases.items(), key=operator.itemgetter(1), reverse=True))
#Applying Threshold to Bigrams
nouns1 = []
for key, value in newPhrases.items():
if value >= 3:
nouns1.append(key)
stopWords = stopwords.words("english")
exclude = set(string.punctuation)
reviewTitle = []
reviewContent = []
#Reading the original file
with open(filename) as f:
review = []
for line in f:
if line[:6] == "[+][t]":
if review:
reviewContent.append(review)
review = []
reviewTitle.append(line.split("[+][t]")[1].rstrip("\r\n"))
elif line[:6] == "[-][t]":
if review:
reviewContent.append(review)
review = []
reviewTitle.append(line.split("[-][t]")[1].rstrip("\r\n"))
else:
if "##" in line:
x = line.split("##")
#if len(x[0]) != 0:
for i in range(1, len(x)):
review.append(x[i].rstrip("\r\n"))
else:
continue
reviewContent.append(review)
#tb = Blobber(pos_tagger=PerceptronTagger())
tb = Blobber(pos_tagger=NLTKTagger())
nounScores = dict()
#Writing to a file
f = open('modified.txt', 'w')
for a in range(len(reviewContent)):
f.write("[t]")
#Finding Bigrams in title
text = reviewTitle[a]
x = tb(text).tags #NLTK tagger
e = 0
while e<len(x):
tagList = ""
temp = ""
wrt = x[e][0]
e = e+1
count = e
tp = 0
if(count<len(x) and (x[count-1][1] == "NN" or "JJ") and (x[count][1] == "NN" or "JJ")):
tagList = x[count-1][0] + " " + x[count][0]
temp = x[count][0]
count = count+1
if tagList != "":
if tagList in nouns1:
tagList = tagList.replace(' ', '')
f.write(tagList)
tp = 1
e = count
if tp == 0:
f.write(wrt)
f.write(" ")
f.write("\r\n")
#Finding bigrams in review
for i in range(len(reviewContent[a])):
text = reviewContent[a][i]
x = tb(text).tags #NLTK tagger
tagList = []
e = 0
f.write("##")
while e<len(x):
tagList = ""
temp = ""
wrt = x[e][0]
e = e+1
count = e
tp = 0
if(count<len(x) and (x[count-1][1] == "NN" or "JJ") and (x[count][1] == "NN" or "JJ")):
tagList = x[count-1][0] + " " + x[count][0]
temp = x[count][0]
count = count+1
if tagList != "":
#Checking if consecutive nouns we found out are in noun phrases
if tagList in nouns1:
tagList = tagList.replace(' ', '')
f.write(tagList)
tp = 1
e = count
if tp == 0:
f.write(wrt)
f.write(" ")
f.write(".\r\n")
def test_can_use_different_pos_tagger(self):
tagger = NLTKTagger()
blob = tb.TextBlob("this is some text", pos_tagger=tagger)
assert_true(isinstance(blob.pos_tagger, NLTKTagger))
def findFeatures(reviewContent,filename):
nounScores = dict()
adjDict = dict()
tb = Blobber(pos_tagger=NLTKTagger())
for a in range(len(reviewContent)): # Stores the score of the nouns
#print("printing words::::")
#print(reviewContent[a])
text = ' '.join([word for word in reviewContent[a].split() if word not in stopwords.words("english")])
text = ''.join(ch for ch in text if ch not in exclude)
text = nltk.word_tokenize(text)
x = nltk.pos_tag(text)
# Get the noun/adjective words and store it in tagList
tagList = []
for e in x:
if (e[1] == "NN" or e[1] == "JJ"):
tagList.append(e)
# Add the nouns(which are not in the nounScores dict) to the dict
for e in tagList:
if e[1] == "NN":
if e[0] not in nounScores:
nounScores[e[0]] = 0
# For every adjective, find nearby noun
for l in range(len(tagList)):
if ("JJ" in tagList[l][1]):
j = k = leftHop = rightHop = -1
for j in range(l + 1, len(tagList)):
if (j == l + maxHops):
break
if ("NN" in tagList[j][1]):
rightHop = (j - l)
break
for k in range(l - 1, -1, -1):
if (j == l - maxHops):
break
if ("NN" in tagList[k][1]):
leftHop = (l - k)
break
# Compare which noun is closer to adjective(left or right) and assign the adj to corresponding noun
if (leftHop > 0 and rightHop > 0):
if (leftHop - rightHop) >= 0:
adjDict[tagList[l][0]] = tagList[j][0]
nounScores[tagList[j][0]] += 1
else:
adjDict[tagList[l][0]] = tagList[k][0]
nounScores[tagList[k][0]] += 1
elif leftHop > 0:
adjDict[tagList[l][0]] = tagList[k][0]
nounScores[tagList[k][0]] += 1
elif rightHop > 0:
adjDict[tagList[l][0]] = tagList[j][0]
nounScores[tagList[j][0]] += 1
nounScores = OrderedDict(sorted(nounScores.items(), key=operator.itemgetter(1)))
return filterAdj(nounScores, adjDict, filename)
from textblob import Blobber
from textblob.sentiments import NaiveBayesAnalyzer
from textblob.taggers import NLTKTagger
# Setup
# nltk.download('punkt')
# nltk.download('wordnet')
# nltk.download('averaged_perceptron_tagger')
# nltk.download('brown')
# nltk.download('movie_reviews')
# Load config
import json
rx = re.compile('(["#\'\\%`])')
tb = Blobber(pos_tagger=NLTKTagger(), analyzer=NaiveBayesAnalyzer())
data = pd.read_csv('/media/salah/e58c5812-2860-4033-90c6-83b7ffaa8b88/MLStock/dataset/Layer1_dataset/Model2/Layer1_base_dataset.csv')
# Keeping only the neccessary columns
data['headline'] = data['headline'].apply(lambda x: str(x).lower().replace(' ## ',''))
from nltk.corpus import sentiwordnet as swn
#result_reduce[1].split(',')[0]
def sent_from_text(text):
test_b = tb( text )
pos_count = 0.0
neg_count = 0.0
pos_sum = 0.0
neg_sum = 0.0
Calculate the semantic similarity between two sentences. The last
parameter is True or False depending on whether information content
normalization is desired or not.
"""
return DELTA * semantic_similarity(sentence_1, sentence_2, info_content_norm) + \
(1.0 - DELTA) * word_order_similarity(sentence_1, sentence_2)
val = similarity("I hate Trump", "I like Trump", False)
if val > .7:
print("close enough")
else:
print("not even")
nltk_tagger = NLTKTagger()
#add this to docker 'python -m textblob.download_corpora'
def analyze(content):
zen = TextBlob(content)
overall_total = 0.0
overall_score = 0.0
for sent in zen.sentences:
res = query(sent)
overall_total += res[0]
overall_score += res[1]
return [overall_total, overall_score]
class BaseBlob(StringlikeMixin, BlobComparableMixin):
"""An abstract base class that all textblob classes will inherit from.
Includes words, POS tag, NP, and word count properties. Also includes
basic dunder and string methods for making objects like Python strings.
:param text: A string.
:param tokenizer: (optional) A tokenizer instance. If ``None``,
defaults to :class:`WordTokenizer() <textblob.tokenizers.WordTokenizer>`.
:param np_extractor: (optional) An NPExtractor instance. If ``None``,
defaults to :class:`FastNPExtractor() <textblob.en.np_extractors.FastNPExtractor>`.
:param pos_tagger: (optional) A Tagger instance. If ``None``,
defaults to :class:`NLTKTagger <textblob.en.taggers.NLTKTagger>`.
:param analyzer: (optional) A sentiment analyzer. If ``None``,
defaults to :class:`PatternAnalyzer <textblob.en.sentiments.PatternAnalyzer>`.
:param parser: A parser. If ``None``, defaults to
:class:`PatternParser <textblob.en.parsers.PatternParser>`.
:param classifier: A classifier.
.. versionchanged:: 0.6.0
``clean_html`` parameter deprecated, as it was in NLTK.
"""
np_extractor = FastNPExtractor()
pos_tagger = NLTKTagger()
tokenizer = WordTokenizer()
translator = Translator()
analyzer = PatternAnalyzer()
parser = PatternParser()
def __init__(self,
text,
tokenizer=None,
pos_tagger=None,
np_extractor=None,
analyzer=None,
parser=None,
classifier=None,
clean_html=False):
if not isinstance(text, basestring):
raise TypeError('The `text` argument passed to `__init__(text)` '
'must be a string, not {0}'.format(type(text)))
if clean_html:
raise NotImplementedError(
"clean_html has been deprecated. "
"To remove HTML markup, use BeautifulSoup's "
"get_text() function")
self.raw = self.string = text
self.stripped = lowerstrip(self.raw, all=True)
_initialize_models(self, tokenizer, pos_tagger, np_extractor, analyzer,
parser, classifier)
@cached_property
def words(self):
"""Return a list of word tokens. This excludes punctuation characters.
If you want to include punctuation characters, access the ``tokens``
property.
:returns: A :class:`WordList <WordList>` of word tokens.
"""
return WordList(word_tokenize(self.raw, include_punc=False))
@cached_property
def tokens(self):
"""Return a list of tokens, using this blob's tokenizer object
(defaults to :class:`WordTokenizer <textblob.tokenizers.WordTokenizer>`).
"""
return WordList(self.tokenizer.tokenize(self.raw))
def tokenize(self, tokenizer=None):
"""Return a list of tokens, using ``tokenizer``.
:param tokenizer: (optional) A tokenizer object. If None, defaults to
this blob's default tokenizer.
"""
t = tokenizer if tokenizer is not None else self.tokenizer
return WordList(t.tokenize(self.raw))
def parse(self, parser=None):
"""Parse the text.
:param parser: (optional) A parser instance. If ``None``, defaults to
this blob's default parser.
.. versionadded:: 0.6.0
"""
p = parser if parser is not None else self.parser
return p.parse(self.raw)
def classify(self):
"""Classify the blob using the blob's ``classifier``."""
if self.classifier is None:
raise NameError("This blob has no classifier. Train one first!")
return self.classifier.classify(self.raw)
@cached_property
def sentiment(self):
"""Return a tuple of form (polarity, subjectivity ) where polarity
is a float within the range [-1.0, 1.0] and subjectivity is a float
within the range [0.0, 1.0] where 0.0 is very objective and 1.0 is
very subjective.
:rtype: namedtuple of the form ``Sentiment(polarity, subjectivity)``
"""
return self.analyzer.analyze(self.raw)
@cached_property
def sentiment_assessments(self):
"""Return a tuple of form (polarity, subjectivity, assessments ) where
polarity is a float within the range [-1.0, 1.0], subjectivity is a
float within the range [0.0, 1.0] where 0.0 is very objective and 1.0
is very subjective, and assessments is a list of polarity and
subjectivity scores for the assessed tokens.
:rtype: namedtuple of the form ``Sentiment(polarity, subjectivity,
assessments)``
"""
return self.analyzer.analyze(self.raw, keep_assessments=True)
@cached_property
def polarity(self):
"""Return the polarity score as a float within the range [-1.0, 1.0]
:rtype: float
"""
return PatternAnalyzer().analyze(self.raw)[0]
@cached_property
def subjectivity(self):
"""Return the subjectivity score as a float within the range [0.0, 1.0]
where 0.0 is very objective and 1.0 is very subjective.
:rtype: float
"""
return PatternAnalyzer().analyze(self.raw)[1]
@cached_property
def noun_phrases(self):
"""Returns a list of noun phrases for this blob."""
return WordList([
phrase.strip().lower()
for phrase in self.np_extractor.extract(self.raw)
if len(phrase) > 1
])
@cached_property
def pos_tags(self):
"""Returns an list of tuples of the form (word, POS tag).
Example:
::
[('At', 'IN'), ('eight', 'CD'), ("o'clock", 'JJ'), ('on', 'IN'),
('Thursday', 'NNP'), ('morning', 'NN')]
:rtype: list of tuples
"""
if isinstance(self, TextBlob):
return [
val for sublist in [s.pos_tags for s in self.sentences]
for val in sublist
]
else:
return [(Word(word, pos_tag=t), unicode(t))
for word, t in self.pos_tagger.tag(self)
if not PUNCTUATION_REGEX.match(unicode(t))]
tags = pos_tags
@cached_property
def word_counts(self):
"""Dictionary of word frequencies in this text.
"""
counts = defaultdict(int)
stripped_words = [lowerstrip(word) for word in self.words]
for word in stripped_words:
counts[word] += 1
return counts
@cached_property
def np_counts(self):
"""Dictionary of noun phrase frequencies in this text.
"""
counts = defaultdict(int)
for phrase in self.noun_phrases:
counts[phrase] += 1
return counts
def ngrams(self, n=3):
"""Return a list of n-grams (tuples of n successive words) for this
blob.
:rtype: List of :class:`WordLists <WordList>`
"""
if n <= 0:
return []
grams = [
WordList(self.words[i:i + n])
for i in range(len(self.words) - n + 1)
]
return grams
def translate(self, from_lang="auto", to="en"):
"""Translate the blob to another language.
Uses the Google Translate API. Returns a new TextBlob.
Requires an internet connection.
Usage:
::
>>> b = TextBlob("Simple is better than complex")
>>> b.translate(to="es")
TextBlob('Lo simple es mejor que complejo')
Language code reference:
https://developers.google.com/translate/v2/using_rest#language-params
.. versionadded:: 0.5.0.
:param str from_lang: Language to translate from. If ``None``, will attempt
to detect the language.
:param str to: Language to translate to.
:rtype: :class:`BaseBlob <BaseBlob>`
"""
return self.__class__(
self.translator.translate(self.raw,
from_lang=from_lang,
to_lang=to))
def detect_language(self):
"""Detect the blob's language using the Google Translate API.
Requires an internet connection.
Usage:
::
>>> b = TextBlob("bonjour")
>>> b.detect_language()
u'fr'
Language code reference:
https://developers.google.com/translate/v2/using_rest#language-params
.. versionadded:: 0.5.0
:rtype: str
"""
return self.translator.detect(self.raw)
def correct(self):
"""Attempt to correct the spelling of a blob.
.. versionadded:: 0.6.0
:rtype: :class:`BaseBlob <BaseBlob>`
"""
# regex matches: word or punctuation or whitespace
tokens = nltk.tokenize.regexp_tokenize(self.raw, r"\w+|[^\w\s]|\s")
corrected = (Word(w).correct() for w in tokens)
ret = ''.join(corrected)
return self.__class__(ret)
def _cmpkey(self):
"""Key used by ComparableMixin to implement all rich comparison
operators.
"""
return self.raw
def _strkey(self):
"""Key used by StringlikeMixin to implement string methods."""
return self.raw
def __hash__(self):
return hash(self._cmpkey())
def __add__(self, other):
'''Concatenates two text objects the same way Python strings are
concatenated.
Arguments:
- `other`: a string or a text object
'''
if isinstance(other, basestring):
return self.__class__(self.raw + other)
elif isinstance(other, BaseBlob):
return self.__class__(self.raw + other.raw)
else:
raise TypeError(
'Operands must be either strings or {0} objects'.format(
self.__class__.__name__))
def split(self, sep=None, maxsplit=sys.maxsize):
"""Behaves like the built-in str.split() except returns a
WordList.
:rtype: :class:`WordList <WordList>`
"""
return WordList(self._strkey().split(sep, maxsplit))
class Blobber(object):
"""A factory for TextBlobs that all share the same tagger,
tokenizer, parser, classifier, and np_extractor.
Usage:
>>> from textblob import Blobber
>>> from textblob.taggers import NLTKTagger
>>> from textblob.tokenizers import SentenceTokenizer
>>> tb = Blobber(pos_tagger=NLTKTagger(), tokenizer=SentenceTokenizer())
>>> blob1 = tb("This is one blob.")
>>> blob2 = tb("This blob has the same tagger and tokenizer.")
>>> blob1.pos_tagger is blob2.pos_tagger
True
:param tokenizer: (optional) A tokenizer instance. If ``None``,
defaults to :class:`WordTokenizer() <textblob.tokenizers.WordTokenizer>`.
:param np_extractor: (optional) An NPExtractor instance. If ``None``,
defaults to :class:`FastNPExtractor() <textblob.en.np_extractors.FastNPExtractor>`.
:param pos_tagger: (optional) A Tagger instance. If ``None``,
defaults to :class:`NLTKTagger <textblob.en.taggers.NLTKTagger>`.
:param analyzer: (optional) A sentiment analyzer. If ``None``,
defaults to :class:`PatternAnalyzer <textblob.en.sentiments.PatternAnalyzer>`.
:param parser: A parser. If ``None``, defaults to
:class:`PatternParser <textblob.en.parsers.PatternParser>`.
:param classifier: A classifier.
.. versionadded:: 0.4.0
"""
np_extractor = FastNPExtractor()
pos_tagger = NLTKTagger()
tokenizer = WordTokenizer()
analyzer = PatternAnalyzer()
parser = PatternParser()
def __init__(self,
tokenizer=None,
pos_tagger=None,
np_extractor=None,
analyzer=None,
parser=None,
classifier=None):
_initialize_models(self, tokenizer, pos_tagger, np_extractor, analyzer,
parser, classifier)
def __call__(self, text):
"""Return a new TextBlob object with this Blobber's ``np_extractor``,
``pos_tagger``, ``tokenizer``, ``analyzer``, and ``classifier``.
:returns: A new :class:`TextBlob <TextBlob>`.
"""
return TextBlob(text,
tokenizer=self.tokenizer,
pos_tagger=self.pos_tagger,
np_extractor=self.np_extractor,
analyzer=self.analyzer,
parser=self.parser,
classifier=self.classifier)
def __repr__(self):
classifier_name = self.classifier.__class__.__name__ + "()" if self.classifier else "None"
return ("Blobber(tokenizer={0}(), pos_tagger={1}(), "
"np_extractor={2}(), analyzer={3}(), parser={4}(), classifier={5})")\
.format(self.tokenizer.__class__.__name__,
self.pos_tagger.__class__.__name__,
self.np_extractor.__class__.__name__,
self.analyzer.__class__.__name__,
self.parser.__class__.__name__,
classifier_name)
__str__ = __repr__
def filterAdj(nounScores, adjDict, filename):
adjectList = list(adjDict.keys())
nouns = []
for key, value in nounScores.items():
if value >= 3:
nouns.append(key)
nouns1 = [
"sound quality", "battery life", "great phone", "cell phone",
"menu option", "color screen", "flip phone", "samsung phone",
"nokia phones", "corporate email", "ring tone", "tmobile service"
]
nouns = set(nouns)
stopWords = stopwords.words("english")
exclude = set(string.punctuation)
reviewTitle = []
reviewContent = []
with open(filename) as f:
review = []
for line in f:
if line[:6] == "[+][t]":
if review:
reviewContent.append(review)
review = []
reviewTitle.append(line.split("[+][t]")[1].rstrip("\r\n"))
elif line[:6] == "[-][t]":
if review:
reviewContent.append(review)
review = []
reviewTitle.append(line.split("[-][t]")[1].rstrip("\r\n"))
else:
if "##" in line:
x = line.split("##")
#if len(x[0]) != 0:
for i in xrange(1, len(x)):
review.append(x[i].rstrip("\r\n"))
else:
continue
reviewContent.append(review)
#tb = Blobber(pos_tagger=PerceptronTagger())
tb = Blobber(pos_tagger=NLTKTagger())
nounScores = dict()
f = open('modified.txt', 'w')
for a in xrange(len(reviewContent)):
f.write("[t]" + reviewTitle[a])
f.write("\r\n")
#Stores the score of the nouns
for i in xrange(len(reviewContent[a])):
text = reviewContent[a][i]
x = tb(text).tags #Perceptron tagger
#Get the noun/adjective words and store it in tagList
tagList = []
e = 0
f.write("##")
while e < len(x):
tagList = []
f.write(x[e][0])
e = e + 1
count = e
if (count < len(x) and x[count - 1][1] == "NN"
and x[count][1] == "NN"):
tagList.append(x[count - 1][0])
while (count < len(x) and x[count][1] == "NN"):
tagList.append(x[count][0])
count = count + 1
if tagList != [] and len(tagList) == 2:
if set(tagList) <= nouns:
for t in range(1, len(tagList)):
f.write(tagList[t])
e = count
f.write(" ")
f.write(".\r\n")
return adjectList
def findFeatures(reviewContent, filename):
#nounScores is the dict containing nouns from all reviews and their respective scores from HAC algorithm
nounScores = dict()
#adjDict dict contains adjective and the corresponding noun which it is assigned to
adjDict = dict()
tb = Blobber(pos_tagger=NLTKTagger())
for a in xrange(len(reviewContent)): #Stores the score of the nouns
for i in xrange(len(reviewContent[a])):
text = ' '.join([
word for word in reviewContent[a][i].split()
if word not in stopwords.words("english")
])
text = ''.join(ch for ch in text if ch not in exclude)
text = nltk.word_tokenize(text)
x = nltk.pos_tag(text)
#Get the noun/adjective words and store it in tagList
tagList = []
for e in x:
if (e[1] == "NN" or e[1] == "JJ"):
tagList.append(e)
#Add the nouns(which are not in the nounScores dict) to the dict
for e in tagList:
if e[1] == "NN":
if e[0] not in nounScores:
nounScores[e[0]] = 0
#For every adjective, find nearby noun
for l in range(len(tagList)):
if ("JJ" in tagList[l][1]):
j = k = leftHop = rightHop = -1
#Find the closest noun to the right of the adjective in the line
for j in range(l + 1, len(tagList)):
if (j == l + maxHops):
break
if ("NN" in tagList[j][1]):
rightHop = (j - l)
break
#Find the closest noun to the left of the adjective in the line
for k in range(l - 1, -1, -1):
#Incase hopped the 'maxHops' number of words and no noun was found, ignore the adjective
if (j == l - maxHops):
break
if ("NN" in tagList[k][1]):
leftHop = (l - k)
break
#Compare which noun is closer to adjective(left or right) and assign the adj to corresponding noun
if (leftHop > 0 and rightHop >
0): #If nouns exist on both sides of adjective
if (leftHop - rightHop) >= 0: #If left noun is farther
adjDict[tagList[l][0]] = tagList[j][0]
nounScores[tagList[j][0]] += 1
else: #If right noun is farther
adjDict[tagList[l][0]] = tagList[k][0]
nounScores[tagList[k][0]] += 1
elif leftHop > 0: #If noun is not found on RHS of adjective
adjDict[tagList[l][0]] = tagList[k][0]
nounScores[tagList[k][0]] += 1
elif rightHop > 0: #If noun is not found on LHS of adjective
adjDict[tagList[l][0]] = tagList[j][0]
nounScores[tagList[j][0]] += 1
nounScores = OrderedDict(
sorted(nounScores.items(), key=operator.itemgetter(1)))
return filterAdj(nounScores, adjDict, filename)
from textblob import Blobber
from textblob.taggers import NLTKTagger
# used to combine commonly used taggers, chunkers, etc to keep code DRY
tb = Blobber(pos_tagger = NLTKTagger())
blob = tb("This is amazing!")
another_blob = tb("This sucks!")
blob.pos_tagger is another_blob.pos_tagger
def getList():
#reading from the created file "modified.txt"
with open("modified.txt") as f:
review = []
for line in f:
if line[:3] == "[t]":
if review:
reviewContent.append(review)
review = []
reviewTitle.append(line.split("[t]")[1].rstrip("\r\n"))
else:
if "##" in line:
x = line.split("##")
for i in range(1, len(x)):
review.append(x[i].rstrip("\r\n"))
else:
continue
reviewContent.append(review)
tb = Blobber(pos_tagger=NLTKTagger())
nounScores = dict()
for a in range(len(reviewContent)): #Stores the score of the nouns
for i in range(len(reviewContent[a])):
#text = reviewContent[a][i]
text = ' '.join([word for word in reviewContent[a][i].split() if word not in stopwords.words("english")])
text = ''.join(ch for ch in text if ch not in exclude)
text = nltk.word_tokenize(text)
x = nltk.pos_tag(text)
#x = TextBlob(text).tags #textblob tagger
#x = tb(text).tags #Perceptron tagger
#Get the noun/adjective words and store it in tagList
tagList = []
for e in x:
if(e[1] == "NN" or e[1] == "JJ"):
tagList.append(e)
#Add the nouns(which are not in the nounScores dict) to the dict
for e in tagList:
if e[1] == "NN":
if e[0] not in nounScores:
nounScores[e[0]] = 0
#For every adjective, find nearby noun
l=0
for l in range(len(tagList)):
if(tagList[l][1] == "JJ"):
check=0
j = 0
k = 0
ct1 = 0
for j in range(l + 1, len(tagList)):
if ct1 == 4:
break
if(tagList[j][1] == "NN"):
#nounScores[tagList[j][0]] += 1
check = 1
break
ct = 0
if(l > 0):
if j == 0:
j = len(tagList)
for k in range(l - 1, 0, -1):
if ct == 4:
break
ct += 1
if(tagList[k][1] == "NN"):
if(j != len(tagList)):
nounScores[tagList[min(j, k)][0]] += 1
else:
nounScores[tagList[k][0]] += 1
break
elif check == 1:
nounScores[tagList[j][0]] += 1
nounScores = OrderedDict(sorted(nounScores.items(), key=operator.itemgetter(1)))
nouns = []
for key, value in nounScores.items():
if value >= 3:
nouns.append(key)
return nouns
def makeTextBlob(txt):
"""Wrapper for TextBlob Call"""
return TextBlob(txt, pos_tagger=NLTKTagger())
def getList():
"""
:rtype: object
"""
# reading from the created file "modified.txt"
adjNounAll = dict()
adjNounList = dict()
with open("modified.txt") as f:
review = []
for line in f:
if line[:3] == "[t]":
if review:
reviewContent.append(review)
review = []
else:
if "##" in line:
x = line.split("##")
for i in range(1, len(x)):
review.append(x[i].rstrip("\r\n"))
else:
continue
reviewContent.append(review)
tb = Blobber(pos_tagger=NLTKTagger())
nounScores = dict()
for a in range(len(reviewContent)):
for i in range(len(reviewContent[a])):
text = ' '.join([
word for word in reviewContent[a][i].split()
if word not in stopwords.words("english")
])
text = ''.join(ch for ch in text if ch not in exclude)
text = nltk.word_tokenize(text)
x = nltk.pos_tag(text)
tagList = []
for e in x:
if (e[1] == "NN" or e[1] == "JJ"):
tagList.append(e)
# Add the nouns(which are not in the nounScores dict) to the dict
for e in tagList:
if e[1] == "NN":
if e[0] not in nounScores:
nounScores[e[0]] = 0
# For every adjective, find nearby noun l=0
for l in range(len(tagList)):
if (tagList[l][1] == "JJ"):
check = 0
j = 0
k = 0
for j in range(l + 1, len(tagList)):
if (tagList[j][1] == "NN"):
check = 1
break
ct = 0
if (l > 0):
if j == 0:
j = len(tagList)
for k in range(l - 1, 0, -1):
if ct == 4:
break
ct += 1
if (tagList[k][1] == "NN"):
if (j != len(tagList)):
nounScores[tagList[min(j, k)][0]] += 1
adjNounAll[tagList[min(
j, k)][0]] = tagList[l][0]
else:
nounScores[tagList[k][0]] += 1
adjNounAll[tagList[k][0]] = tagList[l][0]
break
elif check == 1:
nounScores[tagList[j][0]] += 1
adjNounAll[tagList[j][0]] = tagList[l][0]
nounScores = OrderedDict(
sorted(nounScores.items(), key=operator.itemgetter(1)))
nouns = []
for key, value in nounScores.items():
if value >= 3:
nouns.append(key)
adjNounList[key] = adjNounAll[key]
return [nouns, adjNounList]
#tokenize the sentence into words
for j in word_tokenize(i):
temp.append((j.lower())) # what does this do?
data.append(temp)
# Create CBOW model
model = gensim.models.Word2Vec(data, min_count = 1, size = 100, window = 5)
X = model[model.wv.vocab]
words = list(model.wv.vocab)
#*******************************#
# Naive noun plotting
nltkTagger = NLTKTagger()
blob = TextBlob(s, pos_tagger=nltkTagger)
allTags = blob.pos_tags
nouns = []
# remove non nouns
for n in allTags:
if str(n[1]) == 'NN':
nouns.append(n)
# print('NOUN LIST: ', nouns)
# Remove duplicates
nounFrequencies = {}
for n in nouns:
if n[0] in nounFrequencies:
