class TestPatternSentiment(unittest.TestCase):
def setUp(self):
self.analyzer = PatternAnalyzer()
def test_kind(self):
assert_equal(self.analyzer.kind, CONTINUOUS)
def test_analyze(self):
p1 = "I feel great this morning."
n1 = "This is a terrible car."
p1_result = self.analyzer.analyze(p1)
n1_result = self.analyzer.analyze(n1)
assert_true(p1_result[0] > 0)
assert_true(n1_result[0] < 0)
assert_equal(p1_result.polarity, p1_result[0])
assert_equal(p1_result.subjectivity, p1_result[1])
def test_analyze_assessments(self):
p1 = "I feel great this morning."
n1 = "This is a terrible car."
p1_result = self.analyzer.analyze(p1,keep_assessments=True)
n1_result = self.analyzer.analyze(n1,keep_assessments=True)
p1_assessment = p1_result.assessments[0]
n1_assessment = n1_result.assessments[0]
assert_true(p1_assessment[1] > 0)
assert_true(n1_assessment[1] < 0)
assert_equal(p1_result.polarity, p1_assessment[1])
assert_equal(p1_result.subjectivity, p1_assessment[2])
class TestPatternSentiment(unittest.TestCase):
def setUp(self):
self.analyzer = PatternAnalyzer()
def test_kind(self):
assert_equal(self.analyzer.kind, CONTINUOUS)
def test_analyze(self):
p1 = "I feel great this morning."
n1 = "This is a terrible car."
p1_result = self.analyzer.analyze(p1)
n1_result = self.analyzer.analyze(n1)
assert_true(p1_result[0] > 0)
assert_true(n1_result[0] < 0)
assert_equal(p1_result.polarity, p1_result[0])
assert_equal(p1_result.subjectivity, p1_result[1])
def test_analyze_assessments(self):
p1 = "I feel great this morning."
n1 = "This is a terrible car."
p1_result = self.analyzer.analyze(p1, keep_assessments=True)
n1_result = self.analyzer.analyze(n1, keep_assessments=True)
p1_assessment = p1_result.assessments[0]
n1_assessment = n1_result.assessments[0]
assert_true(p1_assessment[1] > 0)
assert_true(n1_assessment[1] < 0)
assert_equal(p1_result.polarity, p1_assessment[1])
assert_equal(p1_result.subjectivity, p1_assessment[2])
def add_sentiment_features(df):
'''
Add sentiments as columns.
Use TextBlob to do the sentiment analysis.
Input
- df: the pandas dataframe that has Transciprt column
Output
- df: the pandas dataframe with the added columns: senti_p_postivie, senti_polarity, senti_subjectivity
'''
p_positive_list = []
polarity_list = []
subjectivity_list = []
past_sentence = ""
for sentence in df['Transcript']:
if sentence != past_sentence:
blob = TextBlob(sentence, analyzer=NaiveBayesAnalyzer())
p_positive = blob.sentiment.p_pos
blob = TextBlob(sentence, analyzer=PatternAnalyzer())
polarity = blob.sentiment.polarity
subjectivity = blob.sentiment.subjectivity
past_sentence = sentence
p_positive_list.append(p_positive)
polarity_list.append(polarity)
subjectivity_list.append(subjectivity)
df['senti_p_positive'] = p_positive_list
df['senti_polarity'] = polarity_list
df['senti_subjectivity'] = subjectivity_list
return df
def subjectivity(self):
"""Return the subjectivity score as a float within the range [0.0, 1.0]
where 0.0 is very objective and 1.0 is very subjective.
:rtype: float
"""
return PatternAnalyzer().analyze(self.raw)[1]
def calc_sentiment_precise(headline):
sentiment = TextBlob(headline, analyzer=PatternAnalyzer()).sentiment[0]
if sentiment > 0.5: return 0.10
elif sentiment > 0.1: return 0.05
elif sentiment < -0.5: return -0.10
elif sentiment < -0.1: return -0.05
else: return 0
def calc_sentiment(headline):
sentiment = TextBlob(headline, analyzer=PatternAnalyzer()).sentiment[0]
if sentiment > 0.1:
return 1
elif sentiment < -0.1:
return -1
else:
return 0
def tradFeelingPattern(text, analyzer=PatternAnalyzer()):
b = TextBlob(text)
translatedText = text
try:
translatedText = b.translate(to="en")
except exceptions.NotTranslated:
pass
b = TextBlob(str(translatedText), analyzer=analyzer)
return {"polarity": b.polarity, "language": "en"}
def textblob_pattern(df):
labels = [-1, 0 , 1]
analyzer=PatternAnalyzer()
outputs = df['input'].apply(lambda x: TextBlob(x, analyzer=analyzer).sentiment.polarity)
correct = 0
for label, output in zip(df["label"], outputs):
output = min(labels, key=lambda x:abs(x-output))
if label == output: correct += 1
return correct / len(df)
def combine_sentiments(text, verbose=False):
'''
Combine Naive Bayes Classifier with Pattern Analyzer.
'''
blob_nb = TextBlob(text.raw, analyzer=NaiveBayesAnalyzer())
blob_pa = TextBlob(text.raw, analyzer=PatternAnalyzer())
average_sentiment = mean([prob_to_polarity(blob_nb.sentiment.p_pos),
blob_pa.sentiment.polarity])
return average_sentiment
class TestPatternSentiment(unittest.TestCase):
def setUp(self):
self.analyzer = PatternAnalyzer()
def test_kind(self):
assert_equal(self.analyzer.kind, CONTINUOUS)
def test_analyze(self):
p1 = "I feel great this morning."
n1 = "This is a terrible car."
assert_true(self.analyzer.analyze(p1)[0] > 0)
assert_true(self.analyzer.analyze(n1)[0] < 0)
class TestPatternSentiment(unittest.TestCase):
def setUp(self):
self.analyzer = PatternAnalyzer()
def test_kind(self):
assert_equal(self.analyzer.kind, CONTINUOUS)
def test_analyze(self):
p1 = "I feel great this morning."
n1 = "This is a terrible car."
assert_true(self.analyzer.analyze(p1)[0] > 0)
assert_true(self.analyzer.analyze(n1)[0] < 0)
def calc_sentiment_returns(headline_list):
if type(headline_list) == str: return {}
headline = headline_list[0]
tickers = headline_list[1:]
sentiment_dict = {}
sentiment = TextBlob(headline, analyzer=PatternAnalyzer()).sentiment[0]
sentiment = round(sentiment * 4) / 4
for ticker in tickers:
try:
sentiment_dict[ticker] = rf[ticker].loc[sentiment] / 100
except:
sentiment_dict[ticker] = sentiment / 100
return sentiment_dict
def pick_largest_sentiment(text, verbose=False):
'''
Return the polarity of the model that has higher absolute value.
'''
blob_nb = TextBlob(text, analyzer=NaiveBayesAnalyzer())
blob_pa = TextBlob(text, analyzer=PatternAnalyzer())
naive = abs(prob_to_polarity(blob_nb.sentiment.p_pos))
pattern = abs(blob_pa.sentiment.polarity)
if naive > pattern:
return blob_nb
elif pattern > naive:
return blob_pa
else:
return blob_pa
def TestPA_Pretrained(testdatas,text_type="text"):
tb = Blobber(analyzer=PatternAnalyzer())
class_df=testdatas
sent=[]
polarity=[]
subjectivity=[]
for i in tqdm(range(len(class_df))):
#print(i)
blob_object=tb(class_df.loc[i,text_type])
analysis = blob_object.sentiment
if analysis[0]>=0:
sent_loc="pos"
elif analysis[0]<0:
sent_loc="neg"
sent.append(sent_loc)
polarity.append(analysis[0])
subjectivity.append(analysis[1])
class_df["sentiment"]=sent
class_df["polarity"]=polarity
class_df["subjectivity"]=subjectivity
class_df['Classif_Pattern']="None"
class_df.loc[(class_df["mentions_dem"].apply(len)>2)&(class_df["sentiment"]=="pos"),'Classif_Pattern']="Democrat"
class_df.loc[(class_df["mentions_dem"].apply(len)>2)&(class_df["sentiment"]=="neg"),'Classif_Pattern']="Republican"
class_df.loc[(class_df["mentions_rep"].apply(len)>2)&(class_df["sentiment"]=="pos"),'Classif_Pattern']="Republican"
class_df.loc[(class_df["mentions_rep"].apply(len)>2)&(class_df["sentiment"]=="neg"),'Classif_Pattern']="Democrat"
class_df.loc[(class_df["mentions_dem"].apply(len)==2)&(class_df["mentions_rep"].apply(len)==2),'Classif_Pattern']="Und"
class_df.loc[(class_df["mentions_dem"].apply(len)>2)&(class_df["mentions_rep"].apply(len)>2),'Classif_Pattern']="Und"
class_df['Classif_Pattern_Accurate']=(class_df['Classif_Pattern']==class_df['label'])
class_df.loc[class_df['Classif_Pattern']=="Und","Classif_Pattern_Accurate"]="Und"
cross_accuracy_stats=pd.crosstab(class_df["label"],class_df["Classif_Pattern_Accurate"])
cross_accuracy_stats["accuracy"]=cross_accuracy_stats[True]/(cross_accuracy_stats[True]+cross_accuracy_stats[False])
acc=class_df['Classif_Pattern_Accurate'].value_counts()
print("********** Number of undefined "+str(acc["Und"])+" ************")
acc=acc.drop("Und")
return(acc,cross_accuracy_stats)
def polarity(self):
"""Return the polarity score as a float within the range [-1.0, 1.0]
:rtype: float
"""
return PatternAnalyzer().analyze(self.raw)[0]
def pa_mapper(text):
blob = TextBlob(text, analyzer=PatternAnalyzer())
if blob.sentiment[0] < 0:
return 0
elif blob.sentiment[0] > 0:
return 1
import csv
import pickle
import re
from textblob.sentiments import NaiveBayesAnalyzer
from textblob.sentiments import PatternAnalyzer
from pprint import pprint
# import keras
pattern_analyzer = PatternAnalyzer()
bayes_analyzer = NaiveBayesAnalyzer()
link_pattern = r"((http|ftp|https):\/\/)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)"
tag_pattern = r"@[\w]*"
emoji_pattern = r"(\:\w+\:|\<[\/\\]?3|[\(\)\\\D|\*\$][\-\^]?[\:\;\=]|[\:\;\=B8][\-\^]?[3DOPp\@\$\*\\\)\(\/\|])(?=\s|[\!\.\?]|$)"
non_char_pattern = r"^[\W]+"
white_spaces_pattern = r"^[\s]|$[\s]"
def csv_to_pickle(input_path, output_path):
try:
with open(input_path, 'r') as f:
reader = csv.reader(f)
lista = list(reader)
tags = [element[0] for element in lista]
tweets = [element[1:] for element in lista]
with open(output_path, 'wb') as fb:
pickle.dump((tweets, tags), fb)
return True
finally:
return False
class BaseBlob(StringlikeMixin, BlobComparableMixin):
"""An abstract base class that all textblob classes will inherit from.
Includes words, POS tag, NP, and word count properties. Also includes
basic dunder and string methods for making objects like Python strings.
:param text: A string.
:param tokenizer: (optional) A tokenizer instance. If ``None``,
defaults to :class:`WordTokenizer() <textblob.tokenizers.WordTokenizer>`.
:param np_extractor: (optional) An NPExtractor instance. If ``None``,
defaults to :class:`FastNPExtractor() <textblob.en.np_extractors.FastNPExtractor>`.
:param pos_tagger: (optional) A Tagger instance. If ``None``,
defaults to :class:`NLTKTagger <textblob.en.taggers.NLTKTagger>`.
:param analyzer: (optional) A sentiment analyzer. If ``None``,
defaults to :class:`PatternAnalyzer <textblob.en.sentiments.PatternAnalyzer>`.
:param parser: A parser. If ``None``, defaults to
:class:`PatternParser <textblob.en.parsers.PatternParser>`.
:param classifier: A classifier.
.. versionchanged:: 0.6.0
``clean_html`` parameter deprecated, as it was in NLTK.
"""
np_extractor = FastNPExtractor()
pos_tagger = NLTKTagger()
tokenizer = WordTokenizer()
translator = Translator()
analyzer = PatternAnalyzer()
parser = PatternParser()
def __init__(self,
text,
tokenizer=None,
pos_tagger=None,
np_extractor=None,
analyzer=None,
parser=None,
classifier=None,
clean_html=False):
if not isinstance(text, basestring):
raise TypeError('The `text` argument passed to `__init__(text)` '
'must be a string, not {0}'.format(type(text)))
if clean_html:
raise NotImplementedError(
"clean_html has been deprecated. "
"To remove HTML markup, use BeautifulSoup's "
"get_text() function")
self.raw = self.string = text
self.stripped = lowerstrip(self.raw, all=True)
_initialize_models(self, tokenizer, pos_tagger, np_extractor, analyzer,
parser, classifier)
@cached_property
def words(self):
"""Return a list of word tokens. This excludes punctuation characters.
If you want to include punctuation characters, access the ``tokens``
property.
:returns: A :class:`WordList <WordList>` of word tokens.
"""
return WordList(word_tokenize(self.raw, include_punc=False))
@cached_property
def tokens(self):
"""Return a list of tokens, using this blob's tokenizer object
(defaults to :class:`WordTokenizer <textblob.tokenizers.WordTokenizer>`).
"""
return WordList(self.tokenizer.tokenize(self.raw))
def tokenize(self, tokenizer=None):
"""Return a list of tokens, using ``tokenizer``.
:param tokenizer: (optional) A tokenizer object. If None, defaults to
this blob's default tokenizer.
"""
t = tokenizer if tokenizer is not None else self.tokenizer
return WordList(t.tokenize(self.raw))
def parse(self, parser=None):
"""Parse the text.
:param parser: (optional) A parser instance. If ``None``, defaults to
this blob's default parser.
.. versionadded:: 0.6.0
"""
p = parser if parser is not None else self.parser
return p.parse(self.raw)
def classify(self):
"""Classify the blob using the blob's ``classifier``."""
if self.classifier is None:
raise NameError("This blob has no classifier. Train one first!")
return self.classifier.classify(self.raw)
@cached_property
def sentiment(self):
"""Return a tuple of form (polarity, subjectivity ) where polarity
is a float within the range [-1.0, 1.0] and subjectivity is a float
within the range [0.0, 1.0] where 0.0 is very objective and 1.0 is
very subjective.
:rtype: namedtuple of the form ``Sentiment(polarity, subjectivity)``
"""
return self.analyzer.analyze(self.raw)
@cached_property
def sentiment_assessments(self):
"""Return a tuple of form (polarity, subjectivity, assessments ) where
polarity is a float within the range [-1.0, 1.0], subjectivity is a
float within the range [0.0, 1.0] where 0.0 is very objective and 1.0
is very subjective, and assessments is a list of polarity and
subjectivity scores for the assessed tokens.
:rtype: namedtuple of the form ``Sentiment(polarity, subjectivity,
assessments)``
"""
return self.analyzer.analyze(self.raw, keep_assessments=True)
@cached_property
def polarity(self):
"""Return the polarity score as a float within the range [-1.0, 1.0]
:rtype: float
"""
return PatternAnalyzer().analyze(self.raw)[0]
@cached_property
def subjectivity(self):
"""Return the subjectivity score as a float within the range [0.0, 1.0]
where 0.0 is very objective and 1.0 is very subjective.
:rtype: float
"""
return PatternAnalyzer().analyze(self.raw)[1]
@cached_property
def noun_phrases(self):
"""Returns a list of noun phrases for this blob."""
return WordList([
phrase.strip().lower()
for phrase in self.np_extractor.extract(self.raw)
if len(phrase) > 1
])
@cached_property
def pos_tags(self):
"""Returns an list of tuples of the form (word, POS tag).
Example:
::
[('At', 'IN'), ('eight', 'CD'), ("o'clock", 'JJ'), ('on', 'IN'),
('Thursday', 'NNP'), ('morning', 'NN')]
:rtype: list of tuples
"""
if isinstance(self, TextBlob):
return [
val for sublist in [s.pos_tags for s in self.sentences]
for val in sublist
]
else:
return [(Word(word, pos_tag=t), unicode(t))
for word, t in self.pos_tagger.tag(self)
if not PUNCTUATION_REGEX.match(unicode(t))]
tags = pos_tags
@cached_property
def word_counts(self):
"""Dictionary of word frequencies in this text.
"""
counts = defaultdict(int)
stripped_words = [lowerstrip(word) for word in self.words]
for word in stripped_words:
counts[word] += 1
return counts
@cached_property
def np_counts(self):
"""Dictionary of noun phrase frequencies in this text.
"""
counts = defaultdict(int)
for phrase in self.noun_phrases:
counts[phrase] += 1
return counts
def ngrams(self, n=3):
"""Return a list of n-grams (tuples of n successive words) for this
blob.
:rtype: List of :class:`WordLists <WordList>`
"""
if n <= 0:
return []
grams = [
WordList(self.words[i:i + n])
for i in range(len(self.words) - n + 1)
]
return grams
def translate(self, from_lang="auto", to="en"):
"""Translate the blob to another language.
Uses the Google Translate API. Returns a new TextBlob.
Requires an internet connection.
Usage:
::
>>> b = TextBlob("Simple is better than complex")
>>> b.translate(to="es")
TextBlob('Lo simple es mejor que complejo')
Language code reference:
https://developers.google.com/translate/v2/using_rest#language-params
.. versionadded:: 0.5.0.
:param str from_lang: Language to translate from. If ``None``, will attempt
to detect the language.
:param str to: Language to translate to.
:rtype: :class:`BaseBlob <BaseBlob>`
"""
return self.__class__(
self.translator.translate(self.raw,
from_lang=from_lang,
to_lang=to))
def detect_language(self):
"""Detect the blob's language using the Google Translate API.
Requires an internet connection.
Usage:
::
>>> b = TextBlob("bonjour")
>>> b.detect_language()
u'fr'
Language code reference:
https://developers.google.com/translate/v2/using_rest#language-params
.. versionadded:: 0.5.0
:rtype: str
"""
return self.translator.detect(self.raw)
def correct(self):
"""Attempt to correct the spelling of a blob.
.. versionadded:: 0.6.0
:rtype: :class:`BaseBlob <BaseBlob>`
"""
# regex matches: word or punctuation or whitespace
tokens = nltk.tokenize.regexp_tokenize(self.raw, "\w+|[^\w\s]|\s")
corrected = (Word(w).correct() for w in tokens)
ret = ''.join(corrected)
return self.__class__(ret)
def _cmpkey(self):
"""Key used by ComparableMixin to implement all rich comparison
operators.
"""
return self.raw
def _strkey(self):
"""Key used by StringlikeMixin to implement string methods."""
return self.raw
def __hash__(self):
return hash(self._cmpkey())
def __add__(self, other):
'''Concatenates two text objects the same way Python strings are
concatenated.
Arguments:
- `other`: a string or a text object
'''
if isinstance(other, basestring):
return self.__class__(self.raw + other)
elif isinstance(other, BaseBlob):
return self.__class__(self.raw + other.raw)
else:
raise TypeError(
'Operands must be either strings or {0} objects'.format(
self.__class__.__name__))
def split(self, sep=None, maxsplit=sys.maxsize):
"""Behaves like the built-in str.split() except returns a
WordList.
:rtype: :class:`WordList <WordList>`
"""
return WordList(self._strkey().split(sep, maxsplit))
def sentiment(tweet):
#return textblob.TextBlob(tweet).sentiment.polarity
sentiment_analyzer = PatternAnalyzer()
return sentiment_analyzer.analyze(tweet).polarity
def test_can_get_subjectivity_and_polarity_with_different_analyzer(self):
blob = tb.TextBlob("I love this car.", analyzer=NaiveBayesAnalyzer())
pattern = PatternAnalyzer()
assert_equal(blob.polarity, pattern.analyze(str(blob))[0])
assert_equal(blob.subjectivity, pattern.analyze(str(blob))[1])
# describes it--"assimilating flesh on contact.
# Snide comparisons to gelatin be damned, it's a concept with the most
# devastating of potential consequences, not unlike the grey goo scenario
# proposed by technological theorists fearful of
# artificial intelligence run rampant.
# '''
#text1 = "Results were found in agreement with reports by Dantes et al 2019." # NEGATIVE
#text1 = "I am very happy with her research findings." # POSITIVE
#text1 = "We have confirmed previous observation by Andutta et al 2014." # NEGATIVE
text1 = "We have confirmed previous observation by Andutta et al 2014. We disagree with Andutta research findings, and wish to highlight here. We believe these findings to be wrong." # NEGATIVE
print("###################################################################")
print("############ MODEL 1 ###############")
model1 = TextBlob(text1,
analyzer=PatternAnalyzer()) # USE THIS FOR PatternAnalyzer
#model1 = TextBlob(text1) # USE THIS FOR PatternAnalyzer (BY DEFAULT)
# We disagree with Fernando Andutta's research findings, and wish to highlight here.
# Polarity = '0.0'
# Subjectivity = '0.0'
xx = 0
for sentence in model1.sentences:
xx = xx + 1
print("###################################")
print("SENTENCE = '%s' " % xx)
print(sentence)
#print("sentence = " + sentence)
#print(sentence.sentiment.polarity) # it shows only polarity
#print(sentence.sentiment) # it shows polarity and subjectivity
res1 = sentence.sentiment
def get_subjectivity(line: str) -> float:
return PatternAnalyzer().analyze(line)[1]
2) >= 0.37:
search_query_sentiment = 0
elif round(prob_dist.prob("pos"), 2) < 0.37:
search_query_sentiment = -1
print '\nNaive Bayes Classification: ', search_query_sentiment, ', distribution: ', prob_dist.prob(
"pos") # cl.classify("This is an amazing library!")
# Naive Bayes Analyzer -- use this if data set for Classifier is not proper.
blob = TextBlob(raw_query, analyzer=NaiveBayesAnalyzer())
search_query_sentiment = blob.sentiment
print '\nNaive Bayes Analyzer: ', search_query_sentiment
# Pattern Analyzer -- use this if data set for Classifier is not proper.
blob = TextBlob(raw_query, analyzer=PatternAnalyzer())
search_query_sentiment = blob.sentiment
print '\nPattern Analyzer: ', search_query_sentiment
'''
request = asynchronous(Google().search, 'I eat pizza with a fork.', timeout=4)
while not request.done:
time.sleep(0.1)
print "busy ..."
print request.value
engine = Google(license=None, throttle=0.5, language=None)
for i in range(1,2):
for result in engine.search(search_query, type=SEARCH, start=i):
print repr(plaintext(result.text))
'unicode_code', 'pos_score', 'neg_score', 'obj_score'
]]
emot_sent = emot_sent.set_index('unicode_code').to_dict()
emot_sent = [
emot_sent['pos_score'], emot_sent['neg_score'], emot_sent['obj_score']
]
for key in emot_sent[0]:
emoticons_sent_dict[key] = tuple(score[key] for score in emot_sent)
return emoticons_sent_dict
_EMOTICONS_DICT = _get_emoticons_dict()
_SENTIMENT_ANALYZER = PatternAnalyzer()
def get_tweet_score(tweet=None):
"""Returns sentiment score of tweet by averaging the text score as decided
by the PatternAnalyzer with the average score of the detected emojis
"""
text_score = [0, 0] # pos_score, neg_score
emot_score = [0, 0] # pos_score, neg_score
emot_sent_scores = []
# find sentiment for text using pattern analyzer from textblob
polarity = _SENTIMENT_ANALYZER.analyze(tweet).polarity
if polarity > 0:
text_score[0] = polarity
else:
req1 = yt.commentThreads().list(part='snippet',
moderationStatus="published",
order="relevance",
textFormat="plainText",
videoId=mat2)
res = req1.execute()
# print(res)
bags = []
for item in res['items']:
comment = item['snippet']['topLevelComment']['snippet']['textDisplay']
# print(comment)
bags.append(comment)
comments = ''
sentences = comments.join(bags)
sentiment_analyzer = PatternAnalyzer()
# print(sentences)
# print(sentiment_analyzer.analyze(sentences))
vv = sentiment_analyzer.analyze(sentences)
vv1 = vv.polarity
# print(vv1)
if vv1 > 0.5:
print("Positive")
# print(bags)
# if bags == item.values():
# # print(bags['textDisplay'])
# comments = ''
# sentences = comments.join(bags)
# print(sentences)
# sentiment_analyzer = PatternAnalyzer()
# tweet['month'] = time_seq[1]
# tweet['date'] = int(time_seq[2])
# t = time_seq[3].split(":")
#
# tweet['hour'] = int(t[0])
# tweet['minute'] = int(t[1])
# tweet['second'] = int(t[2])
# tweet['plus0000'] = time_seq[4]
# tweet['year'] = int(time_seq[5])
print tweet
#break
#print tweet
#print type(tweet)
#tb = TextBlob(tweet_text)
#print type(tb)
pt_tb = TextBlob(tweet_text, analyzer=PatternAnalyzer())
nb_tb = TextBlob(tweet_text, analyzer=NaiveBayesAnalyzer())
extractor = ConllExtractor()
np_tb = TextBlob("Python is a high-level programming language, high-level programming language.", np_extractor=extractor)
noun_dic = {}
for ele in np_tb:
if ele not in noun_dic:
noun_dic[ele] = 1
else:
noun_dic[ele] += 1
#print str(np_tb.noun_phrases)
print str(pt_tb.sentiment)
# #print tweet['text']
# print str(nb_tb.sentiment.classification)
# print str(nb_tb.sentiment.p_pos)
self.ranges = ranges
self.ax = axes[0]
def plot(self, data, *args, **kw):
sdata = _scale_data(data, self.ranges)
self.ax.plot(self.angle, np.r_[sdata, sdata[0]], *args, **kw)
def fill(self, data, *args, **kw):
sdata = _scale_data(data, self.ranges)
self.ax.fill(self.angle, np.r_[sdata, sdata[0]], *args, **kw)
if __name__ == "__main__":
start = timeit.default_timer()
# init the analyzers
analyzerBayes = NaiveBayesAnalyzer()
analyzerPattern = PatternAnalyzer()
#training first
resultBayes = analyzerBayes.analyze("train this")
resultPattern = analyzerPattern.analyze("train this")
sc = SparkContext(appName="MovieSentiment")
# map reduce
lines = sc.textFile("movieData.txt")
posNneg = lines.map(sentimentAnalysis) \
.reduceByKey(lambda a, b: (a[0] + b[0], a[1] + b[1]))
output = posNneg.collect()
class Blobber(object):
"""A factory for TextBlobs that all share the same tagger,
tokenizer, parser, classifier, and np_extractor.
Usage:
>>> from textblob import Blobber
>>> from textblob.taggers import NLTKTagger
>>> from textblob.tokenizers import SentenceTokenizer
>>> tb = Blobber(pos_tagger=NLTKTagger(), tokenizer=SentenceTokenizer())
>>> blob1 = tb("This is one blob.")
>>> blob2 = tb("This blob has the same tagger and tokenizer.")
>>> blob1.pos_tagger is blob2.pos_tagger
True
:param tokenizer: (optional) A tokenizer instance. If ``None``,
defaults to :class:`WordTokenizer() <textblob.tokenizers.WordTokenizer>`.
:param np_extractor: (optional) An NPExtractor instance. If ``None``,
defaults to :class:`FastNPExtractor() <textblob.en.np_extractors.FastNPExtractor>`.
:param pos_tagger: (optional) A Tagger instance. If ``None``,
defaults to :class:`NLTKTagger <textblob.en.taggers.NLTKTagger>`.
:param analyzer: (optional) A sentiment analyzer. If ``None``,
defaults to :class:`PatternAnalyzer <textblob.en.sentiments.PatternAnalyzer>`.
:param parser: A parser. If ``None``, defaults to
:class:`PatternParser <textblob.en.parsers.PatternParser>`.
:param classifier: A classifier.
.. versionadded:: 0.4.0
"""
np_extractor = FastNPExtractor()
pos_tagger = NLTKTagger()
tokenizer = WordTokenizer()
analyzer = PatternAnalyzer()
parser = PatternParser()
def __init__(self,
tokenizer=None,
pos_tagger=None,
np_extractor=None,
analyzer=None,
parser=None,
classifier=None):
_initialize_models(self, tokenizer, pos_tagger, np_extractor, analyzer,
parser, classifier)
def __call__(self, text):
"""Return a new TextBlob object with this Blobber's ``np_extractor``,
``pos_tagger``, ``tokenizer``, ``analyzer``, and ``classifier``.
:returns: A new :class:`TextBlob <TextBlob>`.
"""
return TextBlob(text,
tokenizer=self.tokenizer,
pos_tagger=self.pos_tagger,
np_extractor=self.np_extractor,
analyzer=self.analyzer,
parser=self.parser,
classifier=self.classifier)
def __repr__(self):
classifier_name = self.classifier.__class__.__name__ + "()" if self.classifier else "None"
return ("Blobber(tokenizer={0}(), pos_tagger={1}(), "
"np_extractor={2}(), analyzer={3}(), parser={4}(), classifier={5})")\
.format(self.tokenizer.__class__.__name__,
self.pos_tagger.__class__.__name__,
self.np_extractor.__class__.__name__,
self.analyzer.__class__.__name__,
self.parser.__class__.__name__,
classifier_name)
__str__ = __repr__
def __init__(self):
self.analyzer = Blobber(analyzer=PatternAnalyzer())
self.sentimentList = []
from textblob.sentiments import NaiveBayesAnalyzer
from textblob.sentiments import PatternAnalyzer
import textblob as tb
print(tb.__version__)
# Read all sentences in the file into one string.
# The `replace(..)` is used to combine sentences that are split into two
# lines in the file back into one line.
# TextBlob will parse into sentences, using . ! ? etc. to separate them.
with open('./assignment3/homework 3 dataset.txt', 'r') as file:
text = file.read().replace('\n', ' ')
blob = TextBlob(text)
print('Naive Bayes Analyzer')
tb = Blobber(analyzer=NaiveBayesAnalyzer())
for sentence in blob.sentences:
sentiment = tb(str(sentence)).sentiment
print('{} +{:.2f} -{:.2f} {}'.format(sentiment.classification,
sentiment.p_pos, sentiment.p_neg,
sentence))
print('\n\nPattern Analyzer')
tb = Blobber(analyzer=PatternAnalyzer())
for sentence in blob.sentences:
sentiment = tb(str(sentence)).sentiment
classification = 'pos' if sentiment.polarity >= 0 else 'neg'
print('{} {:+.2f} {:.2f} {}'.format(classification, sentiment.polarity,
sentiment.subjectivity, sentence))
def test_can_get_subjectivity_and_polarity_with_different_analyzer(self):
blob = tb.TextBlob("I love this car.", analyzer=NaiveBayesAnalyzer())
pattern = PatternAnalyzer()
assert_equal(blob.polarity, pattern.analyze(str(blob))[0])
assert_equal(blob.subjectivity, pattern.analyze(str(blob))[1])
def setUp(self):
self.analyzer = PatternAnalyzer()
def setUp(self):
self.analyzer = PatternAnalyzer()
