def scrape_site(self):
marketwatchScraper = MarketwatchNewsScraper()
all_articles = []
sentimentAnalyzer = SentimentAnalyzer()
while True:
articles = marketwatchScraper.scrape()
for article in articles:
if article not in all_articles:
headline = article.headline
print('headline:', headline)
print('url', article.url)
all_articles.append(article)
# if len(headline.split()) >= 20:
# sentimentAnalyzer.google_analyze(headline)
# else:
sentimentAnalyzer.analyze(headline)
# for headline in newsHeadlines:
# #print('headline: ', headline)
# sentimentAnalyzer.google_analyze(headline)
# # print('headline: ', newsHeadlines[0])
# # sentimentAnalyzer.google_analyze(newsHeadlines[0])
logger.info("Will get news headlines again in %s sec..." %
self.frequency)
print()
print()
print()
print()
time.sleep(self.frequency)
aspect_dict[w] = defaultdict(int)
# Map each aspect to a dictionary holding the count of positive, neutral and negative reviews
for aspect in potentialAspects:
sentiment_dict = aspect_dict[aspect]
for sent, tokens in sentToWords.items():
# for each word check if that word is the desired aspect
# if so that means that the sentence relates to this aspect
# we then check the sentiment of the sentence and add it to
# a positive or negative count
for word in tokens:
if word == aspect:
res = sentimentAnalyzer.analyze(sent)
# print("%s - %s" % (aspect, sent))
# positive sentiment
if len(res) > 1:
sentiment_dict[NEUTRAL_KEY] += 1
elif res[0] == 1:
sentiment_dict[POSITIVE_KEY] += 1
# negative sentiment
elif res[0] == 0:
sentiment_dict[NEGATIVE_KEY] += 1
# this sentence is done being considered for this aspect once a match is found
break
aspect_dict[aspect] = sentiment_dict
