def sentiment_analysis(directory):
form = SearchDFForm()
sa = SentimentAnalysis(directory)
if form.validate_on_submit():
sentiment_df = sa.makeSentDF(form.input.data)
else:
sentiment_df = sa.makeSentDF()
return render_template('sentiment_analysis.html', sentiment_df = sentiment_df, form=form)
def crawl(self, keyword):
""" Crawl individual keyword """
api = self.api
max_id = -1
since_id = None if keyword["since_id"] == -1 else keyword["since_id"]
tweets = api.search(q=keyword["keyword"],
include_entities=True,
lang="en",
count=MAX_COUNT,
since_id=since_id,
geocode=settings.GEO_CODE)
# For the first time, run the search using since_id from database
# update since_id in database with the first tweet id that we get
if len(tweets) > 0:
self.keyword.update_since_id(keyword["id"], tweets[0].id)
for tweet in tweets:
# If we have reach the since_id, break from loop
if tweet.id == since_id:
break
# Update sentiment score
tweet._json["sentiment"] = SentimentAnalysis.get_sentiment(
tweet_text=tweet.text)
self.tw_store.save_tweet(tweet._json)
max_id = tweets[-1].id
self.test_rate_limit(api)
else:
return
while True:
tweets = api.search(q=keyword["keyword"],
include_entities=True,
lang="en",
count=MAX_COUNT,
max_id=max_id - 1,
geocode=settings.GEO_CODE)
# Check if the api return value, otherwise break from loop
# continue crawl next keyword
if len(tweets) > 0:
for tweet in tweets:
# If we have reach the since_id, break from loop
if tweet.id == since_id:
break
# Update sentiment score
tweet._json["sentiment"] = SentimentAnalysis.get_sentiment(
tweet_text=tweet.text)
self.tw_store.save_tweet(tweet._json)
max_id = tweets[-1].id
else:
break
self.test_rate_limit(api)
def on_status(self, status):
""" Handle logic when the data coming """
try:
tweet = json.loads(status)
# Update sentiment score
tweet["sentiment"] = SentimentAnalysis.get_sentiment(tweet_text=tweet["text"])
self.tw_store.save_tweet(tweet)
except Exception as e:
log.error(e)
def GetSentimentAnalysis():
Comment = request.values['Comment']
data = None
if Comment != '':
data = SentimentAnalysis.predict(Comment)
# 评论分析
return json_util.dumps({"result": {
"data": data,
}})
def SentimentAnalysisAnalysis():
cleanMemory(2)
if 'reviewId' in request.values:
reviewId = int(request.values['reviewId'])
result = DB.DATABASE['RatingProcessed'].find({
'reviewId': reviewId
}).sort('created_date', pymongo.DESCENDING).limit(1)[0]
AnalysisFrom = SentimentAnalysisForm(result['comment'])
else:
comment = SentimentAnalysis.get_test_comment()
AnalysisFrom = SentimentAnalysisForm(comment)
return render_template("SentimentAnalysisAnalysis.html", form=AnalysisFrom)
from app.logger import LOGGER as log
import settings
ALL_DOCS_VIEW = '_all_docs'
try:
log.info("START db updater script")
log.info("-----------------------")
server = couchdb.Server(url=settings.COUCHDB_SERVER)
db = server[settings.COUCHDB_DB]
info = db.info()
doc_count = info["doc_count"]
num_per_request = 10000
iteration = math.ceil(doc_count / num_per_request)
for i in range(iteration):
log.info('Run %d iteration' % i)
for row in db.view(ALL_DOCS_VIEW,
limit=num_per_request,
skip=i * num_per_request):
data = db.get(row.id)
data["sentiment"] = SentimentAnalysis.get_sentiment(data["text"])
db.save(data)
log.info('%d iteration success')
log.info("FINISH db updater script")
except Exception as e:
log.error(e)
def project02clean():
SentimentAnalysis.clean()
def project02init():
SentimentAnalysis.Init()