def main(begin_post_id=0, batch_size=100):
while True:
current_chunk = get_chunk(begin_post_id, batch_size)
#print('Fetching sentiment score from {} to {}'.format(current_chunk[0].post_id, current_chunk[-1].post_id))
result = {}
for post in current_chunk:
post_id = post.post_id
content = post.content
if not content:
continue
try:
polarity = analyze_sentiment(content)
result[post_id] = {'post_id': post_id, 'polarity': polarity}
print(result[post_id])
except Exception:
print('{} fail to fetch sentiment score'.format(post_id))
with postgres_db.atomic():
for key, value in result.items():
query = Post.update(
vader_sentiment_score=value['polarity'], ).where(
Post.post_id == key)
query.execute()
#time.sleep(5) # take a break to prevent rate limit
begin_post_id = current_chunk[-1].post_id
def main(begin_post_id=0, batch_size=100):
# sets the decimal precision point to 3
getcontext().prec = 3
while True:
current_chunk = get_chunk(begin_post_id, batch_size)
print("Fetching sentiment score from {} to {}".format(
current_chunk[0].post_id, current_chunk[-1].post_id))
result = {}
for post in current_chunk:
post_id = post.post_id
content = post.content
if not content:
continue
# google score
try:
google_score, magnitude = analyze_sentiment_google(content)
vader_polarity = analyze_sentiment_vader(content)
textBlob_polarity = analyze_sentiment_textBlob(content)
score = (google_score + textBlob_polarity + vader_polarity) / 3
result[post_id] = {
"post_id": post_id,
"score": score,
"google_sentiment_score": google_score,
"google_sentiment_magnitude": magnitude,
"vader_sentiment_score": vader_polarity,
"textblob_sentiment_score": textBlob_polarity,
}
print(result[post_id])
except Exception:
print("{} fail to fetch sentiment score".format(post_id))
with postgres_db.atomic():
for key, value in result.items():
query = Post.update(
bro_sentiment=value["score"],
google_sentiment_score=value["google_sentiment_score"],
google_sentiment_magnitude=value[
"google_sentiment_magnitude"],
textblob_sentiment_score=value["textblob_sentiment_score"],
vader_sentiment_score=value["vader_sentiment_score"],
).where(Post.post_id == key)
query.execute()
time.sleep(5) # take a break to prevent rate limit
begin_post_id = current_chunk[-1].post_id
def main(begin_post_id=0, batch_size=100):
while True:
current_chunk = get_chunk(begin_post_id, batch_size)
print('Fetching Fetching verb and noun count from {} to {}'.format(current_chunk[0].post_id, current_chunk[-1].post_id))
result = {}
for post in current_chunk:
post_id = post.post_id
content = post.content
#print(content)
if not content:
continue
try:
pos_dict = analyze_pos(content)
#print(pos_dict['JJ'])
except Exception:
print('{} fail to fetch verb and noun count'.format(post_id))
result[post_id] = {'post_id': post_id, 'verbs': pos_dict['verbs'], 'nouns': pos_dict['nouns'], 'adjectives': pos_dict['adjectives']}
print(result[post_id])
with postgres_db.atomic():
for key, value in result.items():
try:
query = Post.update(
#verbs='{:6}'.format(value['verbs']),
verbs=value['verbs'],
nouns=value['nouns'],
adjectives=value['adjectives']
).where(Post.post_id == key)
query.execute()
print("successfully added to ")
print(result[post_id])
except Exception as e:
print(e)
print('failed to add pos tag to ')
print(post_id)
continue
time.sleep(5) # take a break to prevent rate limit
begin_post_id = current_chunk[-1].post_id
def get_chunk(begin_post_id=0, limit=500):
posts = Post.select(Post.post_id, Post.content).where(
Post.post_id > begin_post_id).order_by(Post.post_id).limit(limit)
return posts
def get_post(begin_post_id=0, limit=500):
posts = Post.select(Post.post_id, Post.content).where(
Post.post_id > begin_post_id).order_by(Post.post_id).limit(limit)
posts = ("SELECT post_id, content f")
return posts