def fetchsamples(needed_sent_val=None, max_iters=1000):
word_list = sh.english_word_list()
afinn_dict = cs.load_afinn_dictionary('text_sentiment/AFINN-111.txt')
huliu_dict = \
cs.load_huliu_dict('text_sentiment/hu_liu/opinion-lexicon-English/')
url = "https://stream.twitter.com/1/statuses/sample.json"
parameters = []
response = ts.twitterreq(url, "GET", parameters)
num_iters = 0
for line in response:
if num_iters > max_iters:
break
if isinstance(line, bytes):
line = line.decode('utf-8')
# decode if not error message; else wait 1 sec to avoid rate limits
try:
tweet = json.loads(line.strip())
except:
time.sleep(1)
print('waiting....')
continue
# stop processing if tweet doesn't meet basic criteria
if not prt.decide_to_include_tweet(tweet):
continue
if not prt.image_is_original(tweet):
continue
# Calculate tweet sentiment
tweet_txt = tweet['text']
cleaned_text = sh.parse_sentence(tweet_txt, word_list)
vader_sent = cs.calculate_vader(cleaned_text)
afinn_sent = cs.calculate_simple_sentiment(cleaned_text, afinn_dict)
hului_sent = cs.calculate_simple_sentiment(cleaned_text, huliu_dict)
consistent = vader_sent == afinn_sent == hului_sent
if not consistent:
continue
if needed_sent_val and (vader_sent != needed_sent_val):
continue
# retrieve and hash image
image_url = tweet['extended_entities']['media'][0]['media_url']
img = fetch_image(image_url)
image_hash = dedupe.calculate_image_hash(img)
# Ensure not an exact duplicate
match = dedupe.find_matching_hash(image_hash, tweet['id'])
if match:
try:
add_dupe_to_db(tweet, match, vader_sent,
image_hash, cleaned_text)
except Exception as err:
print(err)
continue
# Save image and write info to db
try:
add_new_record_to_db(tweet, vader_sent, image_hash, cleaned_text)
img.save(IMAGE_SAVE_PATH + tweet['id_str'] + '.jpg')
except Exception as err:
print(err)
continue
num_iters += 1
return
"""
Combines all the sentiment dictionaries into one file to use in parsing
hashtags (because other words don't matter)
"""
from Python_code.text_sentiment import compare_sentiments as cs
# words are keys
afinn_dict = cs.load_afinn_dictionary("AFINN-111.txt")
hului_dict = cs.load_huliu_dict("hu_liu/opinion-lexicon-English/")
vader_dict = cs.vader.make_lex_dict("vader/vader_sentiment_lexicon.txt")
sentiment_words = []
for key in afinn_dict:
sentiment_words.append(key)
for key in hului_dict:
if key not in sentiment_words:
sentiment_words.append(key)
for key in vader_dict:
if key not in sentiment_words:
sentiment_words.append(key)
sentiment_words.sort()
with open("hashtag_dict.txt", "w") as file:
for word in sentiment_words:
file.write(word + "\n")
