class MyListener(StreamListener):
def __init__(self):
self.dp = DataProcessor()
self.conf = Config()
self.auth = tweepy.OAuthHandler(self.conf.consumer_key, self.conf.consumer_secret)
self.auth.set_access_token(self.conf.access_token, self.conf.access_token_secret)
self.conn = Connector()
def on_data(self, raw_data):
try:
self.conn.insert_raw_twitter(raw_data, self.conn.twitterdb_demo_pub)
target_info = self.dp.get_geo_twi_target_info(raw_data)
self.conn.insert_raw_twitter_result(target_info, self.conn.twitterdb_demo_results_pub)
if target_info:
self.conn.insert_raw_twitter_result(target_info, self.conn.twitterdb_demo_results_pub)
return True
except BaseException as e:
print("Error on_data:%s" % str(e))
return True
def on_error(self, status_code):
if status_code == 420:
print("ERROR: Rate limit reached")
print(status_code)
return True
def on_timeout(self):
print("ERROR: Timeout...")
return True # Don't kill the stream
# Created by Yuan Wang (yuanw8)
# Cloud and Cluster Computing Assignment 2
# Semester 1, 2018
from Connector import Connector
from DataProcessor import DataProcessor
dp = DataProcessor()
with open('twitterdocs504.json', 'r') as tw:
with open('504target.json', 'w') as w:
for line in tw:
target_info = dp.get_geo_twi_target_info(line)
if target_info is not None:
w.write(target_info)
w.write('\n')
print(target_info)
