class MyListener(StreamListener): def __init__(self): self.dp = DataProcessor() self.conf = Config() self.auth = tweepy.OAuthHandler(self.conf.consumer_key, self.conf.consumer_secret) self.auth.set_access_token(self.conf.access_token, self.conf.access_token_secret) self.conn = Connector() def on_data(self, raw_data): try: self.conn.insert_raw_twitter(raw_data, self.conn.twitterdb_demo_pub) target_info = self.dp.get_geo_twi_target_info(raw_data) self.conn.insert_raw_twitter_result(target_info, self.conn.twitterdb_demo_results_pub) if target_info: self.conn.insert_raw_twitter_result(target_info, self.conn.twitterdb_demo_results_pub) return True except BaseException as e: print("Error on_data:%s" % str(e)) return True def on_error(self, status_code): if status_code == 420: print("ERROR: Rate limit reached") print(status_code) return True def on_timeout(self): print("ERROR: Timeout...") return True # Don't kill the stream
# Created by Yuan Wang (yuanw8) # Cloud and Cluster Computing Assignment 2 # Semester 1, 2018 from Connector import Connector from DataProcessor import DataProcessor dp = DataProcessor() with open('twitterdocs504.json', 'r') as tw: with open('504target.json', 'w') as w: for line in tw: target_info = dp.get_geo_twi_target_info(line) if target_info is not None: w.write(target_info) w.write('\n') print(target_info)