class SListener(StreamListener): def __init__(self, api=None, fprefix="streamer", log_op=True): super(SListener, self).__init__(api) self.api = api self.totaltweets = 0 self.timenow = time.time() self.counter = 0 self.fprefix = fprefix self.log_op = log_op self.mongohandle = MongoDBHandler() # self.__setup_flume_handler() def __setup_flume_handler(self): fh = handler.FlumeHandler() logger = logging.getLogger("AnalyzerLog") logger.setLevel(logging.DEBUG) logger.addHandler(fh) self.flumelogger = logger def on_data(self, data): if "in_reply_to_status" in data: self.on_status(data) elif "delete" in data: delete = json.loads(data)["delete"]["status"] if self.on_delete(delete["id"], delete["user_id"]) is False: return False elif "limit" in data: if self.on_limit(json.loads(data)["limit"]["track"]) is False: return False elif "warning" in data: warning = json.loads(data)["warnings"] print warning["message"] return False def on_status(self, status): # self.flumelogger.info(status.strip("\n")) self.mongohandle.add_tweet(status.strip("\n"), dbcategory="stream") if self.log_op: print status self.counter += 1 if self.counter >= 50: self.totaltweets += self.counter print "Total time passed : " + str(time.time() - self.timenow) print "Total tweets collected : " + str(self.totaltweets) self.counter = 0 return def on_limit(self, track): sys.stderr.write(track + "\n") return def on_error(self, status_code): sys.stderr.write("Error: " + str(status_code) + "\n") return False def on_timeout(self): sys.stderr.write("Timeout, sleeping for 60 seconds...\n") time.sleep(60) return
def sentiment_level_collector(type = "positive",count = 20,topic = "movies"): search_capture = Searcher(consumer_key,consumer_secret,access_token,access_token_secret) db_ins = MongoDBHandler() if type == "positive": results = search_capture.api.search(":)",count = count,lang="en") elif type == "negative": results = search_capture.api.search(":(",count = count,lang="en") for res in results: if 'RT' not in res.text: db_ins.add_tweet({'id' : res.id, 'text' : res.text, 'val' : 1 if type == "positive" else -1}, dbcategory="sentiment_train")
def __init__(self, api=None, fprefix="streamer", log_op=True): super(SListener, self).__init__(api) self.api = api self.totaltweets = 0 self.timenow = time.time() self.counter = 0 self.fprefix = fprefix self.log_op = log_op self.mongohandle = MongoDBHandler()