def searchFuture(search, api, bf, stopEvent, barrier): ## Main loop. Start collection of tweets query_num = 1 lastid = 0 while True: try: if stopEvent.isSet(): logger.info("Stoping search") break ## Get tweets from API tweets = api.getSearch(q = search.query, since_id = lastid, count = 100) ## For each tweets store in db. Find the largest for tweet in tweets: ## This method will save tweet in db and associated with ## current search. If the tweets is already stored, it will ## associated with current search. If the tweets is already ## associated with this search, it will do nothing if not stopEvent.isSet(): bf.insertTweetSearch(tweet, search.id) ## find the largest if lastid < tweet["id"]: lastid = tweet["id"] logger.debug("Last tweet.id = " + str(lastid) + ", query num = " + str(query_num)) query_num += 1 except Exception as err: logger.error(str(err)) logger.debug("Last tweet id read = " + str(lastid)) # Stop timer thread in buffer bf.stopTimer() barrier.wait() logger.info("Searching stoped")
def searchHistorical(search, api, bf, stopEvent, barrier): ## Main loop. Start collection of tweets bf.startTimer() query_num = 1 lastid = 0 first = True while True: try: if stopEvent.isSet(): logger.info("StopEvent occurs!") break ## Get tweets from API tweets = api.getSearch(q = search.query, max_id = lastid, count = 100) ## For each tweets store in db. Find the least ID for tweet in tweets: ## This method will save tweet in db and associated with ## current search. If the tweets is already stored, it will ## associated with current search. If the tweets is already ## associated with this search, it will do nothing if not stopEvent.isSet(): bf.insertTweetSearch(tweet, search.id) ## find the least if lastid > tweet["id"]: lastid = tweet["id"] elif first: lastid = tweet["id"] first = False logger.debug("Last id = " + str(lastid) + " query = " + str(query_num)) query_num = query_num + 1 except Exception as err: logger.error(str(err)) logger.debug("Last tweet id read = " + str(lastid)) # Stop timer thread in buffer) logger.debug("Stoping timer") bf.stopTimer() logger.debug("Waiting in barrier: " + str(barrier)) barrier.wait() logger.info("Searching stoped")