def searchFuture(search, api, bf, stopEvent, barrier):
## Main loop. Start collection of tweets
query_num = 1
lastid = 0
while True:
try:
if stopEvent.isSet():
logger.info("Stoping search")
break
## Get tweets from API
tweets = api.getSearch(q = search.query, since_id = lastid,
count = 100)
## For each tweets store in db. Find the largest
for tweet in tweets:
## This method will save tweet in db and associated with
## current search. If the tweets is already stored, it will
## associated with current search. If the tweets is already
## associated with this search, it will do nothing
if not stopEvent.isSet():
bf.insertTweetSearch(tweet, search.id)
## find the largest
if lastid < tweet["id"]:
lastid = tweet["id"]
logger.debug("Last tweet.id = " + str(lastid) + ", query num = "
+ str(query_num))
query_num += 1
except Exception as err:
logger.error(str(err))
logger.debug("Last tweet id read = " + str(lastid))
# Stop timer thread in buffer
bf.stopTimer()
barrier.wait()
logger.info("Searching stoped")
def searchHistorical(search, api, bf, stopEvent, barrier):
## Main loop. Start collection of tweets
bf.startTimer()
query_num = 1
lastid = 0
first = True
while True:
try:
if stopEvent.isSet():
logger.info("StopEvent occurs!")
break
## Get tweets from API
tweets = api.getSearch(q = search.query, max_id = lastid,
count = 100)
## For each tweets store in db. Find the least ID
for tweet in tweets:
## This method will save tweet in db and associated with
## current search. If the tweets is already stored, it will
## associated with current search. If the tweets is already
## associated with this search, it will do nothing
if not stopEvent.isSet():
bf.insertTweetSearch(tweet, search.id)
## find the least
if lastid > tweet["id"]:
lastid = tweet["id"]
elif first:
lastid = tweet["id"]
first = False
logger.debug("Last id = " + str(lastid) + " query = " + str(query_num))
query_num = query_num + 1
except Exception as err:
logger.error(str(err))
logger.debug("Last tweet id read = " + str(lastid))
# Stop timer thread in buffer)
logger.debug("Stoping timer")
bf.stopTimer()
logger.debug("Waiting in barrier: " + str(barrier))
barrier.wait()
logger.info("Searching stoped")
