def __init__(self, cities):
super(MeetupStatisticsFetcher, self).__init__()
DBManager.init(1)
self.cursor = DBManager.get_cursor()
self.cities = cities
self.logger = utilities.logger_with_name("StatisticsFetcher")
self.query, self.title = "", ""
self.results = dict()
def run(self):
DBManager.get_connection().jconn.setAutoCommit(False)
while self.should_run:
try:
jobs = self.job_queue.get(True, 5)
except:
self.logger.debug("No item in the queue. Waiting for something to do...")
continue
self.logger.info("Executing jobs on the database (approx. " + str(self.job_queue.qsize()) + " remaining)")
for job in jobs:
match = re.search('UPSERT (.*) VALUES .*', job.query_string)
self.logger.info(" Job table: " + match.group(1) if match else "finishing group...")
self.logger.info(" Job size: " + str(get_size(job.values_list) / 1024.0) + " KB")
utilities.try_execution(
lambda: DBManager.get_cursor().executemany(job.query_string, job.values_list),
"Executing query: " + job.query_string + ", Last tuple: " + str(job.values_list[-1]),
self.logger)
DBManager.get_connection().commit()
self.logger.info("Finished executing jobs on the database")
self.logger.info("PersisterThread stopped")
