def __init__(self, cities): super(MeetupStatisticsFetcher, self).__init__() DBManager.init(1) self.cursor = DBManager.get_cursor() self.cities = cities self.logger = utilities.logger_with_name("StatisticsFetcher") self.query, self.title = "", "" self.results = dict()
def run(self): DBManager.get_connection().jconn.setAutoCommit(False) while self.should_run: try: jobs = self.job_queue.get(True, 5) except: self.logger.debug("No item in the queue. Waiting for something to do...") continue self.logger.info("Executing jobs on the database (approx. " + str(self.job_queue.qsize()) + " remaining)") for job in jobs: match = re.search('UPSERT (.*) VALUES .*', job.query_string) self.logger.info(" Job table: " + match.group(1) if match else "finishing group...") self.logger.info(" Job size: " + str(get_size(job.values_list) / 1024.0) + " KB") utilities.try_execution( lambda: DBManager.get_cursor().executemany(job.query_string, job.values_list), "Executing query: " + job.query_string + ", Last tuple: " + str(job.values_list[-1]), self.logger) DBManager.get_connection().commit() self.logger.info("Finished executing jobs on the database") self.logger.info("PersisterThread stopped")