def testConnectionPoolCleanup(self): logger = self.logger logger.clear() cp = ppghelper.DatabaseConnectionPool(config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) conn, cur = cp.connectionCursorPairNoTest() logger.clear() cp.cleanup() assert [logging.DEBUG, logging.DEBUG] == logger.levels expected = [ "%s - killing thread database connections" % threading.currentThread().getName(), "%s - connection %s closed" % (threading.currentThread().getName(), threading.currentThread().getName()) ] assert expected == logger.buffer, "Expected %s got %s" % ( expected, logger.buffer) logger.clear() cp.cleanup() assert [logging.DEBUG, logging.DEBUG] == logger.levels expected = [ "%s - killing thread database connections" % threading.currentThread().getName(), "%s - connection %s already closed" % (threading.currentThread().getName(), threading.currentThread().getName()) ] assert expected == logger.buffer, "Expected %s got %s" % ( expected, logger.buffer)
def recordBuilds(config, backfill_date): databaseConnectionPool = psy.DatabaseConnectionPool( config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) try: connection, cursor = databaseConnectionPool.connectionCursorPair() for product_name in config.products: scrapeReleases(config, cursor, product_name) today = datetime.datetime.today() if backfill_date is not None: currentdate = backfill_date while currentdate <= today: logger.debug('backfilling for date ' + str(currentdate)) scrapeNightlies(config, cursor, product_name, date=currentdate) currentdate += datetime.timedelta(days=1) else: scrapeNightlies(config, cursor, product_name, date=today) finally: databaseConnectionPool.cleanup()
def testConnectionPoolConstructor(self): # just test some variations on constructor calls logger = self.logger logger.clear() try: cp = ppghelper.DatabaseConnectionPool() assert False, 'expected a raised TypeError, not to get here' except TypeError,x: pass
def testConnectionPoolConnectionCursorPair(self): logger = self.logger logger.clear() cp = ppghelper.DatabaseConnectionPool(config.databaseHost,config.databaseName,config.databaseUserName,config.databasePassword,logger) connection0 = cursor0 = None try: connection0,cursor0 = cp.connectionCursorPair() assert connection0 assert cursor0 except Exception,x: assert False, 'expected nothing, got %s: %s'%(type(x),x)
def update_reports_clean(config): databaseConnectionPool = psy.DatabaseConnectionPool( config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) try: connection, cursor = databaseConnectionPool.connectionCursorPair() startTime = utc_now() - timedelta(hours=2) cursor.callproc('update_reports_clean', [startTime]) connection.commit() finally: databaseConnectionPool.cleanup()
def find_duplicates(config): databaseConnectionPool = psy.DatabaseConnectionPool(config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) try: connection, cursor= databaseConnectionPool.connectionCursorPair() startTime = utc_now() - timedelta(hours=3) endTime = startTime + timedelta(hours=1) cursor.callproc('update_reports_duplicates', (startTime, endTime)) connection.commit() startTime += timedelta(minutes=30) endTime = startTime + timedelta(hours=1) cursor.callproc('update_reports_duplicates', (startTime, endTime)) connection.commit() finally: databaseConnectionPool.cleanup()
def update_reports_clean(config): databaseConnectionPool = psy.DatabaseConnectionPool( config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) # Set the temp_buffers for this session databaseTempbuffers = '8MB' # default if 'databaseTempbuffers' in config: databaseTempbuffers = config.databaseTempbuffers try: connection, cursor = databaseConnectionPool.connectionCursorPair() cursor.execute(""" SET TEMP_BUFFERS = %s """, databaseTempbuffers) startTime = utc_now() - timedelta(hours=2) cursor.callproc('update_reports_clean', [startTime]) connection.commit() finally: databaseConnectionPool.cleanup()
def update_adus(config): databaseConnectionPool = psy.DatabaseConnectionPool( config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) try: connection, cursor = databaseConnectionPool.connectionCursorPair() startTime = (utc_now() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') cursor.callproc('update_adu', [startTime]) connection.commit() except psycopg2.InternalError, e: errmsg = 'ERROR: update_adu has already been run for %s\n' % ( startTime) if e.pgerror == errmsg: logger.warn('Update ADU already run for %s, ignoring' % (startTime)) else: raise
def record_associations(config): databaseConnectionPool = psy.DatabaseConnectionPool( config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) try: databaseConnection, databaseCursor = databaseConnectionPool.connectionCursorPair( ) lastRunDate = get_last_run_date(config) lastRunDateAsString = lastRunDate.strftime('%Y-%m-%d') logger.info("beginning search from this date (YYYY-MM-DD): %s", lastRunDateAsString) query = config.bugzillaQuery % lastRunDateAsString for bug, status, resolution, short_desc, signatureSet in bugzilla_iterator( query): logger.debug("bug %s (%s, %s) %s: %s", bug, status, resolution, short_desc, signatureSet) insert_or_update_bug_in_database(bug, status, resolution, short_desc, signatureSet, databaseCursor) save_last_run_date(config) finally: databaseConnectionPool.cleanup()
def find_duplicates(config): databaseConnectionPool = psy.DatabaseConnectionPool( config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) # Set the temp_buffers for this session databaseTempbuffers = '8MB' # default if 'databaseTempbuffers' in config: databaseTempbuffers = config.databaseTempbuffers try: connection, cursor = databaseConnectionPool.connectionCursorPair() cursor.execute(""" SET TEMP_BUFFERS = %s """, (databaseTempbuffers, )) startTime = utc_now() - timedelta(hours=3) endTime = startTime + timedelta(hours=1) cursor.callproc('update_reports_duplicates', (startTime, endTime)) connection.commit() startTime += timedelta(minutes=30) endTime = startTime + timedelta(hours=1) cursor.callproc('update_reports_duplicates', (startTime, endTime)) connection.commit() finally: databaseConnectionPool.cleanup()
def update_adus(config): databaseConnectionPool = psy.DatabaseConnectionPool( config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger) # Set the temp_buffers for this session databaseTempbuffers = '8MB' # default if 'databaseTempbuffers' in config: databaseTempbuffers = config.databaseTempbuffers try: connection, cursor = databaseConnectionPool.connectionCursorPair() cursor.execute(""" SET TEMP_BUFFERS = %s """, databaseTempbuffers) startTime = (utc_now() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') cursor.callproc('update_adu', [startTime]) connection.commit() except psycopg2.InternalError, e: errmsg = 'ERROR: update_adu has already been run for %s\n' % ( startTime) if e.pgerror == errmsg: logger.warn('Update ADU already run for %s, ignoring' % (startTime)) else: raise
except Exception, e: assert False, "must not raise this exception" def testConnectionPoolConstructor(self): # just test some variations on constructor calls logger = self.logger logger.clear() try: cp = ppghelper.DatabaseConnectionPool() assert False, 'expected a raised TypeError, not to get here' except TypeError,x: pass except Exception,x: assert False, 'expected a TypeError, not %s: %s'%(type(x),x) try: cp = ppghelper.DatabaseConnectionPool(*self.connectionData0) except Exception,x: assert False, 'expected the non-logger constructor to succeed, got %s: %s'%(type(x),x) try: cp = ppghelper.DatabaseConnectionPool(*self.connectionDataL) except Exception,x: assert False, 'expected the with-logger constructor to succeed, got %s: %s'%(type(x),x) def testConnectionPoolConnectToDatabase(self): logger = self.logger logger.clear() cp = ppghelper.DatabaseConnectionPool(config.databaseHost,config.databaseName,config.databaseUserName,config.databasePassword,logger) logger.clear() try: connection,cursor = cp.connectToDatabase() assert connection