def testConnectionPoolCleanup(self):
logger = self.logger
logger.clear()
cp = ppghelper.DatabaseConnectionPool(config.databaseHost,
config.databaseName,
config.databaseUserName,
config.databasePassword, logger)
conn, cur = cp.connectionCursorPairNoTest()
logger.clear()
cp.cleanup()
assert [logging.DEBUG, logging.DEBUG] == logger.levels
expected = [
"%s - killing thread database connections" %
threading.currentThread().getName(),
"%s - connection %s closed" % (threading.currentThread().getName(),
threading.currentThread().getName())
]
assert expected == logger.buffer, "Expected %s got %s" % (
expected, logger.buffer)
logger.clear()
cp.cleanup()
assert [logging.DEBUG, logging.DEBUG] == logger.levels
expected = [
"%s - killing thread database connections" %
threading.currentThread().getName(),
"%s - connection %s already closed" %
(threading.currentThread().getName(),
threading.currentThread().getName())
]
assert expected == logger.buffer, "Expected %s got %s" % (
expected, logger.buffer)
def recordBuilds(config, backfill_date):
databaseConnectionPool = psy.DatabaseConnectionPool(
config.databaseHost, config.databaseName, config.databaseUserName,
config.databasePassword, logger)
try:
connection, cursor = databaseConnectionPool.connectionCursorPair()
for product_name in config.products:
scrapeReleases(config, cursor, product_name)
today = datetime.datetime.today()
if backfill_date is not None:
currentdate = backfill_date
while currentdate <= today:
logger.debug('backfilling for date ' + str(currentdate))
scrapeNightlies(config,
cursor,
product_name,
date=currentdate)
currentdate += datetime.timedelta(days=1)
else:
scrapeNightlies(config, cursor, product_name, date=today)
finally:
databaseConnectionPool.cleanup()
def testConnectionPoolConstructor(self):
# just test some variations on constructor calls
logger = self.logger
logger.clear()
try:
cp = ppghelper.DatabaseConnectionPool()
assert False, 'expected a raised TypeError, not to get here'
except TypeError,x:
pass
def testConnectionPoolConnectionCursorPair(self):
logger = self.logger
logger.clear()
cp = ppghelper.DatabaseConnectionPool(config.databaseHost,config.databaseName,config.databaseUserName,config.databasePassword,logger)
connection0 = cursor0 = None
try:
connection0,cursor0 = cp.connectionCursorPair()
assert connection0
assert cursor0
except Exception,x:
assert False, 'expected nothing, got %s: %s'%(type(x),x)
def update_reports_clean(config):
databaseConnectionPool = psy.DatabaseConnectionPool(
config.databaseHost, config.databaseName, config.databaseUserName,
config.databasePassword, logger)
try:
connection, cursor = databaseConnectionPool.connectionCursorPair()
startTime = utc_now() - timedelta(hours=2)
cursor.callproc('update_reports_clean', [startTime])
connection.commit()
finally:
databaseConnectionPool.cleanup()
def find_duplicates(config):
databaseConnectionPool = psy.DatabaseConnectionPool(config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger)
try:
connection, cursor= databaseConnectionPool.connectionCursorPair()
startTime = utc_now() - timedelta(hours=3)
endTime = startTime + timedelta(hours=1)
cursor.callproc('update_reports_duplicates', (startTime, endTime))
connection.commit()
startTime += timedelta(minutes=30)
endTime = startTime + timedelta(hours=1)
cursor.callproc('update_reports_duplicates', (startTime, endTime))
connection.commit()
finally:
databaseConnectionPool.cleanup()
def update_reports_clean(config):
databaseConnectionPool = psy.DatabaseConnectionPool(
config.databaseHost, config.databaseName, config.databaseUserName,
config.databasePassword, logger)
# Set the temp_buffers for this session
databaseTempbuffers = '8MB' # default
if 'databaseTempbuffers' in config:
databaseTempbuffers = config.databaseTempbuffers
try:
connection, cursor = databaseConnectionPool.connectionCursorPair()
cursor.execute(""" SET TEMP_BUFFERS = %s """, databaseTempbuffers)
startTime = utc_now() - timedelta(hours=2)
cursor.callproc('update_reports_clean', [startTime])
connection.commit()
finally:
databaseConnectionPool.cleanup()
def update_adus(config):
databaseConnectionPool = psy.DatabaseConnectionPool(
config.databaseHost, config.databaseName, config.databaseUserName,
config.databasePassword, logger)
try:
connection, cursor = databaseConnectionPool.connectionCursorPair()
startTime = (utc_now() -
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
cursor.callproc('update_adu', [startTime])
connection.commit()
except psycopg2.InternalError, e:
errmsg = 'ERROR: update_adu has already been run for %s\n' % (
startTime)
if e.pgerror == errmsg:
logger.warn('Update ADU already run for %s, ignoring' %
(startTime))
else:
raise
def record_associations(config):
databaseConnectionPool = psy.DatabaseConnectionPool(
config.databaseHost, config.databaseName, config.databaseUserName,
config.databasePassword, logger)
try:
databaseConnection, databaseCursor = databaseConnectionPool.connectionCursorPair(
)
lastRunDate = get_last_run_date(config)
lastRunDateAsString = lastRunDate.strftime('%Y-%m-%d')
logger.info("beginning search from this date (YYYY-MM-DD): %s",
lastRunDateAsString)
query = config.bugzillaQuery % lastRunDateAsString
for bug, status, resolution, short_desc, signatureSet in bugzilla_iterator(
query):
logger.debug("bug %s (%s, %s) %s: %s", bug, status, resolution,
short_desc, signatureSet)
insert_or_update_bug_in_database(bug, status, resolution,
short_desc, signatureSet,
databaseCursor)
save_last_run_date(config)
finally:
databaseConnectionPool.cleanup()
def find_duplicates(config):
databaseConnectionPool = psy.DatabaseConnectionPool(
config.databaseHost, config.databaseName, config.databaseUserName,
config.databasePassword, logger)
# Set the temp_buffers for this session
databaseTempbuffers = '8MB' # default
if 'databaseTempbuffers' in config:
databaseTempbuffers = config.databaseTempbuffers
try:
connection, cursor = databaseConnectionPool.connectionCursorPair()
cursor.execute(""" SET TEMP_BUFFERS = %s """, (databaseTempbuffers, ))
startTime = utc_now() - timedelta(hours=3)
endTime = startTime + timedelta(hours=1)
cursor.callproc('update_reports_duplicates', (startTime, endTime))
connection.commit()
startTime += timedelta(minutes=30)
endTime = startTime + timedelta(hours=1)
cursor.callproc('update_reports_duplicates', (startTime, endTime))
connection.commit()
finally:
databaseConnectionPool.cleanup()
def update_adus(config):
databaseConnectionPool = psy.DatabaseConnectionPool(
config.databaseHost, config.databaseName, config.databaseUserName,
config.databasePassword, logger)
# Set the temp_buffers for this session
databaseTempbuffers = '8MB' # default
if 'databaseTempbuffers' in config:
databaseTempbuffers = config.databaseTempbuffers
try:
connection, cursor = databaseConnectionPool.connectionCursorPair()
cursor.execute(""" SET TEMP_BUFFERS = %s """, databaseTempbuffers)
startTime = (utc_now() -
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
cursor.callproc('update_adu', [startTime])
connection.commit()
except psycopg2.InternalError, e:
errmsg = 'ERROR: update_adu has already been run for %s\n' % (
startTime)
if e.pgerror == errmsg:
logger.warn('Update ADU already run for %s, ignoring' %
(startTime))
else:
raise
except Exception, e:
assert False, "must not raise this exception"
def testConnectionPoolConstructor(self):
# just test some variations on constructor calls
logger = self.logger
logger.clear()
try:
cp = ppghelper.DatabaseConnectionPool()
assert False, 'expected a raised TypeError, not to get here'
except TypeError,x:
pass
except Exception,x:
assert False, 'expected a TypeError, not %s: %s'%(type(x),x)
try:
cp = ppghelper.DatabaseConnectionPool(*self.connectionData0)
except Exception,x:
assert False, 'expected the non-logger constructor to succeed, got %s: %s'%(type(x),x)
try:
cp = ppghelper.DatabaseConnectionPool(*self.connectionDataL)
except Exception,x:
assert False, 'expected the with-logger constructor to succeed, got %s: %s'%(type(x),x)
def testConnectionPoolConnectToDatabase(self):
logger = self.logger
logger.clear()
cp = ppghelper.DatabaseConnectionPool(config.databaseHost,config.databaseName,config.databaseUserName,config.databasePassword,logger)
logger.clear()
try:
connection,cursor = cp.connectToDatabase()
assert connection
