def dailyUrlDump(config,
sdb=sdb,
gzipped_csv_files=gzipped_csv_files,
IdCache=IdCache,
write_row=write_row,
process_crash=process_crash,
logger=logger):
dbConnectionPool = sdb.DatabaseConnectionPool(config, logger)
try:
try:
db_conn, db_cursor = dbConnectionPool.connectionCursorPair()
with gzipped_csv_files(config) as csv_file_handles_tuple:
headers_not_yet_written = True
id_cache = IdCache(db_cursor)
sql_parameters = setup_query_parameters(config)
logger.debug("config.day = %s; now = %s; yesterday = %s",
config.day, sql_parameters.now_str,
sql_parameters.yesterday_str)
sql_query = sql % sql_parameters
logger.debug("SQL is: %s", sql_query)
for crash_row in sdb.execute(db_cursor, sql_query):
if headers_not_yet_written:
write_row(csv_file_handles_tuple,
[x[0] for x in db_cursor.description])
headers_not_yet_written = False
column_value_list = process_crash(crash_row, id_cache)
write_row(csv_file_handles_tuple, column_value_list)
# end for loop over each crash_row
finally:
dbConnectionPool.cleanup()
except:
util.reportExceptionAndContinue(logger)
def test_connection_attempt_count(self):
logger = self.logger
logger.clear()
class ConnectionCountingFakeDatabase(object):
def __init__(self, config, logger=None):
self.connect_counter = 0
def connection(self, database_module=None):
self.connect_counter += 1
return 17
logger = self.logger
temp_Database = db.Database
db.Database = ConnectionCountingFakeDatabase
try:
db_pool = db.DatabaseConnectionPool(config, logger)
c1 = db_pool.connection()
assert db_pool.database.connect_counter == 1
c1 = db_pool.connection()
assert db_pool.database.connect_counter == 1
c1 = db_pool.connection('fred')
assert db_pool.database.connect_counter == 2
c1 = db_pool.connection()
assert db_pool.database.connect_counter == 2
c1 = db_pool.connection('fred')
assert db_pool.database.connect_counter == 2
finally:
db.Database = temp_Database
def testConnectionPoolConstructor(self):
# just test some variations on constructor calls
logger = self.logger
logger.clear()
try:
cp = db.DatabaseConnectionPool()
assert False, 'expected a raised TypeError, not to get here'
except TypeError, x:
pass
def testConnectionPoolConnectionCursorPair(self):
logger = self.logger
logger.clear()
cp = db.DatabaseConnectionPool(config, logger)
connection0 = cursor0 = None
try:
connection0, cursor0 = cp.connectionCursorPair()
assert connection0
assert cursor0
except Exception, x:
assert False, 'expected nothing, got %s: %s' % (type(x), x)
def testConnectionPoolConnectToDatabase(self):
logger = self.logger
logger.clear()
cp = db.DatabaseConnectionPool(config, logger)
logger.clear()
try:
connection, cursor = cp.connectionCursorPair()
assert connection
assert cursor
except Exception, x:
assert False, 'expected no exceptions, got %s: %s' % (type(x), x)
def testConnectionPoolCleanup(self):
class FakeLogger(object):
def __init__(self):
self.logs = []
def debug(self, *args):
self.logs.append(args)
logger = FakeLogger()
cp = db.DatabaseConnectionPool(config, logger)
conn = cp.connection()
cp.cleanup()
conn = cp.connection()
conn = cp.connection('fred')
cp.cleanup()
expected = [('%s - killing database connections', 'MainThread'),
('%s - connection %s closed', 'MainThread', 'MainThread'),
('%s - killing database connections', 'MainThread'),
('%s - connection %s closed', 'MainThread', 'MainThread'),
('%s - connection %s closed', 'MainThread', 'fred')]
assert len(expected) == len(logger.logs)
for e, a in zip(expected, logger.logs):
assert e == a
assert d.dsn == 'host=A port=B dbname=C user=D password=E', 'dsn not created correctly'
assert d.logger == 1, 'logger passed with dictionary was not overridden by logger passed as a parameter, got %s instead' % d.logger
def testConnectionPoolConstructor(self):
# just test some variations on constructor calls
logger = self.logger
logger.clear()
try:
cp = db.DatabaseConnectionPool()
assert False, 'expected a raised TypeError, not to get here'
except TypeError, x:
pass
except Exception, x:
assert False, 'expected a TypeError, not %s: %s' % (type(x), x)
try:
cp = db.DatabaseConnectionPool(config)
except Exception, x:
assert False, 'expected the non-logger constructor to succeed, got %s: %s' % (
type(x), x)
try:
cp = db.DatabaseConnectionPool(config, self.logger)
except Exception, x:
assert False, 'expected the with-logger constructor to succeed, got %s: %s' % (
type(x), x)
def testConnectionPoolConnectToDatabase(self):
logger = self.logger
logger.clear()
cp = db.DatabaseConnectionPool(config, logger)
logger.clear()
try:
