def dailyUrlDump(config, sdb=sdb, gzipped_csv_files=gzipped_csv_files, IdCache=IdCache, write_row=write_row, process_crash=process_crash, logger=logger): dbConnectionPool = sdb.DatabaseConnectionPool(config, logger) try: try: db_conn, db_cursor = dbConnectionPool.connectionCursorPair() with gzipped_csv_files(config) as csv_file_handles_tuple: headers_not_yet_written = True id_cache = IdCache(db_cursor) sql_parameters = setup_query_parameters(config) logger.debug("config.day = %s; now = %s; yesterday = %s", config.day, sql_parameters.now_str, sql_parameters.yesterday_str) sql_query = sql % sql_parameters logger.debug("SQL is: %s", sql_query) for crash_row in sdb.execute(db_cursor, sql_query): if headers_not_yet_written: write_row(csv_file_handles_tuple, [x[0] for x in db_cursor.description]) headers_not_yet_written = False column_value_list = process_crash(crash_row, id_cache) write_row(csv_file_handles_tuple, column_value_list) # end for loop over each crash_row finally: dbConnectionPool.cleanup() except: util.reportExceptionAndContinue(logger)
def test_connection_attempt_count(self): logger = self.logger logger.clear() class ConnectionCountingFakeDatabase(object): def __init__(self, config, logger=None): self.connect_counter = 0 def connection(self, database_module=None): self.connect_counter += 1 return 17 logger = self.logger temp_Database = db.Database db.Database = ConnectionCountingFakeDatabase try: db_pool = db.DatabaseConnectionPool(config, logger) c1 = db_pool.connection() assert db_pool.database.connect_counter == 1 c1 = db_pool.connection() assert db_pool.database.connect_counter == 1 c1 = db_pool.connection('fred') assert db_pool.database.connect_counter == 2 c1 = db_pool.connection() assert db_pool.database.connect_counter == 2 c1 = db_pool.connection('fred') assert db_pool.database.connect_counter == 2 finally: db.Database = temp_Database
def testConnectionPoolConstructor(self): # just test some variations on constructor calls logger = self.logger logger.clear() try: cp = db.DatabaseConnectionPool() assert False, 'expected a raised TypeError, not to get here' except TypeError, x: pass
def testConnectionPoolConnectionCursorPair(self): logger = self.logger logger.clear() cp = db.DatabaseConnectionPool(config, logger) connection0 = cursor0 = None try: connection0, cursor0 = cp.connectionCursorPair() assert connection0 assert cursor0 except Exception, x: assert False, 'expected nothing, got %s: %s' % (type(x), x)
def testConnectionPoolConnectToDatabase(self): logger = self.logger logger.clear() cp = db.DatabaseConnectionPool(config, logger) logger.clear() try: connection, cursor = cp.connectionCursorPair() assert connection assert cursor except Exception, x: assert False, 'expected no exceptions, got %s: %s' % (type(x), x)
def testConnectionPoolCleanup(self): class FakeLogger(object): def __init__(self): self.logs = [] def debug(self, *args): self.logs.append(args) logger = FakeLogger() cp = db.DatabaseConnectionPool(config, logger) conn = cp.connection() cp.cleanup() conn = cp.connection() conn = cp.connection('fred') cp.cleanup() expected = [('%s - killing database connections', 'MainThread'), ('%s - connection %s closed', 'MainThread', 'MainThread'), ('%s - killing database connections', 'MainThread'), ('%s - connection %s closed', 'MainThread', 'MainThread'), ('%s - connection %s closed', 'MainThread', 'fred')] assert len(expected) == len(logger.logs) for e, a in zip(expected, logger.logs): assert e == a
assert d.dsn == 'host=A port=B dbname=C user=D password=E', 'dsn not created correctly' assert d.logger == 1, 'logger passed with dictionary was not overridden by logger passed as a parameter, got %s instead' % d.logger def testConnectionPoolConstructor(self): # just test some variations on constructor calls logger = self.logger logger.clear() try: cp = db.DatabaseConnectionPool() assert False, 'expected a raised TypeError, not to get here' except TypeError, x: pass except Exception, x: assert False, 'expected a TypeError, not %s: %s' % (type(x), x) try: cp = db.DatabaseConnectionPool(config) except Exception, x: assert False, 'expected the non-logger constructor to succeed, got %s: %s' % ( type(x), x) try: cp = db.DatabaseConnectionPool(config, self.logger) except Exception, x: assert False, 'expected the with-logger constructor to succeed, got %s: %s' % ( type(x), x) def testConnectionPoolConnectToDatabase(self): logger = self.logger logger.clear() cp = db.DatabaseConnectionPool(config, logger) logger.clear() try: