def test_disconnectionerror_view_integration(self): # Test setup. self.useFixture(Urllib2Fixture()) bouncer = PGBouncerFixture() # XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03: # In parallel tests, we are rarely encountering instances of # bug 504291 while running this test. These cause the tests # to fail entirely (the store.rollback() described in comment # 11 does not fix the insane state) despite nultiple retries. # As mentioned in that bug, we are trying aborts to see if they # eliminate the problem. If this works, we can find which of # these two aborts are actually needed. transaction.abort() self.useFixture(bouncer) transaction.abort() # Verify things are working initially. url = 'http://launchpad.dev/' self.retryConnection(url, bouncer) # Now break the database, and we get an exception, along with # our view. bouncer.stop() error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) # We keep seeing the correct exception on subsequent requests. error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) # When the database is available again, requests succeed. bouncer.start() self.retryConnection(url, bouncer)
def test_disconnectionerror_view_integration(self): # Test setup. self.useFixture(Urllib2Fixture()) bouncer = PGBouncerFixture() # XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03: # In parallel tests, we are rarely encountering instances of # bug 504291 while running this test. These cause the tests # to fail entirely (the store.rollback() described in comment # 11 does not fix the insane state) despite nultiple retries. # As mentioned in that bug, we are trying aborts to see if they # eliminate the problem. If this works, we can find which of # these two aborts are actually needed. transaction.abort() self.useFixture(bouncer) transaction.abort() # Verify things are working initially. url = "http://launchpad.dev/" self.retryConnection(url, bouncer) # Now break the database, and we get an exception, along with # our view. bouncer.stop() error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) # We keep seeing the correct exception on subsequent requests. error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) # When the database is available again, requests succeed. bouncer.start() self.retryConnection(url, bouncer)
def test_stop_and_start(self): # Database is working. assert self.is_connected() # And database with the fixture is working too. pgbouncer = PGBouncerFixture() with PGBouncerFixture() as pgbouncer: assert self.is_connected() # pgbouncer is transparant. To confirm we are connecting via # pgbouncer, we need to shut it down and confirm our # connections are dropped. pgbouncer.stop() assert not self.is_connected() # If we restart it, things should be back to normal. pgbouncer.start() assert self.is_connected() # Database is still working. assert self.is_connected()
def test_disconnectionerror_view_integration(self): # Test setup. self.useFixture(FakeLogger('SiteError', level=logging.CRITICAL)) self.useFixture(Urllib2Fixture()) bouncer = PGBouncerFixture() # XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03: # In parallel tests, we are rarely encountering instances of # bug 504291 while running this test. These cause the tests # to fail entirely (the store.rollback() described in comment # 11 does not fix the insane state) despite nultiple retries. # As mentioned in that bug, we are trying aborts to see if they # eliminate the problem. If this works, we can find which of # these two aborts are actually needed. transaction.abort() self.useFixture(bouncer) transaction.abort() # Verify things are working initially. url = 'http://launchpad.dev/' self.retryConnection(url, bouncer) # Now break the database, and we get an exception, along with # our view and several OOPSes from the retries. bouncer.stop() class Disconnects(Equals): def __init__(self, message): super(Disconnects, self).__init__( ('DisconnectionError', message)) with CaptureOops() as oopses: error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) self.assertThat( [(oops['type'], oops['value'].split('\n')[0]) for oops in oopses.oopses], MatchesListwise( [MatchesAny( # libpq < 9.5. Disconnects('error with no message from the libpq'), # libpq >= 9.5. Disconnects('server closed the connection unexpectedly'))] * 2 + [Disconnects( 'could not connect to server: Connection refused')] * 6)) # We keep seeing the correct exception on subsequent requests. with CaptureOops() as oopses: error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) self.assertThat( [(oops['type'], oops['value'].split('\n')[0]) for oops in oopses.oopses], MatchesListwise( [Disconnects( 'could not connect to server: Connection refused')] * 8)) # When the database is available again, requests succeed. bouncer.start() self.retryConnection(url, bouncer) # If we ask pgbouncer to disable the database, requests fail and # get the same error page, but we don't log OOPSes except for # the initial connection terminations. Disablement is always # explicit maintenance, and we don't need lots of ongoing OOPSes # to tell us about maintenance that we're doing. dbname = DatabaseLayer._db_fixture.dbname conn = psycopg2.connect("dbname=pgbouncer") conn.autocommit = True cur = conn.cursor() cur.execute("DISABLE " + dbname) cur.execute("KILL " + dbname) cur.execute("RESUME " + dbname) with CaptureOops() as oopses: error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) self.assertThat( [(oops['type'], oops['value'].split('\n')[0]) for oops in oopses.oopses], MatchesListwise([Disconnects('database removed')])) # A second request doesn't log any OOPSes. with CaptureOops() as oopses: error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) self.assertEqual( [], [(oops['type'], oops['value'].split('\n')[0]) for oops in oopses.oopses]) # When the database is available again, requests succeed. cur.execute("ENABLE %s" % DatabaseLayer._db_fixture.dbname) self.retryConnection(url, bouncer)
class MasterFallbackTestCase(TestCase): layer = DatabaseFunctionalLayer def setUp(self): super(MasterFallbackTestCase, self).setUp() self.pgbouncer_fixture = PGBouncerFixture() # The PGBouncerFixture will set the PGPORT environment variable, # causing all DB connections to go via pgbouncer unless an # explicit port is provided. dbname = DatabaseLayer._db_fixture.dbname # Pull the direct db connection string, including explicit port. conn_str_direct = self.pgbouncer_fixture.databases[dbname] # Generate a db connection string that will go via pgbouncer. conn_str_pgbouncer = 'dbname=%s host=localhost' % dbname # Configure slave connections via pgbouncer, so we can shut them # down. Master connections direct so they are unaffected. config_key = 'master-slave-separation' config.push( config_key, dedent('''\ [database] rw_main_master: %s rw_main_slave: %s ''' % (conn_str_direct, conn_str_pgbouncer))) self.addCleanup(lambda: config.pop(config_key)) self.useFixture(self.pgbouncer_fixture) def test_can_shutdown_slave_only(self): '''Confirm that this TestCase's test infrastructure works as needed. ''' master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) # Both Stores work when pgbouncer is up. master_store.get(Person, 1) slave_store.get(Person, 1) # Slave Store breaks when pgbouncer is torn down. Master Store # is fine. self.pgbouncer_fixture.stop() master_store.get(Person, 2) self.assertRaises(DisconnectionError, slave_store.get, Person, 2) def test_startup_with_no_slave(self): '''An attempt is made for the first time to connect to a slave.''' self.pgbouncer_fixture.stop() master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) # The master and slave Stores are the same object. self.assertIs(master_store, slave_store) def test_slave_shutdown_during_transaction(self): '''Slave is shutdown while running, but we can recover.''' master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIsNot(master_store, slave_store) self.pgbouncer_fixture.stop() # The transaction fails if the slave store is used. Robust # processes will handle this and retry (even if just means exit # and wait for the next scheduled invocation). self.assertRaises(DisconnectionError, slave_store.get, Person, 1) transaction.abort() # But in the next transaction, we get the master Store if we ask # for the slave Store so we can continue. master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIs(master_store, slave_store) def test_slave_shutdown_between_transactions(self): '''Slave is shutdown in between transactions.''' master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIsNot(master_store, slave_store) transaction.abort() self.pgbouncer_fixture.stop() # The process doesn't notice the slave going down, and things # will fail the next time the slave is used. master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIsNot(master_store, slave_store) self.assertRaises(DisconnectionError, slave_store.get, Person, 1) # But now it has been discovered the socket is no longer # connected to anything, next transaction we get a master # Store when we ask for a slave. master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIs(master_store, slave_store) def test_slave_reconnect_after_outage(self): '''The slave is again used once it becomes available.''' self.pgbouncer_fixture.stop() master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIs(master_store, slave_store) self.pgbouncer_fixture.start() transaction.abort() master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIsNot(master_store, slave_store)
class MasterFallbackTestCase(TestCase): layer = DatabaseFunctionalLayer def setUp(self): super(MasterFallbackTestCase, self).setUp() self.pgbouncer_fixture = PGBouncerFixture() # The PGBouncerFixture will set the PGPORT environment variable, # causing all DB connections to go via pgbouncer unless an # explicit port is provided. dbname = DatabaseLayer._db_fixture.dbname # Pull the direct db connection string, including explicit port. conn_str_direct = self.pgbouncer_fixture.databases[dbname] # Generate a db connection string that will go via pgbouncer. conn_str_pgbouncer = "dbname=%s host=localhost" % dbname # Configure slave connections via pgbouncer, so we can shut them # down. Master connections direct so they are unaffected. config_key = "master-slave-separation" config.push( config_key, dedent( """\ [database] rw_main_master: %s rw_main_slave: %s """ % (conn_str_direct, conn_str_pgbouncer) ), ) self.addCleanup(lambda: config.pop(config_key)) self.useFixture(self.pgbouncer_fixture) def test_can_shutdown_slave_only(self): """Confirm that this TestCase's test infrastructure works as needed. """ master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) # Both Stores work when pgbouncer is up. master_store.get(Person, 1) slave_store.get(Person, 1) # Slave Store breaks when pgbouncer is torn down. Master Store # is fine. self.pgbouncer_fixture.stop() master_store.get(Person, 2) self.assertRaises(DisconnectionError, slave_store.get, Person, 2) def test_startup_with_no_slave(self): """An attempt is made for the first time to connect to a slave.""" self.pgbouncer_fixture.stop() master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) # The master and slave Stores are the same object. self.assertIs(master_store, slave_store) def test_slave_shutdown_during_transaction(self): """Slave is shutdown while running, but we can recover.""" master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIsNot(master_store, slave_store) self.pgbouncer_fixture.stop() # The transaction fails if the slave store is used. Robust # processes will handle this and retry (even if just means exit # and wait for the next scheduled invocation). self.assertRaises(DisconnectionError, slave_store.get, Person, 1) transaction.abort() # But in the next transaction, we get the master Store if we ask # for the slave Store so we can continue. master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIs(master_store, slave_store) def test_slave_shutdown_between_transactions(self): """Slave is shutdown in between transactions.""" master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIsNot(master_store, slave_store) transaction.abort() self.pgbouncer_fixture.stop() # The process doesn't notice the slave going down, and things # will fail the next time the slave is used. master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIsNot(master_store, slave_store) self.assertRaises(DisconnectionError, slave_store.get, Person, 1) # But now it has been discovered the socket is no longer # connected to anything, next transaction we get a master # Store when we ask for a slave. master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIs(master_store, slave_store) def test_slave_reconnect_after_outage(self): """The slave is again used once it becomes available.""" self.pgbouncer_fixture.stop() master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIs(master_store, slave_store) self.pgbouncer_fixture.start() transaction.abort() master_store = IMasterStore(Person) slave_store = ISlaveStore(Person) self.assertIsNot(master_store, slave_store)