Пример #1
0
 def test_disconnectionerror_view_integration(self):
     # Test setup.
     self.useFixture(Urllib2Fixture())
     bouncer = PGBouncerFixture()
     # XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
     # In parallel tests, we are rarely encountering instances of
     # bug 504291 while running this test.  These cause the tests
     # to fail entirely (the store.rollback() described in comment
     # 11 does not fix the insane state) despite nultiple retries.
     # As mentioned in that bug, we are trying aborts to see if they
     # eliminate the problem.  If this works, we can find which of
     # these two aborts are actually needed.
     transaction.abort()
     self.useFixture(bouncer)
     transaction.abort()
     # Verify things are working initially.
     url = 'http://launchpad.dev/'
     self.retryConnection(url, bouncer)
     # Now break the database, and we get an exception, along with
     # our view.
     bouncer.stop()
     error = self.getHTTPError(url)
     self.assertEqual(503, error.code)
     self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
     # We keep seeing the correct exception on subsequent requests.
     error = self.getHTTPError(url)
     self.assertEqual(503, error.code)
     self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
     # When the database is available again, requests succeed.
     bouncer.start()
     self.retryConnection(url, bouncer)
Пример #2
0
 def test_disconnectionerror_view_integration(self):
     # Test setup.
     self.useFixture(Urllib2Fixture())
     bouncer = PGBouncerFixture()
     # XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
     # In parallel tests, we are rarely encountering instances of
     # bug 504291 while running this test.  These cause the tests
     # to fail entirely (the store.rollback() described in comment
     # 11 does not fix the insane state) despite nultiple retries.
     # As mentioned in that bug, we are trying aborts to see if they
     # eliminate the problem.  If this works, we can find which of
     # these two aborts are actually needed.
     transaction.abort()
     self.useFixture(bouncer)
     transaction.abort()
     # Verify things are working initially.
     url = "http://launchpad.dev/"
     self.retryConnection(url, bouncer)
     # Now break the database, and we get an exception, along with
     # our view.
     bouncer.stop()
     error = self.getHTTPError(url)
     self.assertEqual(503, error.code)
     self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
     # We keep seeing the correct exception on subsequent requests.
     error = self.getHTTPError(url)
     self.assertEqual(503, error.code)
     self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
     # When the database is available again, requests succeed.
     bouncer.start()
     self.retryConnection(url, bouncer)
Пример #3
0
    def test_stop_and_start(self):
        # Database is working.
        assert self.is_connected()

        # And database with the fixture is working too.
        pgbouncer = PGBouncerFixture()
        with PGBouncerFixture() as pgbouncer:
            assert self.is_connected()

            # pgbouncer is transparant. To confirm we are connecting via
            # pgbouncer, we need to shut it down and confirm our
            # connections are dropped.
            pgbouncer.stop()
            assert not self.is_connected()

            # If we restart it, things should be back to normal.
            pgbouncer.start()
            assert self.is_connected()

        # Database is still working.
        assert self.is_connected()
Пример #4
0
    def test_stop_and_start(self):
        # Database is working.
        assert self.is_connected()

        # And database with the fixture is working too.
        pgbouncer = PGBouncerFixture()
        with PGBouncerFixture() as pgbouncer:
            assert self.is_connected()

            # pgbouncer is transparant. To confirm we are connecting via
            # pgbouncer, we need to shut it down and confirm our
            # connections are dropped.
            pgbouncer.stop()
            assert not self.is_connected()

            # If we restart it, things should be back to normal.
            pgbouncer.start()
            assert self.is_connected()

        # Database is still working.
        assert self.is_connected()
Пример #5
0
    def test_disconnectionerror_view_integration(self):
        # Test setup.
        self.useFixture(FakeLogger('SiteError', level=logging.CRITICAL))
        self.useFixture(Urllib2Fixture())
        bouncer = PGBouncerFixture()
        # XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
        # In parallel tests, we are rarely encountering instances of
        # bug 504291 while running this test.  These cause the tests
        # to fail entirely (the store.rollback() described in comment
        # 11 does not fix the insane state) despite nultiple retries.
        # As mentioned in that bug, we are trying aborts to see if they
        # eliminate the problem.  If this works, we can find which of
        # these two aborts are actually needed.
        transaction.abort()
        self.useFixture(bouncer)
        transaction.abort()
        # Verify things are working initially.
        url = 'http://launchpad.dev/'
        self.retryConnection(url, bouncer)
        # Now break the database, and we get an exception, along with
        # our view and several OOPSes from the retries.
        bouncer.stop()

        class Disconnects(Equals):
            def __init__(self, message):
                super(Disconnects, self).__init__(
                    ('DisconnectionError', message))

        with CaptureOops() as oopses:
            error = self.getHTTPError(url)
        self.assertEqual(503, error.code)
        self.assertThat(error.read(),
                        Contains(DisconnectionErrorView.reason))
        self.assertThat(
            [(oops['type'], oops['value'].split('\n')[0])
             for oops in oopses.oopses],
            MatchesListwise(
                [MatchesAny(
                    # libpq < 9.5.
                    Disconnects('error with no message from the libpq'),
                    # libpq >= 9.5.
                    Disconnects('server closed the connection unexpectedly'))]
                * 2 +
                [Disconnects(
                    'could not connect to server: Connection refused')]
                * 6))

        # We keep seeing the correct exception on subsequent requests.
        with CaptureOops() as oopses:
            error = self.getHTTPError(url)
        self.assertEqual(503, error.code)
        self.assertThat(error.read(),
                        Contains(DisconnectionErrorView.reason))
        self.assertThat(
            [(oops['type'], oops['value'].split('\n')[0])
             for oops in oopses.oopses],
            MatchesListwise(
                [Disconnects(
                    'could not connect to server: Connection refused')]
                * 8))

        # When the database is available again, requests succeed.
        bouncer.start()
        self.retryConnection(url, bouncer)

        # If we ask pgbouncer to disable the database, requests fail and
        # get the same error page, but we don't log OOPSes except for
        # the initial connection terminations. Disablement is always
        # explicit maintenance, and we don't need lots of ongoing OOPSes
        # to tell us about maintenance that we're doing.
        dbname = DatabaseLayer._db_fixture.dbname
        conn = psycopg2.connect("dbname=pgbouncer")
        conn.autocommit = True
        cur = conn.cursor()
        cur.execute("DISABLE " + dbname)
        cur.execute("KILL " + dbname)
        cur.execute("RESUME " + dbname)

        with CaptureOops() as oopses:
            error = self.getHTTPError(url)
        self.assertEqual(503, error.code)
        self.assertThat(error.read(),
                        Contains(DisconnectionErrorView.reason))
        self.assertThat(
            [(oops['type'], oops['value'].split('\n')[0])
             for oops in oopses.oopses],
            MatchesListwise([Disconnects('database removed')]))

        # A second request doesn't log any OOPSes.
        with CaptureOops() as oopses:
            error = self.getHTTPError(url)
        self.assertEqual(503, error.code)
        self.assertThat(error.read(),
                        Contains(DisconnectionErrorView.reason))
        self.assertEqual(
            [],
            [(oops['type'], oops['value'].split('\n')[0])
             for oops in oopses.oopses])

        # When the database is available again, requests succeed.
        cur.execute("ENABLE %s" % DatabaseLayer._db_fixture.dbname)
        self.retryConnection(url, bouncer)
Пример #6
0
class MasterFallbackTestCase(TestCase):
    layer = DatabaseFunctionalLayer

    def setUp(self):
        super(MasterFallbackTestCase, self).setUp()

        self.pgbouncer_fixture = PGBouncerFixture()

        # The PGBouncerFixture will set the PGPORT environment variable,
        # causing all DB connections to go via pgbouncer unless an
        # explicit port is provided.
        dbname = DatabaseLayer._db_fixture.dbname
        # Pull the direct db connection string, including explicit port.
        conn_str_direct = self.pgbouncer_fixture.databases[dbname]
        # Generate a db connection string that will go via pgbouncer.
        conn_str_pgbouncer = 'dbname=%s host=localhost' % dbname

        # Configure slave connections via pgbouncer, so we can shut them
        # down. Master connections direct so they are unaffected.
        config_key = 'master-slave-separation'
        config.push(
            config_key,
            dedent('''\
            [database]
            rw_main_master: %s
            rw_main_slave: %s
            ''' % (conn_str_direct, conn_str_pgbouncer)))
        self.addCleanup(lambda: config.pop(config_key))

        self.useFixture(self.pgbouncer_fixture)

    def test_can_shutdown_slave_only(self):
        '''Confirm that this TestCase's test infrastructure works as needed.
        '''
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)

        # Both Stores work when pgbouncer is up.
        master_store.get(Person, 1)
        slave_store.get(Person, 1)

        # Slave Store breaks when pgbouncer is torn down. Master Store
        # is fine.
        self.pgbouncer_fixture.stop()
        master_store.get(Person, 2)
        self.assertRaises(DisconnectionError, slave_store.get, Person, 2)

    def test_startup_with_no_slave(self):
        '''An attempt is made for the first time to connect to a slave.'''
        self.pgbouncer_fixture.stop()

        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)

        # The master and slave Stores are the same object.
        self.assertIs(master_store, slave_store)

    def test_slave_shutdown_during_transaction(self):
        '''Slave is shutdown while running, but we can recover.'''
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)

        self.assertIsNot(master_store, slave_store)

        self.pgbouncer_fixture.stop()

        # The transaction fails if the slave store is used. Robust
        # processes will handle this and retry (even if just means exit
        # and wait for the next scheduled invocation).
        self.assertRaises(DisconnectionError, slave_store.get, Person, 1)

        transaction.abort()

        # But in the next transaction, we get the master Store if we ask
        # for the slave Store so we can continue.
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)

        self.assertIs(master_store, slave_store)

    def test_slave_shutdown_between_transactions(self):
        '''Slave is shutdown in between transactions.'''
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIsNot(master_store, slave_store)

        transaction.abort()
        self.pgbouncer_fixture.stop()

        # The process doesn't notice the slave going down, and things
        # will fail the next time the slave is used.
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIsNot(master_store, slave_store)
        self.assertRaises(DisconnectionError, slave_store.get, Person, 1)

        # But now it has been discovered the socket is no longer
        # connected to anything, next transaction we get a master
        # Store when we ask for a slave.
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIs(master_store, slave_store)

    def test_slave_reconnect_after_outage(self):
        '''The slave is again used once it becomes available.'''
        self.pgbouncer_fixture.stop()

        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIs(master_store, slave_store)

        self.pgbouncer_fixture.start()
        transaction.abort()

        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIsNot(master_store, slave_store)
Пример #7
0
class MasterFallbackTestCase(TestCase):
    layer = DatabaseFunctionalLayer

    def setUp(self):
        super(MasterFallbackTestCase, self).setUp()

        self.pgbouncer_fixture = PGBouncerFixture()

        # The PGBouncerFixture will set the PGPORT environment variable,
        # causing all DB connections to go via pgbouncer unless an
        # explicit port is provided.
        dbname = DatabaseLayer._db_fixture.dbname
        # Pull the direct db connection string, including explicit port.
        conn_str_direct = self.pgbouncer_fixture.databases[dbname]
        # Generate a db connection string that will go via pgbouncer.
        conn_str_pgbouncer = "dbname=%s host=localhost" % dbname

        # Configure slave connections via pgbouncer, so we can shut them
        # down. Master connections direct so they are unaffected.
        config_key = "master-slave-separation"
        config.push(
            config_key,
            dedent(
                """\
            [database]
            rw_main_master: %s
            rw_main_slave: %s
            """
                % (conn_str_direct, conn_str_pgbouncer)
            ),
        )
        self.addCleanup(lambda: config.pop(config_key))

        self.useFixture(self.pgbouncer_fixture)

    def test_can_shutdown_slave_only(self):
        """Confirm that this TestCase's test infrastructure works as needed.
        """
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)

        # Both Stores work when pgbouncer is up.
        master_store.get(Person, 1)
        slave_store.get(Person, 1)

        # Slave Store breaks when pgbouncer is torn down. Master Store
        # is fine.
        self.pgbouncer_fixture.stop()
        master_store.get(Person, 2)
        self.assertRaises(DisconnectionError, slave_store.get, Person, 2)

    def test_startup_with_no_slave(self):
        """An attempt is made for the first time to connect to a slave."""
        self.pgbouncer_fixture.stop()

        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)

        # The master and slave Stores are the same object.
        self.assertIs(master_store, slave_store)

    def test_slave_shutdown_during_transaction(self):
        """Slave is shutdown while running, but we can recover."""
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)

        self.assertIsNot(master_store, slave_store)

        self.pgbouncer_fixture.stop()

        # The transaction fails if the slave store is used. Robust
        # processes will handle this and retry (even if just means exit
        # and wait for the next scheduled invocation).
        self.assertRaises(DisconnectionError, slave_store.get, Person, 1)

        transaction.abort()

        # But in the next transaction, we get the master Store if we ask
        # for the slave Store so we can continue.
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)

        self.assertIs(master_store, slave_store)

    def test_slave_shutdown_between_transactions(self):
        """Slave is shutdown in between transactions."""
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIsNot(master_store, slave_store)

        transaction.abort()
        self.pgbouncer_fixture.stop()

        # The process doesn't notice the slave going down, and things
        # will fail the next time the slave is used.
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIsNot(master_store, slave_store)
        self.assertRaises(DisconnectionError, slave_store.get, Person, 1)

        # But now it has been discovered the socket is no longer
        # connected to anything, next transaction we get a master
        # Store when we ask for a slave.
        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIs(master_store, slave_store)

    def test_slave_reconnect_after_outage(self):
        """The slave is again used once it becomes available."""
        self.pgbouncer_fixture.stop()

        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIs(master_store, slave_store)

        self.pgbouncer_fixture.start()
        transaction.abort()

        master_store = IMasterStore(Person)
        slave_store = ISlaveStore(Person)
        self.assertIsNot(master_store, slave_store)