def test_disconnectionerror_view_integration(self):
# Test setup.
self.useFixture(Urllib2Fixture())
bouncer = PGBouncerFixture()
# XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
# In parallel tests, we are rarely encountering instances of
# bug 504291 while running this test. These cause the tests
# to fail entirely (the store.rollback() described in comment
# 11 does not fix the insane state) despite nultiple retries.
# As mentioned in that bug, we are trying aborts to see if they
# eliminate the problem. If this works, we can find which of
# these two aborts are actually needed.
transaction.abort()
self.useFixture(bouncer)
transaction.abort()
# Verify things are working initially.
url = 'http://launchpad.dev/'
self.retryConnection(url, bouncer)
# Now break the database, and we get an exception, along with
# our view.
bouncer.stop()
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
# We keep seeing the correct exception on subsequent requests.
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
# When the database is available again, requests succeed.
bouncer.start()
self.retryConnection(url, bouncer)
def test_disconnectionerror_view_integration(self):
# Test setup.
self.useFixture(Urllib2Fixture())
bouncer = PGBouncerFixture()
# XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
# In parallel tests, we are rarely encountering instances of
# bug 504291 while running this test. These cause the tests
# to fail entirely (the store.rollback() described in comment
# 11 does not fix the insane state) despite nultiple retries.
# As mentioned in that bug, we are trying aborts to see if they
# eliminate the problem. If this works, we can find which of
# these two aborts are actually needed.
transaction.abort()
self.useFixture(bouncer)
transaction.abort()
# Verify things are working initially.
url = "http://launchpad.dev/"
self.retryConnection(url, bouncer)
# Now break the database, and we get an exception, along with
# our view.
bouncer.stop()
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
# We keep seeing the correct exception on subsequent requests.
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
# When the database is available again, requests succeed.
bouncer.start()
self.retryConnection(url, bouncer)
def test_stop_and_start(self):
# Database is working.
assert self.is_connected()
# And database with the fixture is working too.
pgbouncer = PGBouncerFixture()
with PGBouncerFixture() as pgbouncer:
assert self.is_connected()
# pgbouncer is transparant. To confirm we are connecting via
# pgbouncer, we need to shut it down and confirm our
# connections are dropped.
pgbouncer.stop()
assert not self.is_connected()
# If we restart it, things should be back to normal.
pgbouncer.start()
assert self.is_connected()
# Database is still working.
assert self.is_connected()
def test_stop_and_start(self):
# Database is working.
assert self.is_connected()
# And database with the fixture is working too.
pgbouncer = PGBouncerFixture()
with PGBouncerFixture() as pgbouncer:
assert self.is_connected()
# pgbouncer is transparant. To confirm we are connecting via
# pgbouncer, we need to shut it down and confirm our
# connections are dropped.
pgbouncer.stop()
assert not self.is_connected()
# If we restart it, things should be back to normal.
pgbouncer.start()
assert self.is_connected()
# Database is still working.
assert self.is_connected()
def test_disconnectionerror_view_integration(self):
# Test setup.
self.useFixture(FakeLogger('SiteError', level=logging.CRITICAL))
self.useFixture(Urllib2Fixture())
bouncer = PGBouncerFixture()
# XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
# In parallel tests, we are rarely encountering instances of
# bug 504291 while running this test. These cause the tests
# to fail entirely (the store.rollback() described in comment
# 11 does not fix the insane state) despite nultiple retries.
# As mentioned in that bug, we are trying aborts to see if they
# eliminate the problem. If this works, we can find which of
# these two aborts are actually needed.
transaction.abort()
self.useFixture(bouncer)
transaction.abort()
# Verify things are working initially.
url = 'http://launchpad.dev/'
self.retryConnection(url, bouncer)
# Now break the database, and we get an exception, along with
# our view and several OOPSes from the retries.
bouncer.stop()
class Disconnects(Equals):
def __init__(self, message):
super(Disconnects, self).__init__(
('DisconnectionError', message))
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise(
[MatchesAny(
# libpq < 9.5.
Disconnects('error with no message from the libpq'),
# libpq >= 9.5.
Disconnects('server closed the connection unexpectedly'))]
* 2 +
[Disconnects(
'could not connect to server: Connection refused')]
* 6))
# We keep seeing the correct exception on subsequent requests.
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise(
[Disconnects(
'could not connect to server: Connection refused')]
* 8))
# When the database is available again, requests succeed.
bouncer.start()
self.retryConnection(url, bouncer)
# If we ask pgbouncer to disable the database, requests fail and
# get the same error page, but we don't log OOPSes except for
# the initial connection terminations. Disablement is always
# explicit maintenance, and we don't need lots of ongoing OOPSes
# to tell us about maintenance that we're doing.
dbname = DatabaseLayer._db_fixture.dbname
conn = psycopg2.connect("dbname=pgbouncer")
conn.autocommit = True
cur = conn.cursor()
cur.execute("DISABLE " + dbname)
cur.execute("KILL " + dbname)
cur.execute("RESUME " + dbname)
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise([Disconnects('database removed')]))
# A second request doesn't log any OOPSes.
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertEqual(
[],
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses])
# When the database is available again, requests succeed.
cur.execute("ENABLE %s" % DatabaseLayer._db_fixture.dbname)
self.retryConnection(url, bouncer)
class MasterFallbackTestCase(TestCase):
layer = DatabaseFunctionalLayer
def setUp(self):
super(MasterFallbackTestCase, self).setUp()
self.pgbouncer_fixture = PGBouncerFixture()
# The PGBouncerFixture will set the PGPORT environment variable,
# causing all DB connections to go via pgbouncer unless an
# explicit port is provided.
dbname = DatabaseLayer._db_fixture.dbname
# Pull the direct db connection string, including explicit port.
conn_str_direct = self.pgbouncer_fixture.databases[dbname]
# Generate a db connection string that will go via pgbouncer.
conn_str_pgbouncer = 'dbname=%s host=localhost' % dbname
# Configure slave connections via pgbouncer, so we can shut them
# down. Master connections direct so they are unaffected.
config_key = 'master-slave-separation'
config.push(
config_key,
dedent('''\
[database]
rw_main_master: %s
rw_main_slave: %s
''' % (conn_str_direct, conn_str_pgbouncer)))
self.addCleanup(lambda: config.pop(config_key))
self.useFixture(self.pgbouncer_fixture)
def test_can_shutdown_slave_only(self):
'''Confirm that this TestCase's test infrastructure works as needed.
'''
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
# Both Stores work when pgbouncer is up.
master_store.get(Person, 1)
slave_store.get(Person, 1)
# Slave Store breaks when pgbouncer is torn down. Master Store
# is fine.
self.pgbouncer_fixture.stop()
master_store.get(Person, 2)
self.assertRaises(DisconnectionError, slave_store.get, Person, 2)
def test_startup_with_no_slave(self):
'''An attempt is made for the first time to connect to a slave.'''
self.pgbouncer_fixture.stop()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
# The master and slave Stores are the same object.
self.assertIs(master_store, slave_store)
def test_slave_shutdown_during_transaction(self):
'''Slave is shutdown while running, but we can recover.'''
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
self.pgbouncer_fixture.stop()
# The transaction fails if the slave store is used. Robust
# processes will handle this and retry (even if just means exit
# and wait for the next scheduled invocation).
self.assertRaises(DisconnectionError, slave_store.get, Person, 1)
transaction.abort()
# But in the next transaction, we get the master Store if we ask
# for the slave Store so we can continue.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
def test_slave_shutdown_between_transactions(self):
'''Slave is shutdown in between transactions.'''
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
transaction.abort()
self.pgbouncer_fixture.stop()
# The process doesn't notice the slave going down, and things
# will fail the next time the slave is used.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
self.assertRaises(DisconnectionError, slave_store.get, Person, 1)
# But now it has been discovered the socket is no longer
# connected to anything, next transaction we get a master
# Store when we ask for a slave.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
def test_slave_reconnect_after_outage(self):
'''The slave is again used once it becomes available.'''
self.pgbouncer_fixture.stop()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
self.pgbouncer_fixture.start()
transaction.abort()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
class MasterFallbackTestCase(TestCase):
layer = DatabaseFunctionalLayer
def setUp(self):
super(MasterFallbackTestCase, self).setUp()
self.pgbouncer_fixture = PGBouncerFixture()
# The PGBouncerFixture will set the PGPORT environment variable,
# causing all DB connections to go via pgbouncer unless an
# explicit port is provided.
dbname = DatabaseLayer._db_fixture.dbname
# Pull the direct db connection string, including explicit port.
conn_str_direct = self.pgbouncer_fixture.databases[dbname]
# Generate a db connection string that will go via pgbouncer.
conn_str_pgbouncer = "dbname=%s host=localhost" % dbname
# Configure slave connections via pgbouncer, so we can shut them
# down. Master connections direct so they are unaffected.
config_key = "master-slave-separation"
config.push(
config_key,
dedent(
"""\
[database]
rw_main_master: %s
rw_main_slave: %s
"""
% (conn_str_direct, conn_str_pgbouncer)
),
)
self.addCleanup(lambda: config.pop(config_key))
self.useFixture(self.pgbouncer_fixture)
def test_can_shutdown_slave_only(self):
"""Confirm that this TestCase's test infrastructure works as needed.
"""
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
# Both Stores work when pgbouncer is up.
master_store.get(Person, 1)
slave_store.get(Person, 1)
# Slave Store breaks when pgbouncer is torn down. Master Store
# is fine.
self.pgbouncer_fixture.stop()
master_store.get(Person, 2)
self.assertRaises(DisconnectionError, slave_store.get, Person, 2)
def test_startup_with_no_slave(self):
"""An attempt is made for the first time to connect to a slave."""
self.pgbouncer_fixture.stop()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
# The master and slave Stores are the same object.
self.assertIs(master_store, slave_store)
def test_slave_shutdown_during_transaction(self):
"""Slave is shutdown while running, but we can recover."""
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
self.pgbouncer_fixture.stop()
# The transaction fails if the slave store is used. Robust
# processes will handle this and retry (even if just means exit
# and wait for the next scheduled invocation).
self.assertRaises(DisconnectionError, slave_store.get, Person, 1)
transaction.abort()
# But in the next transaction, we get the master Store if we ask
# for the slave Store so we can continue.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
def test_slave_shutdown_between_transactions(self):
"""Slave is shutdown in between transactions."""
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
transaction.abort()
self.pgbouncer_fixture.stop()
# The process doesn't notice the slave going down, and things
# will fail the next time the slave is used.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
self.assertRaises(DisconnectionError, slave_store.get, Person, 1)
# But now it has been discovered the socket is no longer
# connected to anything, next transaction we get a master
# Store when we ask for a slave.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
def test_slave_reconnect_after_outage(self):
"""The slave is again used once it becomes available."""
self.pgbouncer_fixture.stop()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
self.pgbouncer_fixture.start()
transaction.abort()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)