def setUp(self):
super(MasterFallbackTestCase, self).setUp()
self.pgbouncer_fixture = PGBouncerFixture()
# The PGBouncerFixture will set the PGPORT environment variable,
# causing all DB connections to go via pgbouncer unless an
# explicit port is provided.
dbname = DatabaseLayer._db_fixture.dbname
# Pull the direct db connection string, including explicit port.
conn_str_direct = self.pgbouncer_fixture.databases[dbname]
# Generate a db connection string that will go via pgbouncer.
conn_str_pgbouncer = 'dbname=%s host=localhost' % dbname
# Configure slave connections via pgbouncer, so we can shut them
# down. Master connections direct so they are unaffected.
config_key = 'master-slave-separation'
config.push(
config_key,
dedent('''\
[database]
rw_main_master: %s
rw_main_slave: %s
''' % (conn_str_direct, conn_str_pgbouncer)))
self.addCleanup(lambda: config.pop(config_key))
self.useFixture(self.pgbouncer_fixture)
def test_disconnectionerror_view_integration(self):
# Test setup.
self.useFixture(Urllib2Fixture())
bouncer = PGBouncerFixture()
# XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
# In parallel tests, we are rarely encountering instances of
# bug 504291 while running this test. These cause the tests
# to fail entirely (the store.rollback() described in comment
# 11 does not fix the insane state) despite nultiple retries.
# As mentioned in that bug, we are trying aborts to see if they
# eliminate the problem. If this works, we can find which of
# these two aborts are actually needed.
transaction.abort()
self.useFixture(bouncer)
transaction.abort()
# Verify things are working initially.
url = "http://launchpad.dev/"
self.retryConnection(url, bouncer)
# Now break the database, and we get an exception, along with
# our view.
bouncer.stop()
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
# We keep seeing the correct exception on subsequent requests.
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
# When the database is available again, requests succeed.
bouncer.start()
self.retryConnection(url, bouncer)
def setUp(self):
super(TestFastDowntimeRollout, self).setUp()
self.master_dbname = DatabaseLayer._db_fixture.dbname
self.slave_dbname = self.master_dbname + '_slave'
self.pgbouncer_fixture = PGBouncerFixture()
self.pgbouncer_fixture.databases[self.slave_dbname] = (
self.pgbouncer_fixture.databases[self.master_dbname])
# Configure master and slave connections to go via different
# pgbouncer aliases.
config_key = 'master-slave-separation'
config.push(
config_key,
dedent('''\
[database]
rw_main_master: dbname=%s host=localhost
rw_main_slave: dbname=%s host=localhost
''' % (self.master_dbname, self.slave_dbname)))
self.addCleanup(lambda: config.pop(config_key))
self.useFixture(self.pgbouncer_fixture)
self.pgbouncer_con = psycopg2.connect(
'dbname=pgbouncer user=pgbouncer host=localhost')
self.pgbouncer_con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
self.pgbouncer_cur = self.pgbouncer_con.cursor()
transaction.abort()
def setUp(self):
super(MasterFallbackTestCase, self).setUp()
self.pgbouncer_fixture = PGBouncerFixture()
# The PGBouncerFixture will set the PGPORT environment variable,
# causing all DB connections to go via pgbouncer unless an
# explicit port is provided.
dbname = DatabaseLayer._db_fixture.dbname
# Pull the direct db connection string, including explicit port.
conn_str_direct = self.pgbouncer_fixture.databases[dbname]
# Generate a db connection string that will go via pgbouncer.
conn_str_pgbouncer = "dbname=%s host=localhost" % dbname
# Configure slave connections via pgbouncer, so we can shut them
# down. Master connections direct so they are unaffected.
config_key = "master-slave-separation"
config.push(
config_key,
dedent(
"""\
[database]
rw_main_master: %s
rw_main_slave: %s
"""
% (conn_str_direct, conn_str_pgbouncer)
),
)
self.addCleanup(lambda: config.pop(config_key))
self.useFixture(self.pgbouncer_fixture)
def test_install_fixture(self):
self.assertTrue(self.is_db_available())
with PGBouncerFixture() as pgbouncer:
self.assertTrue(self.is_db_available())
pgbouncer.stop()
self.assertFalse(self.is_db_available())
# This confirms that we are again connecting directly to the
# database, as the pgbouncer process was shutdown.
self.assertTrue(self.is_db_available())
def test_stop_and_start(self):
# Database is working.
assert self.is_connected()
# And database with the fixture is working too.
pgbouncer = PGBouncerFixture()
with PGBouncerFixture() as pgbouncer:
assert self.is_connected()
# pgbouncer is transparant. To confirm we are connecting via
# pgbouncer, we need to shut it down and confirm our
# connections are dropped.
pgbouncer.stop()
assert not self.is_connected()
# If we restart it, things should be back to normal.
pgbouncer.start()
assert self.is_connected()
# Database is still working.
assert self.is_connected()
def test_disconnectionerror_view_integration(self):
# Test setup.
self.useFixture(Urllib2Fixture())
bouncer = PGBouncerFixture()
# XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
# In parallel tests, we are rarely encountering instances of
# bug 504291 while running this test. These cause the tests
# to fail entirely (the store.rollback() described in comment
# 11 does not fix the insane state) despite nultiple retries.
# As mentioned in that bug, we are trying aborts to see if they
# eliminate the problem. If this works, we can find which of
# these two aborts are actually needed.
transaction.abort()
self.useFixture(bouncer)
transaction.abort()
# Verify things are working initially.
url = 'http://launchpad.dev/'
self.retryConnection(url, bouncer)
# Now break the database, and we get an exception, along with
# our view.
bouncer.stop()
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
# We keep seeing the correct exception on subsequent requests.
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(), Contains(DisconnectionErrorView.reason))
# When the database is available again, requests succeed.
bouncer.start()
self.retryConnection(url, bouncer)
def test_install_fixture_with_restart(self):
self.assertTrue(self.is_db_available())
with PGBouncerFixture() as pgbouncer:
self.assertTrue(self.is_db_available())
pgbouncer.stop()
self.assertFalse(self.is_db_available())
pgbouncer.start()
self.assertTrue(self.is_db_available())
# Note that because pgbouncer was left running, we can't confirm
# that we are now connecting directly to the database.
self.assertTrue(self.is_db_available())
def setUp(cls):
# Fixture to hold other fixtures.
cls._fixture = Fixture()
cls._fixture.setUp()
cls.pgbouncer_fixture = PGBouncerFixture()
# Install the PGBouncer fixture so we shut it down to
# create database outages.
cls._fixture.useFixture(cls.pgbouncer_fixture)
# Bring up the Librarian, which will be connecting via
# pgbouncer.
cls.librarian_fixture = LibrarianServerFixture(
BaseLayer.config_fixture)
cls._fixture.useFixture(cls.librarian_fixture)
def test_stop_no_start(self):
# Database is working.
assert self.is_connected()
# And database with the fixture is working too.
with PGBouncerFixture() as pgbouncer:
assert self.is_connected()
# pgbouncer is transparant. To confirm we are connecting via
# pgbouncer, we need to shut it down and confirm our
# connections are dropped.
pgbouncer.stop()
assert not self.is_connected()
# Database is working again.
assert self.is_connected()
def test_starts_with_db_down(self):
pgbouncer = self.useFixture(PGBouncerFixture())
# Start with the database down.
pgbouncer.stop()
self.spawn()
for count in range(5):
out = self.request('foo')
self.assertEqual(out, 'NULL')
pgbouncer.start()
out = self.request('foo')
self.assertEndsWith(out, '/foo')
def test_stop_and_start(self):
# Database is working.
assert self.is_connected()
# And database with the fixture is working too.
pgbouncer = PGBouncerFixture()
with PGBouncerFixture() as pgbouncer:
assert self.is_connected()
# pgbouncer is transparant. To confirm we are connecting via
# pgbouncer, we need to shut it down and confirm our
# connections are dropped.
pgbouncer.stop()
assert not self.is_connected()
# If we restart it, things should be back to normal.
pgbouncer.start()
assert self.is_connected()
# Database is still working.
assert self.is_connected()
def test_reconnects_when_disconnected(self):
pgbouncer = self.useFixture(PGBouncerFixture())
self.spawn()
# Everything should be working, and we get valid output.
out = self.request('foo')
self.assertEndsWith(out, '/foo')
pgbouncer.stop()
# Now with pgbouncer down, we should get NULL messages and
# stderr spam, and this keeps happening. We test more than
# once to ensure that we will keep trying to reconnect even
# after several failures.
for count in range(5):
out = self.request('foo')
self.assertEqual(out, 'NULL')
pgbouncer.start()
# Everything should be working, and we get valid output.
out = self.request('foo')
self.assertEndsWith(out, '/foo')
def test_disconnectionerror_view_integration(self):
# Test setup.
self.useFixture(FakeLogger('SiteError', level=logging.CRITICAL))
self.useFixture(Urllib2Fixture())
bouncer = PGBouncerFixture()
# XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
# In parallel tests, we are rarely encountering instances of
# bug 504291 while running this test. These cause the tests
# to fail entirely (the store.rollback() described in comment
# 11 does not fix the insane state) despite nultiple retries.
# As mentioned in that bug, we are trying aborts to see if they
# eliminate the problem. If this works, we can find which of
# these two aborts are actually needed.
transaction.abort()
self.useFixture(bouncer)
transaction.abort()
# Verify things are working initially.
url = 'http://launchpad.dev/'
self.retryConnection(url, bouncer)
# Now break the database, and we get an exception, along with
# our view and several OOPSes from the retries.
bouncer.stop()
class Disconnects(Equals):
def __init__(self, message):
super(Disconnects, self).__init__(
('DisconnectionError', message))
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise(
[MatchesAny(
# libpq < 9.5.
Disconnects('error with no message from the libpq'),
# libpq >= 9.5.
Disconnects('server closed the connection unexpectedly'))]
* 2 +
[Disconnects(
'could not connect to server: Connection refused')]
* 6))
# We keep seeing the correct exception on subsequent requests.
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise(
[Disconnects(
'could not connect to server: Connection refused')]
* 8))
# When the database is available again, requests succeed.
bouncer.start()
self.retryConnection(url, bouncer)
# If we ask pgbouncer to disable the database, requests fail and
# get the same error page, but we don't log OOPSes except for
# the initial connection terminations. Disablement is always
# explicit maintenance, and we don't need lots of ongoing OOPSes
# to tell us about maintenance that we're doing.
dbname = DatabaseLayer._db_fixture.dbname
conn = psycopg2.connect("dbname=pgbouncer")
conn.autocommit = True
cur = conn.cursor()
cur.execute("DISABLE " + dbname)
cur.execute("KILL " + dbname)
cur.execute("RESUME " + dbname)
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise([Disconnects('database removed')]))
# A second request doesn't log any OOPSes.
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertEqual(
[],
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses])
# When the database is available again, requests succeed.
cur.execute("ENABLE %s" % DatabaseLayer._db_fixture.dbname)
self.retryConnection(url, bouncer)
class MasterFallbackTestCase(TestCase):
layer = DatabaseFunctionalLayer
def setUp(self):
super(MasterFallbackTestCase, self).setUp()
self.pgbouncer_fixture = PGBouncerFixture()
# The PGBouncerFixture will set the PGPORT environment variable,
# causing all DB connections to go via pgbouncer unless an
# explicit port is provided.
dbname = DatabaseLayer._db_fixture.dbname
# Pull the direct db connection string, including explicit port.
conn_str_direct = self.pgbouncer_fixture.databases[dbname]
# Generate a db connection string that will go via pgbouncer.
conn_str_pgbouncer = 'dbname=%s host=localhost' % dbname
# Configure slave connections via pgbouncer, so we can shut them
# down. Master connections direct so they are unaffected.
config_key = 'master-slave-separation'
config.push(
config_key,
dedent('''\
[database]
rw_main_master: %s
rw_main_slave: %s
''' % (conn_str_direct, conn_str_pgbouncer)))
self.addCleanup(lambda: config.pop(config_key))
self.useFixture(self.pgbouncer_fixture)
def test_can_shutdown_slave_only(self):
'''Confirm that this TestCase's test infrastructure works as needed.
'''
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
# Both Stores work when pgbouncer is up.
master_store.get(Person, 1)
slave_store.get(Person, 1)
# Slave Store breaks when pgbouncer is torn down. Master Store
# is fine.
self.pgbouncer_fixture.stop()
master_store.get(Person, 2)
self.assertRaises(DisconnectionError, slave_store.get, Person, 2)
def test_startup_with_no_slave(self):
'''An attempt is made for the first time to connect to a slave.'''
self.pgbouncer_fixture.stop()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
# The master and slave Stores are the same object.
self.assertIs(master_store, slave_store)
def test_slave_shutdown_during_transaction(self):
'''Slave is shutdown while running, but we can recover.'''
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
self.pgbouncer_fixture.stop()
# The transaction fails if the slave store is used. Robust
# processes will handle this and retry (even if just means exit
# and wait for the next scheduled invocation).
self.assertRaises(DisconnectionError, slave_store.get, Person, 1)
transaction.abort()
# But in the next transaction, we get the master Store if we ask
# for the slave Store so we can continue.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
def test_slave_shutdown_between_transactions(self):
'''Slave is shutdown in between transactions.'''
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
transaction.abort()
self.pgbouncer_fixture.stop()
# The process doesn't notice the slave going down, and things
# will fail the next time the slave is used.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
self.assertRaises(DisconnectionError, slave_store.get, Person, 1)
# But now it has been discovered the socket is no longer
# connected to anything, next transaction we get a master
# Store when we ask for a slave.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
def test_slave_reconnect_after_outage(self):
'''The slave is again used once it becomes available.'''
self.pgbouncer_fixture.stop()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
self.pgbouncer_fixture.start()
transaction.abort()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
class MasterFallbackTestCase(TestCase):
layer = DatabaseFunctionalLayer
def setUp(self):
super(MasterFallbackTestCase, self).setUp()
self.pgbouncer_fixture = PGBouncerFixture()
# The PGBouncerFixture will set the PGPORT environment variable,
# causing all DB connections to go via pgbouncer unless an
# explicit port is provided.
dbname = DatabaseLayer._db_fixture.dbname
# Pull the direct db connection string, including explicit port.
conn_str_direct = self.pgbouncer_fixture.databases[dbname]
# Generate a db connection string that will go via pgbouncer.
conn_str_pgbouncer = "dbname=%s host=localhost" % dbname
# Configure slave connections via pgbouncer, so we can shut them
# down. Master connections direct so they are unaffected.
config_key = "master-slave-separation"
config.push(
config_key,
dedent(
"""\
[database]
rw_main_master: %s
rw_main_slave: %s
"""
% (conn_str_direct, conn_str_pgbouncer)
),
)
self.addCleanup(lambda: config.pop(config_key))
self.useFixture(self.pgbouncer_fixture)
def test_can_shutdown_slave_only(self):
"""Confirm that this TestCase's test infrastructure works as needed.
"""
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
# Both Stores work when pgbouncer is up.
master_store.get(Person, 1)
slave_store.get(Person, 1)
# Slave Store breaks when pgbouncer is torn down. Master Store
# is fine.
self.pgbouncer_fixture.stop()
master_store.get(Person, 2)
self.assertRaises(DisconnectionError, slave_store.get, Person, 2)
def test_startup_with_no_slave(self):
"""An attempt is made for the first time to connect to a slave."""
self.pgbouncer_fixture.stop()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
# The master and slave Stores are the same object.
self.assertIs(master_store, slave_store)
def test_slave_shutdown_during_transaction(self):
"""Slave is shutdown while running, but we can recover."""
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
self.pgbouncer_fixture.stop()
# The transaction fails if the slave store is used. Robust
# processes will handle this and retry (even if just means exit
# and wait for the next scheduled invocation).
self.assertRaises(DisconnectionError, slave_store.get, Person, 1)
transaction.abort()
# But in the next transaction, we get the master Store if we ask
# for the slave Store so we can continue.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
def test_slave_shutdown_between_transactions(self):
"""Slave is shutdown in between transactions."""
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
transaction.abort()
self.pgbouncer_fixture.stop()
# The process doesn't notice the slave going down, and things
# will fail the next time the slave is used.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
self.assertRaises(DisconnectionError, slave_store.get, Person, 1)
# But now it has been discovered the socket is no longer
# connected to anything, next transaction we get a master
# Store when we ask for a slave.
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
def test_slave_reconnect_after_outage(self):
"""The slave is again used once it becomes available."""
self.pgbouncer_fixture.stop()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIs(master_store, slave_store)
self.pgbouncer_fixture.start()
transaction.abort()
master_store = IMasterStore(Person)
slave_store = ISlaveStore(Person)
self.assertIsNot(master_store, slave_store)
