def test_run_with_disconnection_error_retries(self):
"""
If the given function raises a L{DisconnectionError}, then the
function will be retried another two times before letting the exception
bubble up.
"""
zstorm = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def _check_disconnect(self, function, *args, **kwargs):
"""Run the given function, checking for database disconnections."""
try:
return function(*args, **kwargs)
except Error, exc:
if self.is_disconnection_error(exc):
self._state = STATE_DISCONNECTED
self._raw_connection = None
raise DisconnectionError(str(exc))
else:
raise
def _ensure_connected(self):
"""Ensure that we are connected to the database.
If the connection is marked as dead, or if we can't reconnect,
then raise DisconnectionError.
"""
if self._blocked:
raise ConnectionBlockedError("Access to connection is blocked")
if self._state == STATE_CONNECTED:
return
elif self._state == STATE_DISCONNECTED:
raise DisconnectionError("Already disconnected")
elif self._state == STATE_RECONNECT:
try:
self._raw_connection = self._database.raw_connect()
except DatabaseError as exc:
self._state = STATE_DISCONNECTED
self._raw_connection = None
raise DisconnectionError(str(exc))
else:
self._state = STATE_CONNECTED
def test_run_with_disconnection_error_in_execute_is_ignored(self):
"""
If the given function raises a L{DisconnectionError}, then a C{SELECT
1} will be executed in each registered store such that C{psycopg}
actually detects the disconnection. If another L{DisconnectionError}
happens during C{execute}, then it is ignored.
"""
self.transactor.retries = 0
zstorm = self.mocker.mock()
store1 = self.mocker.mock()
store2 = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.mocker.order()
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(
iter((("store1", store1), ("store2", store2))))
self.expect(store1.execute("SELECT 1")).throw(DisconnectionError())
self.expect(store2.execute("SELECT 1"))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def _check_disconnect(self, function, *args, **kwargs):
"""Run the given function, checking for database disconnections."""
# Allow the caller to specify additional exception types that
# should be treated as possible disconnection errors.
extra_disconnection_errors = kwargs.pop('extra_disconnection_errors',
())
try:
return function(*args, **kwargs)
except Exception as exc:
if self.is_disconnection_error(exc, extra_disconnection_errors):
self._state = STATE_DISCONNECTED
self._raw_connection = None
raise DisconnectionError(str(exc))
else:
raise
def test_disconnect_logs_oops(self):
# Ensure that OOPS reports are generated for database
# disconnections, as per Bug #373837.
request = LaunchpadTestRequest()
publication = WebServicePublication(None)
dbadapter.set_request_started()
try:
raise DisconnectionError('Fake')
except DisconnectionError:
self.assertRaises(Retry, publication.handleException, None,
request, sys.exc_info(), True)
dbadapter.clear_request_started()
self.assertEqual(1, len(self.oopses))
oops = self.oopses[0]
# Ensure the OOPS mentions the correct exception
self.assertEqual(oops['type'], "DisconnectionError")
def get_connected_store(name, flavor):
"""Retrieve a store from the IZStorm Utility and ensure it is connected.
:raises storm.exceptions.DisconnectionError: On failures.
"""
store_name = '%s-%s' % (name, flavor)
try:
store = getUtility(IZStorm).get(store_name,
'launchpad:%s' % store_name)
store._connection._ensure_connected()
return store
except DisconnectionError:
# If the Store is in a disconnected state, ensure it is
# registered with the transaction manager. Otherwise, if
# _ensure_connected() caused the disconnected state it may not
# be put into reconnect state at the end of the transaction.
store._connection._event.emit('register-transaction')
raise
except psycopg2.OperationalError, exc:
# Per Bug #1025264, Storm emits psycopg2 errors when we
# want DisconnonnectionErrors, eg. attempting to open a
# new connection to a non-existent database.
raise DisconnectionError(str(exc))
def function():
if not self.flag:
self.flag = True
raise DisconnectionError('Disconnected')
return 'success'
def test_disconnectionerror_view(self):
request = LaunchpadTestRequest()
DisconnectionErrorView(DisconnectionError(), request)
self.assertEquals(503, request.response.getStatus())
