def test_oops_reporting(self):
capture = CaptureOops()
capture.setUp()
with one_loop_exception(self.runner):
self.runner._oneloop()
oops = capture.oopses[0]
capture.cleanUp()
self.assertEqual('T-mailman', oops['reporter'])
self.assertTrue(oops['id'].startswith('OOPS-'))
self.assertEqual('Exception', oops['type'])
self.assertEqual('Test exception handling.', oops['value'])
self.assertTrue(
oops['tb_text'].startswith('Traceback (most recent call last):'))
def test_successful_start_then_stop_logs_no_oops(self):
# Starting and stopping the lp-serve process leaves no OOPS.
capture = self.useFixture(CaptureOops())
process, transport = self.start_server_inet()
self.finish_lpserve_subprocess(process)
capture.sync()
self.assertEqual([], capture.oopses)
def test_run_200(self):
# A request that returns 200 is a success.
with CaptureOops() as oopses:
job, reqs = self.makeAndRunJob(response_status=200)
self.assertThat(
job,
MatchesStructure(
status=Equals(JobStatus.COMPLETED),
pending=Is(False),
successful=Is(True),
date_sent=Not(Is(None)),
error_message=Is(None),
json_data=ContainsDict(
{'result': MatchesAll(
KeysEqual('request', 'response'),
ContainsDict(
{'response': ContainsDict(
{'status_code': Equals(200)})}))})))
self.assertEqual(1, len(reqs))
self.assertEqual([
('POST', 'http://example.com/ep',
{'Content-Type': 'application/json',
'User-Agent': 'launchpad.dev-Webhooks/r%s' % (
versioninfo.revision),
'X-Launchpad-Event-Type': 'test',
'X-Launchpad-Delivery': str(job.job_id)}),
], reqs)
self.assertEqual([], oopses.oopses)
def test_subscribes_to_events(self):
capture = self.useFixture(CaptureOops())
publisher = globalErrorUtility._oops_config.publisher
try:
globalErrorUtility._oops_config.publisher = notify_publisher
id = globalErrorUtility.raising(sys.exc_info())['id']
self.assertEqual(id, capture.oopses[0]['id'])
self.assertEqual(1, len(capture.oopses))
finally:
globalErrorUtility._oops_config.publisher = publisher
def test_sync_grabs_pending_oopses(self):
factory = rabbit.connect
exchange = config.error_reports.error_exchange
routing_key = config.error_reports.error_queue_key
capture = self.useFixture(CaptureOops())
amqp_publisher = oops_amqp.Publisher(factory,
exchange,
routing_key,
inherit_id=True)
oops = {'id': 'fnor', 'foo': 'dr'}
self.assertEqual(['fnor'], amqp_publisher(oops))
oops2 = {'id': 'quux', 'foo': 'strangelove'}
self.assertEqual(['quux'], amqp_publisher(oops2))
capture.sync()
self.assertEqual([oops, oops2], capture.oopses)
def block_on_job(test_case=None):
with CaptureOops() as capture:
with monitor_celery() as responses:
yield
if len(responses) == 0:
raise Exception('No Job was requested to run via Celery.')
try:
responses[-1].wait(30)
finally:
if test_case is not None and responses[-1].traceback is not None:
test_case.addDetail('Worker traceback',
text_content(responses[-1].traceback))
if test_case is not None:
capture.sync()
for oops in capture.oopses:
test_case.addDetail('oops', text_content(str(oops)))
def test_run_signature(self):
# If the webhook has a secret, the request is signed in a
# PubSubHubbub-compatible way.
with CaptureOops() as oopses:
job, reqs = self.makeAndRunJob(
response_status=200, secret=u'sekrit')
self.assertEqual([
('POST', 'http://example.com/ep',
{'Content-Type': 'application/json',
'User-Agent': 'launchpad.dev-Webhooks/r%s' % (
versioninfo.revision),
'X-Hub-Signature':
'sha1=de75f136c37d89f5eb24834468c1ecd602fa95dd',
'X-Launchpad-Event-Type': 'test',
'X-Launchpad-Delivery': str(job.job_id)}),
], reqs)
self.assertEqual([], oopses.oopses)
def test_oops_reporting(self):
capture = CaptureOops()
capture.setUp()
with one_loop_exception(self.runner):
self.runner._oneloop()
oops = capture.oopses[0]
capture.cleanUp()
self.assertEqual('T-mailman', oops['reporter'])
self.assertTrue(oops['id'].startswith('OOPS-'))
self.assertEqual('Exception', oops['type'])
self.assertEqual('Test exception handling.', oops['value'])
self.assertTrue(
oops['tb_text'].startswith('Traceback (most recent call last):'))
def test_unexpected_error_logs_oops(self):
# If an unexpected error is raised in the plugin, then an OOPS is
# recorded.
capture = self.useFixture(CaptureOops())
process, transport = self.start_server_inet()
# This will trigger an error, because the XML-RPC server is not
# running, and any filesystem access tries to get at the XML-RPC
# server. If this *doesn'* raise, then the test is no longer valid and
# we need a new way of triggering errors in the smart server.
self.assertRaises(
errors.UnknownErrorFromSmartServer,
transport.list_dir, 'foo/bar/baz')
result = self.finish_lpserve_subprocess(process)
self.assertFinishedCleanly(result)
capture.sync()
self.assertEqual(1, len(capture.oopses))
self.assertEqual(
'[Errno 111] Connection refused', capture.oopses[0]['value'],
capture.oopses)
def test_run_inactive(self):
# A delivery for a webhook that has been deactivated immediately
# fails.
with CaptureOops() as oopses:
job, reqs = self.makeAndRunJob(
raises=requests.ConnectionError('Connection refused'),
active=False)
self.assertThat(
job,
MatchesStructure(
status=Equals(JobStatus.FAILED),
pending=Is(False),
successful=Is(False),
date_sent=Is(None),
error_message=Equals('Webhook deactivated'),
json_data=ContainsDict(
{'result': MatchesDict(
{'webhook_deactivated': Is(True)})})))
self.assertEqual([], reqs)
self.assertEqual([], oopses.oopses)
def test_run_no_proxy(self):
# Since users can cause the webhook runner to make somewhat
# controlled POST requests to arbitrary URLs, they're forced to
# go through a locked-down HTTP proxy. If none is configured,
# the job crashes.
self.pushConfig('webhooks', http_proxy=None)
with CaptureOops() as oopses:
job, reqs = self.makeAndRunJob(response_status=200, mock=False)
self.assertThat(
job,
MatchesStructure(
status=Equals(JobStatus.FAILED),
pending=Is(False),
successful=Is(None),
date_sent=Is(None),
error_message=Is(None),
json_data=Not(Contains('result'))))
self.assertEqual([], reqs)
self.assertEqual(1, len(oopses.oopses))
self.assertEqual(
'No webhook proxy configured.', oopses.oopses[0]['value'])
def test_run_404(self):
# A request that returns a non-2xx response is a failure and
# gets retried.
with CaptureOops() as oopses:
job, reqs = self.makeAndRunJob(response_status=404)
self.assertThat(
job,
MatchesStructure(
status=Equals(JobStatus.WAITING),
pending=Is(True),
successful=Is(False),
date_sent=Not(Is(None)),
error_message=Equals('Bad HTTP response: 404'),
json_data=ContainsDict(
{'result': MatchesAll(
KeysEqual('request', 'response'),
ContainsDict(
{'response': ContainsDict(
{'status_code': Equals(404)})}))})))
self.assertEqual(1, len(reqs))
self.assertEqual([], oopses.oopses)
def test_run_connection_error(self):
# Jobs that fail to connect have a connection_error rather than a
# response. They too trigger a retry.
with CaptureOops() as oopses:
job, reqs = self.makeAndRunJob(
raises=requests.ConnectionError('Connection refused'))
self.assertThat(
job,
MatchesStructure(
status=Equals(JobStatus.WAITING),
pending=Is(True),
successful=Is(False),
date_sent=Not(Is(None)),
error_message=Equals('Connection error: Connection refused'),
json_data=ContainsDict(
{'result': MatchesAll(
KeysEqual('request', 'connection_error'),
ContainsDict(
{'connection_error': Equals('Connection refused')})
)})))
self.assertEqual([], reqs)
self.assertEqual([], oopses.oopses)
def test_sync_twice_works(self):
capture = self.useFixture(CaptureOops())
capture.sync()
capture.sync()
def test_no_oopses_no_hang_on_sync(self):
capture = self.useFixture(CaptureOops())
capture.sync()
def test_disconnectionerror_view_integration(self):
# Test setup.
self.useFixture(FakeLogger('SiteError', level=logging.CRITICAL))
self.useFixture(Urllib2Fixture())
bouncer = PGBouncerFixture()
# XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03:
# In parallel tests, we are rarely encountering instances of
# bug 504291 while running this test. These cause the tests
# to fail entirely (the store.rollback() described in comment
# 11 does not fix the insane state) despite nultiple retries.
# As mentioned in that bug, we are trying aborts to see if they
# eliminate the problem. If this works, we can find which of
# these two aborts are actually needed.
transaction.abort()
self.useFixture(bouncer)
transaction.abort()
# Verify things are working initially.
url = 'http://launchpad.dev/'
self.retryConnection(url, bouncer)
# Now break the database, and we get an exception, along with
# our view and several OOPSes from the retries.
bouncer.stop()
class Disconnects(Equals):
def __init__(self, message):
super(Disconnects, self).__init__(
('DisconnectionError', message))
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise(
[MatchesAny(
# libpq < 9.5.
Disconnects('error with no message from the libpq'),
# libpq >= 9.5.
Disconnects('server closed the connection unexpectedly'))]
* 2 +
[Disconnects(
'could not connect to server: Connection refused')]
* 6))
# We keep seeing the correct exception on subsequent requests.
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise(
[Disconnects(
'could not connect to server: Connection refused')]
* 8))
# When the database is available again, requests succeed.
bouncer.start()
self.retryConnection(url, bouncer)
# If we ask pgbouncer to disable the database, requests fail and
# get the same error page, but we don't log OOPSes except for
# the initial connection terminations. Disablement is always
# explicit maintenance, and we don't need lots of ongoing OOPSes
# to tell us about maintenance that we're doing.
dbname = DatabaseLayer._db_fixture.dbname
conn = psycopg2.connect("dbname=pgbouncer")
conn.autocommit = True
cur = conn.cursor()
cur.execute("DISABLE " + dbname)
cur.execute("KILL " + dbname)
cur.execute("RESUME " + dbname)
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertThat(
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses],
MatchesListwise([Disconnects('database removed')]))
# A second request doesn't log any OOPSes.
with CaptureOops() as oopses:
error = self.getHTTPError(url)
self.assertEqual(503, error.code)
self.assertThat(error.read(),
Contains(DisconnectionErrorView.reason))
self.assertEqual(
[],
[(oops['type'], oops['value'].split('\n')[0])
for oops in oopses.oopses])
# When the database is available again, requests succeed.
cur.execute("ENABLE %s" % DatabaseLayer._db_fixture.dbname)
self.retryConnection(url, bouncer)