def test_oops_reporting(self): capture = CaptureOops() capture.setUp() with one_loop_exception(self.runner): self.runner._oneloop() oops = capture.oopses[0] capture.cleanUp() self.assertEqual('T-mailman', oops['reporter']) self.assertTrue(oops['id'].startswith('OOPS-')) self.assertEqual('Exception', oops['type']) self.assertEqual('Test exception handling.', oops['value']) self.assertTrue( oops['tb_text'].startswith('Traceback (most recent call last):'))
def test_successful_start_then_stop_logs_no_oops(self): # Starting and stopping the lp-serve process leaves no OOPS. capture = self.useFixture(CaptureOops()) process, transport = self.start_server_inet() self.finish_lpserve_subprocess(process) capture.sync() self.assertEqual([], capture.oopses)
def test_run_200(self): # A request that returns 200 is a success. with CaptureOops() as oopses: job, reqs = self.makeAndRunJob(response_status=200) self.assertThat( job, MatchesStructure( status=Equals(JobStatus.COMPLETED), pending=Is(False), successful=Is(True), date_sent=Not(Is(None)), error_message=Is(None), json_data=ContainsDict( {'result': MatchesAll( KeysEqual('request', 'response'), ContainsDict( {'response': ContainsDict( {'status_code': Equals(200)})}))}))) self.assertEqual(1, len(reqs)) self.assertEqual([ ('POST', 'http://example.com/ep', {'Content-Type': 'application/json', 'User-Agent': 'launchpad.dev-Webhooks/r%s' % ( versioninfo.revision), 'X-Launchpad-Event-Type': 'test', 'X-Launchpad-Delivery': str(job.job_id)}), ], reqs) self.assertEqual([], oopses.oopses)
def test_subscribes_to_events(self): capture = self.useFixture(CaptureOops()) publisher = globalErrorUtility._oops_config.publisher try: globalErrorUtility._oops_config.publisher = notify_publisher id = globalErrorUtility.raising(sys.exc_info())['id'] self.assertEqual(id, capture.oopses[0]['id']) self.assertEqual(1, len(capture.oopses)) finally: globalErrorUtility._oops_config.publisher = publisher
def test_sync_grabs_pending_oopses(self): factory = rabbit.connect exchange = config.error_reports.error_exchange routing_key = config.error_reports.error_queue_key capture = self.useFixture(CaptureOops()) amqp_publisher = oops_amqp.Publisher(factory, exchange, routing_key, inherit_id=True) oops = {'id': 'fnor', 'foo': 'dr'} self.assertEqual(['fnor'], amqp_publisher(oops)) oops2 = {'id': 'quux', 'foo': 'strangelove'} self.assertEqual(['quux'], amqp_publisher(oops2)) capture.sync() self.assertEqual([oops, oops2], capture.oopses)
def block_on_job(test_case=None): with CaptureOops() as capture: with monitor_celery() as responses: yield if len(responses) == 0: raise Exception('No Job was requested to run via Celery.') try: responses[-1].wait(30) finally: if test_case is not None and responses[-1].traceback is not None: test_case.addDetail('Worker traceback', text_content(responses[-1].traceback)) if test_case is not None: capture.sync() for oops in capture.oopses: test_case.addDetail('oops', text_content(str(oops)))
def test_run_signature(self): # If the webhook has a secret, the request is signed in a # PubSubHubbub-compatible way. with CaptureOops() as oopses: job, reqs = self.makeAndRunJob( response_status=200, secret=u'sekrit') self.assertEqual([ ('POST', 'http://example.com/ep', {'Content-Type': 'application/json', 'User-Agent': 'launchpad.dev-Webhooks/r%s' % ( versioninfo.revision), 'X-Hub-Signature': 'sha1=de75f136c37d89f5eb24834468c1ecd602fa95dd', 'X-Launchpad-Event-Type': 'test', 'X-Launchpad-Delivery': str(job.job_id)}), ], reqs) self.assertEqual([], oopses.oopses)
def test_unexpected_error_logs_oops(self): # If an unexpected error is raised in the plugin, then an OOPS is # recorded. capture = self.useFixture(CaptureOops()) process, transport = self.start_server_inet() # This will trigger an error, because the XML-RPC server is not # running, and any filesystem access tries to get at the XML-RPC # server. If this *doesn'* raise, then the test is no longer valid and # we need a new way of triggering errors in the smart server. self.assertRaises( errors.UnknownErrorFromSmartServer, transport.list_dir, 'foo/bar/baz') result = self.finish_lpserve_subprocess(process) self.assertFinishedCleanly(result) capture.sync() self.assertEqual(1, len(capture.oopses)) self.assertEqual( '[Errno 111] Connection refused', capture.oopses[0]['value'], capture.oopses)
def test_run_inactive(self): # A delivery for a webhook that has been deactivated immediately # fails. with CaptureOops() as oopses: job, reqs = self.makeAndRunJob( raises=requests.ConnectionError('Connection refused'), active=False) self.assertThat( job, MatchesStructure( status=Equals(JobStatus.FAILED), pending=Is(False), successful=Is(False), date_sent=Is(None), error_message=Equals('Webhook deactivated'), json_data=ContainsDict( {'result': MatchesDict( {'webhook_deactivated': Is(True)})}))) self.assertEqual([], reqs) self.assertEqual([], oopses.oopses)
def test_run_no_proxy(self): # Since users can cause the webhook runner to make somewhat # controlled POST requests to arbitrary URLs, they're forced to # go through a locked-down HTTP proxy. If none is configured, # the job crashes. self.pushConfig('webhooks', http_proxy=None) with CaptureOops() as oopses: job, reqs = self.makeAndRunJob(response_status=200, mock=False) self.assertThat( job, MatchesStructure( status=Equals(JobStatus.FAILED), pending=Is(False), successful=Is(None), date_sent=Is(None), error_message=Is(None), json_data=Not(Contains('result')))) self.assertEqual([], reqs) self.assertEqual(1, len(oopses.oopses)) self.assertEqual( 'No webhook proxy configured.', oopses.oopses[0]['value'])
def test_run_404(self): # A request that returns a non-2xx response is a failure and # gets retried. with CaptureOops() as oopses: job, reqs = self.makeAndRunJob(response_status=404) self.assertThat( job, MatchesStructure( status=Equals(JobStatus.WAITING), pending=Is(True), successful=Is(False), date_sent=Not(Is(None)), error_message=Equals('Bad HTTP response: 404'), json_data=ContainsDict( {'result': MatchesAll( KeysEqual('request', 'response'), ContainsDict( {'response': ContainsDict( {'status_code': Equals(404)})}))}))) self.assertEqual(1, len(reqs)) self.assertEqual([], oopses.oopses)
def test_run_connection_error(self): # Jobs that fail to connect have a connection_error rather than a # response. They too trigger a retry. with CaptureOops() as oopses: job, reqs = self.makeAndRunJob( raises=requests.ConnectionError('Connection refused')) self.assertThat( job, MatchesStructure( status=Equals(JobStatus.WAITING), pending=Is(True), successful=Is(False), date_sent=Not(Is(None)), error_message=Equals('Connection error: Connection refused'), json_data=ContainsDict( {'result': MatchesAll( KeysEqual('request', 'connection_error'), ContainsDict( {'connection_error': Equals('Connection refused')}) )}))) self.assertEqual([], reqs) self.assertEqual([], oopses.oopses)
def test_sync_twice_works(self): capture = self.useFixture(CaptureOops()) capture.sync() capture.sync()
def test_no_oopses_no_hang_on_sync(self): capture = self.useFixture(CaptureOops()) capture.sync()
def test_disconnectionerror_view_integration(self): # Test setup. self.useFixture(FakeLogger('SiteError', level=logging.CRITICAL)) self.useFixture(Urllib2Fixture()) bouncer = PGBouncerFixture() # XXX gary bug=974617, bug=1011847, bug=504291 2011-07-03: # In parallel tests, we are rarely encountering instances of # bug 504291 while running this test. These cause the tests # to fail entirely (the store.rollback() described in comment # 11 does not fix the insane state) despite nultiple retries. # As mentioned in that bug, we are trying aborts to see if they # eliminate the problem. If this works, we can find which of # these two aborts are actually needed. transaction.abort() self.useFixture(bouncer) transaction.abort() # Verify things are working initially. url = 'http://launchpad.dev/' self.retryConnection(url, bouncer) # Now break the database, and we get an exception, along with # our view and several OOPSes from the retries. bouncer.stop() class Disconnects(Equals): def __init__(self, message): super(Disconnects, self).__init__( ('DisconnectionError', message)) with CaptureOops() as oopses: error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) self.assertThat( [(oops['type'], oops['value'].split('\n')[0]) for oops in oopses.oopses], MatchesListwise( [MatchesAny( # libpq < 9.5. Disconnects('error with no message from the libpq'), # libpq >= 9.5. Disconnects('server closed the connection unexpectedly'))] * 2 + [Disconnects( 'could not connect to server: Connection refused')] * 6)) # We keep seeing the correct exception on subsequent requests. with CaptureOops() as oopses: error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) self.assertThat( [(oops['type'], oops['value'].split('\n')[0]) for oops in oopses.oopses], MatchesListwise( [Disconnects( 'could not connect to server: Connection refused')] * 8)) # When the database is available again, requests succeed. bouncer.start() self.retryConnection(url, bouncer) # If we ask pgbouncer to disable the database, requests fail and # get the same error page, but we don't log OOPSes except for # the initial connection terminations. Disablement is always # explicit maintenance, and we don't need lots of ongoing OOPSes # to tell us about maintenance that we're doing. dbname = DatabaseLayer._db_fixture.dbname conn = psycopg2.connect("dbname=pgbouncer") conn.autocommit = True cur = conn.cursor() cur.execute("DISABLE " + dbname) cur.execute("KILL " + dbname) cur.execute("RESUME " + dbname) with CaptureOops() as oopses: error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) self.assertThat( [(oops['type'], oops['value'].split('\n')[0]) for oops in oopses.oopses], MatchesListwise([Disconnects('database removed')])) # A second request doesn't log any OOPSes. with CaptureOops() as oopses: error = self.getHTTPError(url) self.assertEqual(503, error.code) self.assertThat(error.read(), Contains(DisconnectionErrorView.reason)) self.assertEqual( [], [(oops['type'], oops['value'].split('\n')[0]) for oops in oopses.oopses]) # When the database is available again, requests succeed. cur.execute("ENABLE %s" % DatabaseLayer._db_fixture.dbname) self.retryConnection(url, bouncer)