def test_configure(self):
"""Test ErrorReportingUtility.setConfigSection()."""
utility = ErrorReportingUtility()
# The ErrorReportingUtility uses the config.error_reports section
# by default.
self.assertEqual(config.error_reports.oops_prefix,
utility.oops_prefix)
self.assertEqual(config.error_reports.error_dir,
utility._oops_datedir_repo.root)
# Some external processes may extend the reporter/prefix with
# extra information.
utility.configure(section_name='branchscanner')
self.assertEqual('T-branchscanner', utility.oops_prefix)
# The default error section can be restored.
utility.configure()
self.assertEqual(config.error_reports.oops_prefix,
utility.oops_prefix)
# We should have had three publishers setup:
oops_config = utility._oops_config
self.assertEqual(3, len(oops_config.publishers))
# - a rabbit publisher
self.assertIsInstance(oops_config.publishers[0], oops_amqp.Publisher)
# - a datedir publisher wrapped in a publish_new_only wrapper
datedir_repo = utility._oops_datedir_repo
publisher = oops_config.publishers[1].func_closure[0].cell_contents
self.assertEqual(publisher, datedir_repo.publish)
# - a notify publisher
self.assertEqual(oops_config.publishers[2], notify_publisher)
def make_error_utility(pid=None):
"""Make an error utility for logging errors from bzr."""
if pid is None:
pid = os.getpid()
error_utility = ErrorReportingUtility()
error_utility.configure('bzr_lpserve')
return error_utility
def test_raising_with_webservice_request(self):
# Test ErrorReportingUtility.raising() with a WebServiceRequest
# request. Only some exceptions result in OOPSes.
request = TestRequest()
directlyProvides(request, WebServiceLayer)
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
# Exceptions that don't use error_status result in OOPSes.
try:
raise ArbitraryException('xyz\nabc')
except ArbitraryException:
self.assertNotEqual(None,
utility.raising(sys.exc_info(), request))
# Exceptions with a error_status in the 500 range result
# in OOPSes.
@error_status(httplib.INTERNAL_SERVER_ERROR)
class InternalServerError(Exception):
pass
try:
raise InternalServerError("")
except InternalServerError:
self.assertNotEqual(None,
utility.raising(sys.exc_info(), request))
# Exceptions with any other error_status do not result
# in OOPSes.
@error_status(httplib.BAD_REQUEST)
class BadDataError(Exception):
pass
try:
raise BadDataError("")
except BadDataError:
self.assertEqual(None, utility.raising(sys.exc_info(), request))
def test_raising_with_webservice_request(self):
# Test ErrorReportingUtility.raising() with a WebServiceRequest
# request. Only some exceptions result in OOPSes.
request = TestRequest()
directlyProvides(request, WebServiceLayer)
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
# Exceptions that don't use error_status result in OOPSes.
try:
raise ArbitraryException('xyz\nabc')
except ArbitraryException:
self.assertNotEqual(None, utility.raising(sys.exc_info(), request))
# Exceptions with a error_status in the 500 range result
# in OOPSes.
@error_status(httplib.INTERNAL_SERVER_ERROR)
class InternalServerError(Exception):
pass
try:
raise InternalServerError("")
except InternalServerError:
self.assertNotEqual(None, utility.raising(sys.exc_info(), request))
# Exceptions with any other error_status do not result
# in OOPSes.
@error_status(httplib.BAD_REQUEST)
class BadDataError(Exception):
pass
try:
raise BadDataError("")
except BadDataError:
self.assertEqual(None, utility.raising(sys.exc_info(), request))
def make_error_utility(pid=None):
"""Make an error utility for logging errors from bzr."""
if pid is None:
pid = os.getpid()
error_utility = ErrorReportingUtility()
error_utility.configure('bzr_lpserve')
return error_utility
def test_raising_unauthorized_without_request(self):
"""Unauthorized exceptions are logged when there's no request."""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
try:
raise Unauthorized('xyz')
except Unauthorized:
oops = utility.raising(sys.exc_info())
self.assertNotEqual(None, oops)
def test_raising_unauthorized_without_request(self):
"""Unauthorized exceptions are logged when there's no request."""
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
try:
raise Unauthorized('xyz')
except Unauthorized:
oops = utility.raising(sys.exc_info())
self.assertNotEqual(None, oops)
def process(self):
"""Process an upload that is the result of a build.
The name of the leaf is the build id of the build.
Build uploads always contain a single package per leaf.
"""
logger = BufferLogger()
if self.build.status != BuildStatus.UPLOADING:
self.processor.log.warn(
"Expected build status to be 'UPLOADING', was %s. Ignoring." %
self.build.status.name)
return
try:
# The recipe may have been deleted so we need to flag that here
# and will handle below. We check so that we don't go to the
# expense of doing an unnecessary upload. We don't just exit here
# because we want the standard cleanup to occur.
recipe_deleted = (ISourcePackageRecipeBuild.providedBy(self.build)
and self.build.recipe is None)
if recipe_deleted:
result = UploadStatusEnum.FAILED
else:
self.processor.log.debug("Build %s found" % self.build.id)
[changes_file] = self.locateChangesFiles()
logger.debug("Considering changefile %s" % changes_file)
result = self.processChangesFile(changes_file, logger)
except (KeyboardInterrupt, SystemExit):
raise
except:
info = sys.exc_info()
message = (
'Exception while processing upload %s' % self.upload_path)
properties = [('error-explanation', message)]
request = ScriptRequest(properties)
error_utility = ErrorReportingUtility()
error_utility.raising(info, request)
logger.error('%s (%s)' % (message, request.oopsid))
result = UploadStatusEnum.FAILED
if (result != UploadStatusEnum.ACCEPTED or
not self.build.verifySuccessfulUpload()):
self.build.updateStatus(BuildStatus.FAILEDTOUPLOAD)
if self.build.status != BuildStatus.FULLYBUILT:
if recipe_deleted:
# For a deleted recipe, no need to notify that uploading has
# failed - we just log a warning.
self.processor.log.warn(
"Recipe for build %s was deleted. Ignoring." %
self.upload)
else:
self.build.storeUploadLog(logger.getLogBuffer())
self.build.notify(extra_info="Uploading build %s failed." %
self.upload)
else:
self.build.notify()
self.processor.ztm.commit()
self.moveProcessedUpload(result, logger)
def test_raising_unauthorized_with_authenticated_principal(self):
"""Unauthorized exceptions are logged when the request has an
authenticated principal."""
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
request = TestRequestWithPrincipal()
try:
raise Unauthorized('xyz')
except Unauthorized:
self.assertNotEqual(None, utility.raising(sys.exc_info(), request))
def test_raising_unauthorized_without_principal(self):
"""Unauthorized exceptions are logged when the request has no
principal."""
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
request = ScriptRequest([('name2', 'value2')])
try:
raise Unauthorized('xyz')
except Unauthorized:
self.assertNotEqual(None, utility.raising(sys.exc_info(), request))
def test_raising_unauthorized_with_unauthenticated_principal(self):
"""Unauthorized exceptions are not logged when the request has an
unauthenticated principal."""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
request = TestRequestWithUnauthenticatedPrincipal()
try:
raise Unauthorized('xyz')
except Unauthorized:
self.assertEqual(None, utility.raising(sys.exc_info(), request))
def process(self):
"""Process an upload that is the result of a build.
The name of the leaf is the build id of the build.
Build uploads always contain a single package per leaf.
"""
logger = BufferLogger()
if self.build.status != BuildStatus.UPLOADING:
self.processor.log.warn(
"Expected build status to be 'UPLOADING', was %s. Ignoring." %
self.build.status.name)
return
try:
# The recipe may have been deleted so we need to flag that here
# and will handle below. We check so that we don't go to the
# expense of doing an unnecessary upload. We don't just exit here
# because we want the standard cleanup to occur.
recipe_deleted = (ISourcePackageRecipeBuild.providedBy(self.build)
and self.build.recipe is None)
if recipe_deleted:
result = UploadStatusEnum.FAILED
else:
self.processor.log.debug("Build %s found" % self.build.id)
[changes_file] = self.locateChangesFiles()
logger.debug("Considering changefile %s" % changes_file)
result = self.processChangesFile(changes_file, logger)
except (KeyboardInterrupt, SystemExit):
raise
except:
info = sys.exc_info()
message = ('Exception while processing upload %s' %
self.upload_path)
properties = [('error-explanation', message)]
request = ScriptRequest(properties)
error_utility = ErrorReportingUtility()
error_utility.raising(info, request)
logger.error('%s (%s)' % (message, request.oopsid))
result = UploadStatusEnum.FAILED
if (result != UploadStatusEnum.ACCEPTED
or not self.build.verifySuccessfulUpload()):
self.build.updateStatus(BuildStatus.FAILEDTOUPLOAD)
if self.build.status != BuildStatus.FULLYBUILT:
if recipe_deleted:
# For a deleted recipe, no need to notify that uploading has
# failed - we just log a warning.
self.processor.log.warn(
"Recipe for build %s was deleted. Ignoring." % self.upload)
else:
self.build.storeUploadLog(logger.getLogBuffer())
self.build.notify(extra_info="Uploading build %s failed." %
self.upload)
else:
self.build.notify()
self.processor.ztm.commit()
self.moveProcessedUpload(result, logger)
def test_raising_unauthorized_without_principal(self):
"""Unauthorized exceptions are logged when the request has no
principal."""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
request = ScriptRequest([('name2', 'value2')])
try:
raise Unauthorized('xyz')
except Unauthorized:
self.assertNotEqual(None,
utility.raising(sys.exc_info(), request))
def test_ignored_exceptions_for_offsite_referer_not_reported(self):
# Oopses are not reported when Launchpad is not the referer.
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
# There is no HTTP_REFERER header in this request
request = TestRequest(
environ={'SERVER_URL': 'http://launchpad.dev/fnord'})
try:
raise GoneError('fnord')
except GoneError:
self.assertEqual(None, utility.raising(sys.exc_info(), request))
def test_ignored_exceptions_for_offsite_referer_not_reported(self):
# Oopses are not reported when Launchpad is not the referer.
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
# There is no HTTP_REFERER header in this request
request = TestRequest(
environ={'SERVER_URL': 'http://launchpad.dev/fnord'})
try:
raise GoneError('fnord')
except GoneError:
self.assertEqual(None, utility.raising(sys.exc_info(), request))
def test_raising_with_xmlrpc_request(self):
# Test ErrorReportingUtility.raising() with an XML-RPC request.
request = TestRequest()
directlyProvides(request, IXMLRPCRequest)
request.getPositionalArguments = lambda: (1, 2)
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
try:
raise ArbitraryException('xyz\nabc')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
self.assertEqual("(1, 2)", report['req_vars']['xmlrpc args'])
def test_raising_with_xmlrpc_request(self):
# Test ErrorReportingUtility.raising() with an XML-RPC request.
request = TestRequest()
directlyProvides(request, IXMLRPCRequest)
request.getPositionalArguments = lambda: (1, 2)
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
try:
raise ArbitraryException('xyz\nabc')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
self.assertEqual("(1, 2)", report['req_vars']['xmlrpc args'])
def test__makeErrorReport_includes_oops_messages(self):
"""The error report should include the oops messages."""
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
with utility.oopsMessage(dict(a='b', c='d')):
try:
raise ArbitraryException('foo')
except ArbitraryException:
info = sys.exc_info()
oops = utility._oops_config.create(dict(exc_info=info))
self.assertEqual({'<oops-message-0>': "{'a': 'b', 'c': 'd'}"},
oops['req_vars'])
def test_raising_no_referrer_error(self):
"""Test ErrorReportingUtility.raising() with a NoReferrerError
exception.
An OOPS is not recorded when a NoReferrerError exception is
raised.
"""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
try:
raise NoReferrerError('xyz')
except NoReferrerError:
self.assertEqual(None, utility.raising(sys.exc_info()))
def test__makeErrorReport_includes_oops_messages(self):
"""The error report should include the oops messages."""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
with utility.oopsMessage(dict(a='b', c='d')):
try:
raise ArbitraryException('foo')
except ArbitraryException:
info = sys.exc_info()
oops = utility._oops_config.create(dict(exc_info=info))
self.assertEqual(
{'<oops-message-0>': "{'a': 'b', 'c': 'd'}"},
oops['req_vars'])
def test_raising_no_referrer_error(self):
"""Test ErrorReportingUtility.raising() with a NoReferrerError
exception.
An OOPS is not recorded when a NoReferrerError exception is
raised.
"""
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
try:
raise NoReferrerError('xyz')
except NoReferrerError:
self.assertEqual(None, utility.raising(sys.exc_info()))
def setUp(self):
super(BaseRequestEndHandlerTest, self).setUp()
self.profile_dir = self.makeTemporaryDirectory()
self.memory_profile_log = os.path.join(self.profile_dir, 'memory_log')
self.pushConfig('profiling', profile_dir=self.profile_dir)
eru = queryUtility(IErrorReportingUtility)
if eru is None:
# Register an Error reporting utility for this layer.
# This will break tests when run with an ERU already registered.
self.eru = ErrorReportingUtility()
sm = getSiteManager()
sm.registerUtility(self.eru)
self.addCleanup(sm.unregisterUtility, self.eru)
def report_oops(message=None, properties=None, info=None,
transaction_manager=None):
"""Record an oops for the current exception.
This must only be called while handling an exception.
Searches for 'URL', 'url', or 'baseurl' properties, in order of
preference, to use as the linked URL of the OOPS report.
:param message: custom explanatory error message. Do not use
str(exception) to fill in this parameter, it should only be
set when a human readable error has been explicitly generated.
:param properties: Properties to record in the OOPS report.
:type properties: An iterable of (name, value) tuples.
:param info: Exception info.
:type info: The return value of `sys.exc_info()`.
:param transaction_manager: A transaction manager. If specified,
further commit() calls will be logged.
"""
# Get the current exception info first of all.
if info is None:
info = sys.exc_info()
# Collect properties to report.
if properties is None:
properties = []
else:
properties = list(properties)
if message is not None:
properties.append(('error-explanation', message))
# Find a candidate for the request URL.
def find_url():
for name in 'URL', 'url', 'baseurl':
for key, value in properties:
if key == name:
return value
return None
url = find_url()
# Create the dummy request object.
request = ScriptRequest(properties, url)
error_utility = ErrorReportingUtility()
error_utility.configure(section_name='checkwatches')
error_utility.raising(info, request)
return request
def test_ignored_exceptions_for_offsite_referer_reported(self):
# Oopses are reported when Launchpad is the referer for a URL
# that caused an exception.
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
request = TestRequest(
environ={
'SERVER_URL': 'http://launchpad.dev/fnord',
'HTTP_REFERER': 'http://launchpad.dev/snarf'
})
try:
raise GoneError('fnord')
except GoneError:
self.assertNotEqual(None, utility.raising(sys.exc_info(), request))
def test_ignored_exceptions_for_criss_cross_vhost_referer_reported(self):
# Oopses are reported when a Launchpad referer for a bad URL on a
# vhost that caused an exception.
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
request = TestRequest(
environ={
'SERVER_URL': 'http://bazaar.launchpad.dev/fnord',
'HTTP_REFERER': 'http://launchpad.dev/snarf'})
try:
raise GoneError('fnord')
except GoneError:
self.assertNotEqual(
None, utility.raising(sys.exc_info(), request))
def test_ignored_exceptions_for_criss_cross_vhost_referer_reported(self):
# Oopses are reported when a Launchpad referer for a bad URL on a
# vhost that caused an exception.
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
request = TestRequest(
environ={
'SERVER_URL': 'http://bazaar.launchpad.dev/fnord',
'HTTP_REFERER': 'http://launchpad.dev/snarf'
})
try:
raise GoneError('fnord')
except GoneError:
self.assertNotEqual(None, utility.raising(sys.exc_info(), request))
def test_raising_non_utf8_request_param_key_bug_896959(self):
# When a form has a nonutf8 request param, the key in req_vars must
# still be unicode (or utf8).
request = TestRequest(form={'foo\x85': 'bar'})
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
try:
raise ArbitraryException('foo')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
for key in report['req_vars'].keys():
if isinstance(key, str):
key.decode('utf8')
else:
self.assertIsInstance(key, unicode)
def test_raising_non_utf8_request_param_key_bug_896959(self):
# When a form has a nonutf8 request param, the key in req_vars must
# still be unicode (or utf8).
request = TestRequest(form={'foo\x85': 'bar'})
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
try:
raise ArbitraryException('foo')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
for key in report['req_vars'].keys():
if isinstance(key, str):
key.decode('utf8')
else:
self.assertIsInstance(key, unicode)
def test__makeErrorReport_combines_request_and_error_vars(self):
"""The oops messages should be distinct from real request vars."""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
request = ScriptRequest([('c', 'd')])
with utility.oopsMessage(dict(a='b')):
try:
raise ArbitraryException('foo')
except ArbitraryException:
info = sys.exc_info()
oops = utility._oops_config.create(
dict(exc_info=info, http_request=request))
self.assertEqual(
{'<oops-message-0>': "{'a': 'b'}", 'c': 'd'},
oops['req_vars'])
def test__makeErrorReport_combines_request_and_error_vars(self):
"""The oops messages should be distinct from real request vars."""
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
request = ScriptRequest([('c', 'd')])
with utility.oopsMessage(dict(a='b')):
try:
raise ArbitraryException('foo')
except ArbitraryException:
info = sys.exc_info()
oops = utility._oops_config.create(
dict(exc_info=info, http_request=request))
self.assertEqual({
'<oops-message-0>': "{'a': 'b'}",
'c': 'd'
}, oops['req_vars'])
def test_raising_translation_unavailable(self):
"""Test ErrorReportingUtility.raising() with a TranslationUnavailable
exception.
An OOPS is not recorded when a TranslationUnavailable exception is
raised.
"""
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
self.assertTrue(
TranslationUnavailable.__name__ in utility._ignored_exceptions,
'TranslationUnavailable is not in _ignored_exceptions.')
try:
raise TranslationUnavailable('xyz')
except TranslationUnavailable:
self.assertEqual(None, utility.raising(sys.exc_info()))
def test_raising_translation_unavailable(self):
"""Test ErrorReportingUtility.raising() with a TranslationUnavailable
exception.
An OOPS is not recorded when a TranslationUnavailable exception is
raised.
"""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
self.assertTrue(
TranslationUnavailable.__name__ in utility._ignored_exceptions,
'TranslationUnavailable is not in _ignored_exceptions.')
try:
raise TranslationUnavailable('xyz')
except TranslationUnavailable:
self.assertEqual(None, utility.raising(sys.exc_info()))
def test_raising_with_unprintable_exception(self):
class UnprintableException(Exception):
def __str__(self):
raise RuntimeError('arrgh')
__repr__ = __str__
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
try:
raise UnprintableException()
except UnprintableException:
report = utility.raising(sys.exc_info())
unprintable = '<unprintable UnprintableException object>'
self.assertEqual(unprintable, report['value'])
self.assertIn(
'UnprintableException: ' + unprintable, report['tb_text'])
def test_oopsMessage(self):
"""oopsMessage pushes and pops the messages."""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
with utility.oopsMessage({'a': 'b', 'c': 'd'}):
self.assertEqual(
{0: {'a': 'b', 'c': 'd'}}, utility._oops_messages)
# An additional message doesn't supplant the original message.
with utility.oopsMessage(dict(e='f', a='z', c='d')):
self.assertEqual({
0: {'a': 'b', 'c': 'd'},
1: {'a': 'z', 'e': 'f', 'c': 'd'},
}, utility._oops_messages)
# Messages are removed when out of context.
self.assertEqual(
{0: {'a': 'b', 'c': 'd'}},
utility._oops_messages)
def test_raising_with_unprintable_exception(self):
class UnprintableException(Exception):
def __str__(self):
raise RuntimeError('arrgh')
__repr__ = __str__
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
try:
raise UnprintableException()
except UnprintableException:
report = utility.raising(sys.exc_info())
unprintable = '<unprintable UnprintableException object>'
self.assertEqual(unprintable, report['value'])
self.assertIn('UnprintableException: ' + unprintable,
report['tb_text'])
def log_exception(message, *args):
"""Write the current exception traceback into the Mailman log file.
This is really just a convenience function for a refactored chunk of
common code.
:param message: The message to appear in the xmlrpc and error logs. It
may be a format string.
:param args: Optional arguments to be interpolated into a format string.
"""
error_utility = ErrorReportingUtility()
error_utility.configure(section_name="mailman")
error_utility.raising(sys.exc_info())
out_file = StringIO()
traceback.print_exc(file=out_file)
traceback_text = out_file.getvalue()
syslog("xmlrpc", message, *args)
syslog("error", message, *args)
syslog("error", traceback_text)
def report_oops(file_alias_url=None, error_msg=None):
"""Record an OOPS for the current exception and return the OOPS ID."""
info = sys.exc_info()
properties = []
if file_alias_url is not None:
properties.append(('Sent message', file_alias_url))
if error_msg is not None:
properties.append(('Error message', error_msg))
request = ScriptRequest(properties)
request.principal = get_current_principal()
errorUtility = ErrorReportingUtility()
# Report all exceptions: the mail handling code doesn't expect any in
# normal operation.
errorUtility._ignored_exceptions = set()
report = errorUtility.raising(info, request)
# Note that this assert is arguably bogus: raising is permitted to filter
# reports.
assert report is not None, ('No OOPS generated.')
return report['id']
def report_oops(file_alias_url=None, error_msg=None):
"""Record an OOPS for the current exception and return the OOPS ID."""
info = sys.exc_info()
properties = []
if file_alias_url is not None:
properties.append(('Sent message', file_alias_url))
if error_msg is not None:
properties.append(('Error message', error_msg))
request = ScriptRequest(properties)
request.principal = get_current_principal()
errorUtility = ErrorReportingUtility()
# Report all exceptions: the mail handling code doesn't expect any in
# normal operation.
errorUtility._ignored_exceptions = set()
report = errorUtility.raising(info, request)
# Note that this assert is arguably bogus: raising is permitted to filter
# reports.
assert report is not None, ('No OOPS generated.')
return report['id']
def test_ignored_exceptions_for_offsite_referer(self):
# Exceptions caused by bad URLs that may not be an Lp code issue.
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
errors = set([
GoneError.__name__, InvalidBatchSizeError.__name__,
NotFound.__name__
])
self.assertEqual(errors,
utility._ignored_exceptions_for_offsite_referer)
def test_raising_with_request(self):
"""Test ErrorReportingUtility.raising() with a request"""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[0]
request = TestRequestWithPrincipal(
environ={
'SERVER_URL': 'http://localhost:9000/foo',
'HTTP_COOKIE': 'lp=cookies_hidden_for_security_reasons',
'name1': 'value1',
},
form={
'name1': 'value3 \xa7',
'name2': 'value2',
u'\N{BLACK SQUARE}': u'value4',
})
request.setInWSGIEnvironment('launchpad.pageid', 'IFoo:+foo-template')
try:
raise ArbitraryException('xyz\nabc')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
# topic is obtained from the request
self.assertEqual('IFoo:+foo-template', report['topic'])
self.assertEqual(u'Login, 42, title, description |\u25a0|',
report['username'])
self.assertEqual('http://localhost:9000/foo', report['url'])
self.assertEqual({
'CONTENT_LENGTH': '0',
'GATEWAY_INTERFACE': 'TestFooInterface/1.0',
'HTTP_COOKIE': '<hidden>',
'HTTP_HOST': '127.0.0.1',
'SERVER_URL': 'http://localhost:9000/foo',
u'\u25a0': 'value4',
'lp': '<hidden>',
'name1': 'value3 \xa7',
'name2': 'value2',
}, report['req_vars'])
# verify that the oopsid was set on the request
self.assertEqual(request.oopsid, report['id'])
self.assertEqual(request.oops, report)
def test_raising_with_request(self):
"""Test ErrorReportingUtility.raising() with a request"""
utility = ErrorReportingUtility()
utility._main_publishers[0].__call__ = lambda report: []
request = TestRequestWithPrincipal(environ={
'SERVER_URL': 'http://localhost:9000/foo',
'HTTP_COOKIE': 'lp=cookies_hidden_for_security_reasons',
'name1': 'value1',
},
form={
'name1': 'value3 \xa7',
'name2': 'value2',
u'\N{BLACK SQUARE}': u'value4',
})
request.setInWSGIEnvironment('launchpad.pageid', 'IFoo:+foo-template')
try:
raise ArbitraryException('xyz\nabc')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
# topic is obtained from the request
self.assertEqual('IFoo:+foo-template', report['topic'])
self.assertEqual(u'Login, 42, title, description |\u25a0|',
report['username'])
self.assertEqual('http://localhost:9000/foo', report['url'])
self.assertEqual(
{
'CONTENT_LENGTH': '0',
'GATEWAY_INTERFACE': 'TestFooInterface/1.0',
'HTTP_COOKIE': '<hidden>',
'HTTP_HOST': '127.0.0.1',
'SERVER_URL': 'http://localhost:9000/foo',
u'\u25a0': 'value4',
'lp': '<hidden>',
'name1': 'value3 \xa7',
'name2': 'value2',
}, report['req_vars'])
# verify that the oopsid was set on the request
self.assertEqual(request.oopsid, report['id'])
self.assertEqual(request.oops, report)
def test_raising_with_string_as_traceback(self):
# ErrorReportingUtility.raising() can be called with a string in the
# place of a traceback. This is useful when the original traceback
# object is unavailable - e.g. when logging a failure reported by a
# non-oops-enabled service.
try:
raise RuntimeError('hello')
except RuntimeError:
exc_type, exc_value, exc_tb = sys.exc_info()
# Turn the traceback into a string. When the traceback itself
# cannot be passed to ErrorReportingUtility.raising, a string like
# one generated by format_exc is sometimes passed instead.
exc_tb = traceback.format_exc()
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
report = utility.raising((exc_type, exc_value, exc_tb))
# traceback is what we supplied.
self.assertEqual(exc_tb, report['tb_text'])
def test_raising_with_string_as_traceback(self):
# ErrorReportingUtility.raising() can be called with a string in the
# place of a traceback. This is useful when the original traceback
# object is unavailable - e.g. when logging a failure reported by a
# non-oops-enabled service.
try:
raise RuntimeError('hello')
except RuntimeError:
exc_type, exc_value, exc_tb = sys.exc_info()
# Turn the traceback into a string. When the traceback itself
# cannot be passed to ErrorReportingUtility.raising, a string like
# one generated by format_exc is sometimes passed instead.
exc_tb = traceback.format_exc()
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
report = utility.raising((exc_type, exc_value, exc_tb))
# traceback is what we supplied.
self.assertEqual(exc_tb, report['tb_text'])
def test_marked_exception_is_ignored(self):
# If an exception has been marked as ignorable, then it is ignored in
# the report.
utility = ErrorReportingUtility()
try:
raise ArbitraryException('xyz\nabc')
except ArbitraryException:
exc_info = sys.exc_info()
directlyProvides(exc_info[1], IUnloggedException)
report = utility._oops_config.create(dict(exc_info=exc_info))
self.assertTrue(report['ignore'])
def test_multiple_raises_in_request(self):
"""An OOPS links to the previous OOPS in the request, if any."""
utility = ErrorReportingUtility()
utility._main_publishers[0].__call__ = lambda report: []
request = TestRequestWithPrincipal()
try:
raise ArbitraryException('foo')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
self.assertFalse('last_oops' in report)
last_oopsid = request.oopsid
try:
raise ArbitraryException('foo')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
self.assertTrue('last_oops' in report)
self.assertEqual(report['last_oops'], last_oopsid)
def test_multiple_raises_in_request(self):
"""An OOPS links to the previous OOPS in the request, if any."""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[0]
request = TestRequestWithPrincipal()
try:
raise ArbitraryException('foo')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
self.assertFalse('last_oops' in report)
last_oopsid = request.oopsid
try:
raise ArbitraryException('foo')
except ArbitraryException:
report = utility.raising(sys.exc_info(), request)
self.assertTrue('last_oops' in report)
self.assertEqual(report['last_oops'], last_oopsid)
def test_404_without_referer_is_ignored(self):
# If a 404 is generated and there is no HTTP referer, we don't produce
# an OOPS.
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
report = {
'type': 'NotFound',
'url': 'http://example.com',
'req_vars': {}
}
self.assertEqual(None, utility._oops_config.publish(report))
def test_configure(self):
"""Test ErrorReportingUtility.setConfigSection()."""
utility = ErrorReportingUtility()
# The ErrorReportingUtility uses the config.error_reports section
# by default.
self.assertEqual(config.error_reports.oops_prefix, utility.oops_prefix)
self.assertEqual(config.error_reports.error_dir,
utility._oops_datedir_repo.root)
# Some external processes may extend the reporter/prefix with
# extra information.
utility.configure(section_name='branchscanner')
self.assertEqual('T-branchscanner', utility.oops_prefix)
# The default error section can be restored.
utility.configure()
self.assertEqual(config.error_reports.oops_prefix, utility.oops_prefix)
# We should have had two publishers set up:
self.assertEqual(2, len(utility._all_publishers))
# - a fallback publisher chaining a rabbit publisher and a datedir
# publisher
self.assertIsInstance(utility._main_publishers[0], oops_amqp.Publisher)
self.assertEqual(utility._main_publishers[1],
utility._oops_datedir_repo.publish)
# - a notify publisher
self.assertEqual(utility._all_publishers[1], notify_publisher)
def process(self):
"""Process an upload's changes files, and move it to a new directory.
The destination directory depends on the result of the processing
of the changes files. If there are no changes files, the result
is 'failed', otherwise it is the worst of the results from the
individual changes files, in order 'failed', 'rejected', 'accepted'.
"""
changes_files = self.locateChangesFiles()
results = set()
for changes_file in changes_files:
self.processor.log.debug(
"Considering changefile %s" % changes_file)
try:
results.add(self.processChangesFile(
changes_file, self.processor.log))
except (KeyboardInterrupt, SystemExit):
raise
except:
info = sys.exc_info()
message = (
'Exception while processing upload %s' % self.upload_path)
properties = [('error-explanation', message)]
request = ScriptRequest(properties)
error_utility = ErrorReportingUtility()
error_utility.raising(info, request)
self.processor.log.error(
'%s (%s)' % (message, request.oopsid))
results.add(UploadStatusEnum.FAILED)
if len(results) == 0:
destination = UploadStatusEnum.FAILED
else:
for destination in [
UploadStatusEnum.FAILED, UploadStatusEnum.REJECTED,
UploadStatusEnum.ACCEPTED]:
if destination in results:
break
self.moveProcessedUpload(destination, self.processor.log)
def process(self):
"""Process an upload's changes files, and move it to a new directory.
The destination directory depends on the result of the processing
of the changes files. If there are no changes files, the result
is 'failed', otherwise it is the worst of the results from the
individual changes files, in order 'failed', 'rejected', 'accepted'.
"""
changes_files = self.locateChangesFiles()
results = set()
for changes_file in changes_files:
self.processor.log.debug("Considering changefile %s" %
changes_file)
try:
results.add(
self.processChangesFile(changes_file, self.processor.log))
except (KeyboardInterrupt, SystemExit):
raise
except:
info = sys.exc_info()
message = ('Exception while processing upload %s' %
self.upload_path)
properties = [('error-explanation', message)]
request = ScriptRequest(properties)
error_utility = ErrorReportingUtility()
error_utility.raising(info, request)
self.processor.log.error('%s (%s)' % (message, request.oopsid))
results.add(UploadStatusEnum.FAILED)
if len(results) == 0:
destination = UploadStatusEnum.FAILED
else:
for destination in [
UploadStatusEnum.FAILED, UploadStatusEnum.REJECTED,
UploadStatusEnum.ACCEPTED
]:
if destination in results:
break
self.moveProcessedUpload(destination, self.processor.log)
def test_configure(self):
"""Test ErrorReportingUtility.setConfigSection()."""
utility = ErrorReportingUtility()
# The ErrorReportingUtility uses the config.error_reports section
# by default.
self.assertEqual(config.error_reports.oops_prefix, utility.oops_prefix)
self.assertEqual(config.error_reports.error_dir,
utility._oops_datedir_repo.root)
# Some external processes may extend the reporter/prefix with
# extra information.
utility.configure(section_name='branchscanner')
self.assertEqual('T-branchscanner', utility.oops_prefix)
# The default error section can be restored.
utility.configure()
self.assertEqual(config.error_reports.oops_prefix, utility.oops_prefix)
# We should have had three publishers setup:
oops_config = utility._oops_config
self.assertEqual(3, len(oops_config.publishers))
# - a rabbit publisher
self.assertIsInstance(oops_config.publishers[0], oops_amqp.Publisher)
# - a datedir publisher wrapped in a publish_new_only wrapper
datedir_repo = utility._oops_datedir_repo
publisher = oops_config.publishers[1].func_closure[0].cell_contents
self.assertEqual(publisher, datedir_repo.publish)
# - a notify publisher
self.assertEqual(oops_config.publishers[2], notify_publisher)
def test_onsite_404_not_ignored(self):
# A request originating from a local site that generates a NotFound
# (404) produces an OOPS.
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
report = {
'type': 'NotFound',
'url': 'http://example.com',
'req_vars': {
'HTTP_REFERER': 'http://launchpad.dev/'
}
}
self.assertNotEqual(None, utility._oops_config.publish(report))
def test_offsite_404_ignored(self):
# A request originating from another site that generates a NotFound
# (404) is ignored (i.e., no OOPS is logged).
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
report = {
'type': 'NotFound',
'url': 'http://example.com',
'req_vars': {
'HTTP_REFERER': 'example.com'
}
}
self.assertEqual(None, utility._oops_config.publish(report))
def test_session_queries_filtered(self):
"""Test that session queries are filtered."""
utility = ErrorReportingUtility()
utility._oops_config.publisher = None
timeline = Timeline()
timeline.start("SQL-session", "SELECT 'gone'").finish()
try:
raise ArbitraryException('foo')
except ArbitraryException:
info = sys.exc_info()
oops = utility._oops_config.create(
dict(exc_info=info, timeline=timeline))
self.assertEqual("SELECT '%s'", oops['timeline'][0][3])
def setUp(self):
super(BaseRequestEndHandlerTest, self).setUp()
self.profile_dir = self.makeTemporaryDirectory()
self.memory_profile_log = os.path.join(self.profile_dir, 'memory_log')
self.pushConfig('profiling', profile_dir=self.profile_dir)
eru = queryUtility(IErrorReportingUtility)
if eru is None:
# Register an Error reporting utility for this layer.
# This will break tests when run with an ERU already registered.
self.eru = ErrorReportingUtility()
sm = getSiteManager()
sm.registerUtility(self.eru)
self.addCleanup(sm.unregisterUtility, self.eru)
def test_raising_for_script(self):
"""Test ErrorReportingUtility.raising with a ScriptRequest."""
utility = ErrorReportingUtility()
del utility._oops_config.publishers[:]
# A list because code using ScriptRequest expects that - ScriptRequest
# translates it to a dict for now.
req_vars = [
('name2', 'value2'),
('name1', 'value1'),
('name1', 'value3'),
]
url = 'https://launchpad.net/example'
try:
raise ArbitraryException('xyz\nabc')
except ArbitraryException:
# Do not test escaping of request vars here, it is already tested
# in test_raising_with_request.
request = ScriptRequest(req_vars, URL=url)
report = utility.raising(sys.exc_info(), request)
self.assertEqual(url, report['url'])
self.assertEqual(dict(req_vars), report['req_vars'])
def _log(self, exc):
"""Log the exception in a log file and as an OOPS."""
Runner._log(self, exc)
error_utility = ErrorReportingUtility()
error_utility.configure(section_name="mailman")
error_utility.raising(sys.exc_info())
def make_error_utility():
"""Make an error utility for logging errors from codebrowse."""
error_utility = ErrorReportingUtility()
error_utility.configure('codebrowse')
return error_utility
class BaseRequestEndHandlerTest(BaseTest):
def setUp(self):
super(BaseRequestEndHandlerTest, self).setUp()
self.profile_dir = self.makeTemporaryDirectory()
self.memory_profile_log = os.path.join(self.profile_dir, 'memory_log')
self.pushConfig('profiling', profile_dir=self.profile_dir)
eru = queryUtility(IErrorReportingUtility)
if eru is None:
# Register an Error reporting utility for this layer.
# This will break tests when run with an ERU already registered.
self.eru = ErrorReportingUtility()
sm = getSiteManager()
sm.registerUtility(self.eru)
self.addCleanup(sm.unregisterUtility, self.eru)
def endRequest(self, path='/', exception=None, pageid=None, work=None):
start_event = self._get_start_event(path)
da.set_request_started()
profile.start_request(start_event)
request = start_event.request
if pageid is not None:
request.setInWSGIEnvironment('launchpad.pageid', pageid)
if work is not None:
work()
request.response.setResult(EXAMPLE_HTML)
context = object()
event = EndRequestEvent(context, request)
if exception is not None:
self.eru.raising(
(type(exception), exception, None), event.request)
profile.end_request(event)
da.clear_request_started()
return event.request
def getAddedResponse(self, request,
start=EXAMPLE_HTML_START, end=EXAMPLE_HTML_END):
output = request.response.consumeBody()
return output[len(start):-len(end)]
def getMemoryLog(self):
if not os.path.exists(self.memory_profile_log):
return []
f = open(self.memory_profile_log)
result = f.readlines()
f.close()
return result
def getPStatsProfilePaths(self):
return glob.glob(os.path.join(self.profile_dir, '*.prof'))
def getCallgrindProfilePaths(self):
return glob.glob(os.path.join(self.profile_dir, 'callgrind.out.*'))
def getAllProfilePaths(self):
return self.getPStatsProfilePaths() + self.getCallgrindProfilePaths()
def assertBasicProfileExists(self, request, show=False):
self.assertNotEqual(None, request.oops)
response = self.getAddedResponse(request)
self.assertIn('Profile was logged to', response)
if show:
self.assertIn('Top Inline Time', response)
else:
self.assertNotIn('Top Inline Time', response)
self.assertEqual(self.getMemoryLog(), [])
self.assertCleanProfilerState()
return response
def assertPStatsProfile(self, response):
paths = self.getPStatsProfilePaths()
self.assertEqual(len(paths), 1)
self.assertIn(paths[0], response)
self.assertEqual(0, len(self.getCallgrindProfilePaths()))
def assertCallgrindProfile(self, response):
paths = self.getCallgrindProfilePaths()
self.assertEqual(len(paths), 1)
self.assertIn(paths[0], response)
self.assertEqual(0, len(self.getPStatsProfilePaths()))
def assertBothProfiles(self, response):
paths = self.getAllProfilePaths()
self.assertEqual(2, len(paths))
for path in paths:
self.assertIn(path, response)
def assertNoProfiles(self):
self.assertEqual([], self.getAllProfilePaths())