def test_transform_polystorage_error_without_raven_configured(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*_):
exception = PolyStorageError()
exception.exceptions.append(NameError('waldo'))
raise exception
pa.destination.save_raw_and_processed.side_effect = (
mocked_save_raw_and_processed
)
# The important thing is that this is the exception that
# is raised and not something from the raven error handling.
assert_raises(
PolyStorageError,
pa.transform,
'mycrashid'
)
config.logger.warning.assert_called_with(
'Sentry DSN is not configured and an exception happened'
)
def test_transform_polystorage_error_with_raven_configured_failing(
self,
mock_raven,
):
def mock_capture_exception(exc_info=None):
raise ValueError('Someone is wrong on the Internet')
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.side_effect = mock_capture_exception
mock_raven.Client.return_value = raven_mock_client
config = self.get_standard_config(
sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*_):
exception = PolyStorageError()
exception.exceptions.append(NameError('waldo'))
exception.exceptions.append(AssertionError(False))
raise exception
pa.destination.save_raw_and_processed.side_effect = (
mocked_save_raw_and_processed)
# The important thing is that this is the exception that
# is raised and not something from the raven error handling.
assert_raises(PolyStorageError, pa.transform, 'mycrashid')
config.logger.error.assert_called_with(
'Unable to report error with Raven', exc_info=True)
def test_transform_success(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
fake_raw_crash = DotDict()
mocked_get_raw_crash = mock.Mock(return_value=fake_raw_crash)
pa.source.get_raw_crash = mocked_get_raw_crash
fake_dump = {'upload_file_minidump': 'fake_dump_TEMPORARY.dump'}
mocked_get_raw_dumps_as_files = mock.Mock(return_value=fake_dump)
pa.source.get_raw_dumps_as_files = mocked_get_raw_dumps_as_files
fake_processed_crash = DotDict()
mocked_get_unredacted_processed = mock.Mock(return_value=fake_processed_crash)
pa.source.get_unredacted_processed = mocked_get_unredacted_processed
mocked_process_crash = mock.Mock(return_value=7)
pa.processor.process_crash = mocked_process_crash
pa.destination.save_processed = mock.Mock()
finished_func = mock.Mock()
with mock.patch('socorro.processor.processor_app.os.unlink') as mocked_unlink:
# the call being tested
pa.transform(17, finished_func)
# test results
mocked_unlink.assert_called_with('fake_dump_TEMPORARY.dump')
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.process_crash.assert_called_with(
fake_raw_crash,
fake_dump,
fake_processed_crash
)
pa.destination.save_raw_and_processed.assert_called_with(fake_raw_crash, None, 7, 17)
eq_(finished_func.call_count, 1)
def test_transform_polystorage_error_with_raven_configured_successful(self, mock_raven):
# Mock everything
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.return_value = 'someidentifier'
mock_raven.Client.return_value = raven_mock_client
# Set up a processor and mock out .save_raw_and_processed() with multiple
# errors
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = PolyStorageError()
expected_exception.exceptions.append(NameError('waldo'))
expected_exception.exceptions.append(AssertionError(False))
pa.destination.save_raw_and_processed.side_effect = expected_exception
# The important thing is that this is the exception that is raised and
# not something from the raven error handling
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
# Assert that we sent both exceptions to Sentry
raven_mock_client.captureException.assert_has_calls(
[mock.call(exc) for exc in expected_exception.exceptions]
)
# Assert that the logger logged the appropriate thing
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier'
)
def test_transform_polystorage_error_with_raven_configured_failing(self, mock_raven):
raven_mock_client = mock.MagicMock()
# Mock this to throw an error if it's called because it shouldn't get called
raven_mock_client.captureException.side_effect = ValueError('raven error')
mock_raven.Client.return_value = raven_mock_client
# Set up processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = PolyStorageError()
expected_exception.exceptions.append(NameError('waldo'))
expected_exception.exceptions.append(AssertionError(False))
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Make sure the PolyStorageError is raised and not the error from
# .captureException()
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
# Assert that the logger logged raven isn't right
config.logger.error.assert_called_with(
'Unable to report error with Raven', exc_info=True
)
def test_transform_polystorage_error_without_raven_configured(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*_):
exception = PolyStorageError()
exception.exceptions.append(NameError('waldo'))
raise exception
pa.destination.save_raw_and_processed.side_effect = (
mocked_save_raw_and_processed
)
# The important thing is that this is the exception that
# is raised and not something from the raven error handling.
assert_raises(
PolyStorageError,
pa.transform,
'mycrashid'
)
config.logger.warning.assert_called_with(
'Raven DSN is not configured and an exception happened'
)
def test_transform_save_error_with_sentry_configured_successful(
self, mock_get_client, caplogpp):
caplogpp.set_level('DEBUG')
mock_client = mock.MagicMock()
mock_client.captureException.return_value = 'someidentifier'
mock_get_client.return_value = mock_client
# Set up a processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = ValueError('simulated error')
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Run .transform() and make sure it raises the ValueError
with pytest.raises(ValueError):
pa.transform('mycrashid')
# Assert that we sent the exception to Sentry
assert (
mock_client.captureException.call_args_list == [
mock.call((ValueError, expected_exception, WHATEVER))
]
)
# Assert that the logger logged the appropriate thing
logging_msgs = [rec.message for rec in caplogpp.records]
assert 'Error captured in Sentry! Reference: someidentifier' in logging_msgs
def test_transform_get_error_with_sentry_configured_successful(
self, mock_get_client, caplogpp):
caplogpp.set_level('DEBUG')
mock_client = mock.MagicMock()
mock_client.captureException.return_value = 'someidentifier'
mock_get_client.return_value = mock_client
# Set up a processor and mock .get_raw_crash() to raise an exception
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
expected_exception = ValueError('simulated error')
pa.source.get_raw_crash.side_effect = expected_exception
# The processor catches all exceptions from .get_raw_crash() and
# .get_raw_dumps_as_files(), so there's nothing we need to catch here
pa.transform('mycrashid')
# Assert that the processor sent something to Sentry
assert (
mock_client.captureException.call_args_list == [
mock.call((ValueError, expected_exception, WHATEVER))
]
)
# Assert that the logger logged the appropriate thing
logging_msgs = [rec.message for rec in caplogpp.records]
assert 'Error captured in Sentry! Reference: someidentifier' in logging_msgs
def test_transform_save_error_with_raven_configured_successful(self, mock_raven):
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.return_value = 'someidentifier'
mock_raven.Client.return_value = raven_mock_client
# Set up a processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = ValueError('simulated error')
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Run .transform() and make sure it raises the ValueError
with pytest.raises(ValueError):
pa.transform('mycrashid')
# Assert that we sent the exception to Sentry
raven_mock_client.captureException.assert_called_once_with(expected_exception)
# Assert that the logger logged the appropriate thing
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier'
)
def test_transform_success(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
fake_raw_crash = DotDict()
mocked_get_raw_crash = mock.Mock(return_value=fake_raw_crash)
pa.source.get_raw_crash = mocked_get_raw_crash
fake_dump = {'upload_file_minidump': 'fake dump'}
mocked_get_raw_dumps_as_files = mock.Mock(return_value=fake_dump)
pa.source.get_raw_dumps_as_files = mocked_get_raw_dumps_as_files
mocked_convert_raw_crash_to_processed_crash = mock.Mock(return_value=7)
pa.processor.convert_raw_crash_to_processed_crash = \
mocked_convert_raw_crash_to_processed_crash
pa.destination.save_processed = mock.Mock()
finished_func = mock.Mock()
# the call being tested
pa.transform(17, finished_func)
# test results
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.convert_raw_crash_to_processed_crash.assert_called_with(
fake_raw_crash,
fake_dump
)
pa.destination.save_raw_and_processed.assert_called_with(fake_raw_crash, None, 7, 17)
finished_func.assert_called_once()
def test_setup(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
self.assertEqual(pa.registrar.id, 'mocked_registrar')
self.assertEqual(pa.processor.id, 'mocked_processor')
self.assertEqual(pa.waiting_func.id, 'mocked_registrar.checkin')
self.assertEqual(pa.processor.id, 'mocked_processor')
def test_setup(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
self.assertEqual(pa.registrar.id, 'mocked_registrar')
self.assertEqual(pa.processor.id, 'mocked_processor')
self.assertEqual(pa.waiting_func.id, 'mocked_registrar.checkin')
self.assertEqual(pa.processor.id, 'mocked_processor')
def test_source_iterator(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
g = pa.source_iterator()
eq_(g.next(), ((1,), {}))
eq_(g.next(), ((2,), {}))
eq_(g.next(), None)
eq_(g.next(), ((3,), {}))
def test_source_iterator(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
g = pa.source_iterator()
assert next(g) == ((1, ), {})
assert next(g) == ((2, ), {})
assert next(g) is None
assert next(g) == ((3, ), {})
def test_source_iterator(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
g = pa.source_iterator()
eq_(g.next(), ((1,), {}))
eq_(g.next(), ((2,), {}))
eq_(g.next(), None)
eq_(g.next(), ((3,), {}))
def test_source_iterator(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
g = pa.source_iterator()
assert g.next() == ((1,), {})
assert g.next() == ((2,), {})
assert g.next() is None
assert g.next() == ((3,), {})
def test_transform_crash_id_missing(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
mocked_get_raw_crash = mock.Mock(side_effect=CrashIDNotFound(17))
pa.source.get_raw_crash = mocked_get_raw_crash
pa.transform(17)
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.reject_raw_crash.assert_called_with(17, "this crash cannot be found in raw crash storage")
def test_transform_unexpected_exception(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
mocked_get_raw_crash = mock.Mock(side_effect=Exception("bummer"))
pa.source.get_raw_crash = mocked_get_raw_crash
pa.transform(17)
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.reject_raw_crash.assert_called_with(17, "error in loading: bummer")
def test_transform_polystorage_error_with_raven_configured_successful(
self,
mock_raven,
):
captured_exceptions = [] # a global
def mock_capture_exception(exc_info=None):
captured_exceptions.append(exc_info)
return 'someidentifier'
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.side_effect = mock_capture_exception
mock_raven.Client.return_value = raven_mock_client
config = self.get_standard_config(
sentry_dsn='https://[email protected]/project'
)
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*_):
exception = PolyStorageError()
exception.exceptions.append(NameError('waldo'))
exception.exceptions.append(AssertionError(False))
raise exception
pa.destination.save_raw_and_processed.side_effect = (
mocked_save_raw_and_processed
)
# The important thing is that this is the exception that
# is raised and not something from the raven error handling.
assert_raises(
PolyStorageError,
pa.transform,
'mycrashid'
)
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier'
)
eq_(len(captured_exceptions), 2)
captured_exception, captured_exception_2 = captured_exceptions
eq_(captured_exception.__class__, NameError)
eq_(captured_exception.message, 'waldo')
eq_(captured_exception_2.__class__, AssertionError)
eq_(captured_exception_2.message, False)
def test_transform_polystorage_error_with_raven_configured_successful(
self,
mock_raven,
):
captured_exceptions = [] # a global
def mock_capture_exception(exc_info=None):
captured_exceptions.append(exc_info)
return 'someidentifier'
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.side_effect = mock_capture_exception
mock_raven.Client.return_value = raven_mock_client
config = self.get_standard_config(
sentry_dsn='https://[email protected]/project'
)
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*_):
exception = PolyStorageError()
exception.exceptions.append(NameError('waldo'))
exception.exceptions.append(AssertionError(False))
raise exception
pa.destination.save_raw_and_processed.side_effect = (
mocked_save_raw_and_processed
)
# The important thing is that this is the exception that
# is raised and not something from the raven error handling.
assert_raises(
PolyStorageError,
pa.transform,
'mycrashid'
)
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier'
)
eq_(len(captured_exceptions), 2)
captured_exception, captured_exception_2 = captured_exceptions
eq_(captured_exception.__class__, NameError)
eq_(captured_exception.message, 'waldo')
eq_(captured_exception_2.__class__, AssertionError)
eq_(captured_exception_2.message, False)
def test_transform_unexpected_exception(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
mocked_get_raw_crash = mock.Mock(side_effect=Exception('bummer'))
pa.source.get_raw_crash = mocked_get_raw_crash
finished_func = mock.Mock()
pa.transform(17, finished_func)
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.reject_raw_crash.assert_called_with(
17,
'error in loading: bummer'
)
eq_(finished_func.call_count, 1)
def test_transform_polystorage_error_with_sentry_configured_failing(
self, mock_get_client, caplogpp):
caplogpp.set_level('DEBUG')
mock_client = mock.MagicMock()
# Mock this to throw an error if it's called because it shouldn't get called
mock_client.captureException.side_effect = ValueError('sentry error')
mock_get_client.return_value = mock_client
# Set up processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
first_exc_info = (NameError, NameError('waldo'), 'fake tb 1')
second_exc_info = (AssertionError, AssertionError(False), 'fake tb 2')
expected_exception = PolyStorageError()
expected_exception.exceptions.append(first_exc_info)
expected_exception.exceptions.append(second_exc_info)
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Make sure the PolyStorageError is raised and not the error from
# .captureException()
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
# Assert calls to logger--one set for each of the errors in
# PolyStorageError
expected = [
('Unable to report error with Raven', WHATEVER),
('Sentry DSN is not configured and an exception happened', None),
('Exception occurred', first_exc_info),
('error in processing or saving crash mycrashid', None),
('Unable to report error with Raven', WHATEVER),
('Sentry DSN is not configured and an exception happened', None),
('Exception occurred', second_exc_info),
('error in processing or saving crash mycrashid', None)
]
actual = [(rec.message, rec.exc_info) for rec in caplogpp.records]
assert actual == expected
def test_transform_polystorage_error_with_raven_configured_failing(
self,
mock_raven,
):
def mock_capture_exception(exc_info=None):
raise ValueError('Someone is wrong on the Internet')
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.side_effect = mock_capture_exception
mock_raven.Client.return_value = raven_mock_client
config = self.get_standard_config(
sentry_dsn='https://[email protected]/project'
)
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*_):
exception = PolyStorageError()
exception.exceptions.append(NameError('waldo'))
exception.exceptions.append(AssertionError(False))
raise exception
pa.destination.save_raw_and_processed.side_effect = (
mocked_save_raw_and_processed
)
# The important thing is that this is the exception that
# is raised and not something from the raven error handling.
assert_raises(
PolyStorageError,
pa.transform,
'mycrashid'
)
config.logger.error.assert_called_with(
'Unable to report error with Raven', exc_info=True
)
def test_transform_polystorage_error_with_raven_configured_failing(
self, mock_raven):
raven_mock_client = mock.MagicMock()
# Mock this to throw an error if it's called because it shouldn't get called
raven_mock_client.captureException.side_effect = ValueError(
'raven error')
mock_raven.Client.return_value = raven_mock_client
# Set up processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config(
sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = PolyStorageError()
expected_exception.exceptions.append(NameError('waldo'))
expected_exception.exceptions.append(AssertionError(False))
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Make sure the PolyStorageError is raised and not the error from
# .captureException()
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
# Assert that the logger logged raven isn't right
config.logger.error.assert_called_with(
'Unable to report error with Raven', exc_info=True)
def test_transform_polystorage_error_without_sentry_configured(
self, caplogpp):
caplogpp.set_level("DEBUG")
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({"raw": "crash"})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*args, **kwargs):
exception = PolyStorageError()
exception.exceptions.append(NameError("waldo"))
raise exception
pa.destination.save_raw_and_processed.side_effect = (
mocked_save_raw_and_processed)
# The important thing is that this is the exception that
# is raised and not something from the sentry error handling.
with pytest.raises(PolyStorageError):
pa.transform("mycrashid")
logging_msgs = [rec.message for rec in caplogpp.records]
assert "Sentry DSN is not configured and an exception happened" in logging_msgs
def test_transform_success(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
fake_raw_crash = DotDict()
mocked_get_raw_crash = mock.Mock(return_value=fake_raw_crash)
pa.source.get_raw_crash = mocked_get_raw_crash
fake_dump = {'upload_file_minidump': 'fake_dump_TEMPORARY.dump'}
mocked_get_raw_dumps_as_files = mock.Mock(return_value=fake_dump)
pa.source.get_raw_dumps_as_files = mocked_get_raw_dumps_as_files
fake_processed_crash = DotDict()
mocked_get_unredacted_processed = mock.Mock(
return_value=fake_processed_crash)
pa.source.get_unredacted_processed = mocked_get_unredacted_processed
mocked_process_crash = mock.Mock(return_value=7)
pa.processor.process_crash = mocked_process_crash
pa.destination.save_processed = mock.Mock()
finished_func = mock.Mock()
patch_path = 'socorro.processor.processor_app.os.unlink'
with mock.patch(patch_path) as mocked_unlink:
# the call being tested
pa.transform(17, finished_func)
# test results
mocked_unlink.assert_called_with('fake_dump_TEMPORARY.dump')
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.process_crash.assert_called_with(fake_raw_crash,
fake_dump,
fake_processed_crash)
pa.destination.save_raw_and_processed.assert_called_with(
fake_raw_crash, None, 7, 17)
assert finished_func.call_count == 1
def test_transform_polystorage_error_with_sentry_configured_successful(
self, is_enabled, mock_get_hub, caplogpp):
caplogpp.set_level('DEBUG')
# Mock everything
mock_hub = mock.MagicMock()
mock_hub.capture_exception.return_value = 'someidentifier'
mock_get_hub.return_value = mock_hub
# Set up a processor and mock out .save_raw_and_processed() with multiple
# errors
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = PolyStorageError()
expected_exception.exceptions.append(NameError('waldo'))
expected_exception.exceptions.append(AssertionError(False))
pa.destination.save_raw_and_processed.side_effect = expected_exception
# The important thing is that this is the exception that is raised and
# not something from the sentry error handling
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
# Assert that we sent both exceptions to Sentry
mock_hub.capture_exception.assert_has_calls(
[mock.call(error=exc) for exc in expected_exception.exceptions])
# Assert that the logger logged the appropriate thing
logging_msgs = [rec.message for rec in caplogpp.records]
assert 'Error captured in Sentry! Reference: someidentifier' in logging_msgs
def test_transform_get_error_with_raven_configured_successful(self, mock_raven):
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.return_value = 'someidentifier'
mock_raven.Client.return_value = raven_mock_client
# Set up a processor and mock .get_raw_crash() to raise an exception
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
expected_exception = ValueError('simulated error')
pa.source.get_raw_crash.side_effect = expected_exception
# The processor catches all exceptions from .get_raw_crash() and
# .get_raw_dumps_as_files(), so there's nothing we need to catch here
pa.transform('mycrashid')
# Assert that the processor sent something to Sentry
assert (
raven_mock_client.captureException.call_args_list == [
mock.call((ValueError, expected_exception, WHATEVER))
]
)
# Assert that the logger logged the appropriate thing
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier'
)
def test_transform_get_error_with_sentry_configured_successful(
self, is_enabled, mock_get_hub, caplogpp):
caplogpp.set_level('DEBUG')
mock_hub = mock.MagicMock()
mock_hub.capture_exception.return_value = 'someidentifier'
mock_get_hub.return_value = mock_hub
# Set up a processor and mock .get_raw_crash() to raise an exception
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
expected_exception = ValueError('simulated error')
pa.source.get_raw_crash.side_effect = expected_exception
# The processor catches all exceptions from .get_raw_crash() and
# .get_raw_dumps_as_files(), so there's nothing we need to catch here
pa.transform('mycrashid')
# Assert that the processor sent something to Sentry
assert (mock_hub.capture_exception.call_args_list == [
mock.call(error=(ValueError, expected_exception, WHATEVER))
])
# Assert that the logger logged the appropriate thing
logging_msgs = [rec.message for rec in caplogpp.records]
assert 'Error captured in Sentry! Reference: someidentifier' in logging_msgs
def test_transform_polystorage_error_with_raven_configured_successful(self, mock_raven):
# Mock everything
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.return_value = 'someidentifier'
mock_raven.Client.return_value = raven_mock_client
# Set up a processor and mock out .save_raw_and_processed() with multiple
# errors
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = PolyStorageError()
expected_exception.exceptions.append(NameError('waldo'))
expected_exception.exceptions.append(AssertionError(False))
pa.destination.save_raw_and_processed.side_effect = expected_exception
# The important thing is that this is the exception that is raised and
# not something from the raven error handling
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
# Assert that we sent both exceptions to Sentry
raven_mock_client.captureException.assert_has_calls(
[mock.call(exc) for exc in expected_exception.exceptions]
)
# Assert that the logger logged the appropriate thing
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier'
)
def test_transform_save_error_with_raven_configured_successful(self, mock_raven):
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.return_value = 'someidentifier'
mock_raven.Client.return_value = raven_mock_client
# Set up a processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = ValueError('simulated error')
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Run .transform() and make sure it raises the ValueError
with pytest.raises(ValueError):
pa.transform('mycrashid')
# Assert that we sent the exception to Sentry
assert (
raven_mock_client.captureException.call_args_list == [
mock.call((ValueError, expected_exception, WHATEVER))
]
)
# Assert that the logger logged the appropriate thing
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier'
)
def test_transform_save_error_with_sentry_configured_successful(
self, mock_get_hub, caplogpp):
caplogpp.set_level('DEBUG')
mock_hub = mock.MagicMock()
mock_hub.capture_exception.side_effect = RuntimeError(
'should not be called')
mock_get_hub.return_value = mock_hub
# Set up a processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = ValueError('simulated error')
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Run .transform() and make sure it raises the ValueError
with pytest.raises(ValueError):
pa.transform('mycrashid')
# Assert that the exception was not sent to Sentry and not logged
assert not mock_hub.capture_exception.called
assert len(caplogpp.records) == 0
def test_transform_save_error_with_sentry_configured_successful(
self, is_enabled, mock_get_hub, caplogpp):
caplogpp.set_level('DEBUG')
mock_hub = mock.MagicMock()
mock_hub.capture_exception.return_value = 'someidentifier'
mock_get_hub.return_value = mock_hub
# Set up a processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
expected_exception = ValueError('simulated error')
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Run .transform() and make sure it raises the ValueError
with pytest.raises(ValueError):
pa.transform('mycrashid')
# Assert that we sent the exception to Sentry
assert (mock_hub.capture_exception.call_args_list == [
mock.call(error=(ValueError, expected_exception, WHATEVER))
])
# Assert that the logger logged the appropriate thing
logging_msgs = [rec.message for rec in caplogpp.records]
assert 'Error captured in Sentry! Reference: someidentifier' in logging_msgs
def test_transform_success(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
fake_raw_crash = DotDict()
mocked_get_raw_crash = mock.Mock(return_value=fake_raw_crash)
pa.source.get_raw_crash = mocked_get_raw_crash
fake_dump = 'fake dump'
mocked_get_raw_dump = mock.Mock(return_value=fake_dump)
pa.source.get_raw_dump = mocked_get_raw_dump
mocked_convert_raw_crash_to_processed_crash = mock.Mock(return_value=7)
pa.processor.convert_raw_crash_to_processed_crash = \
mocked_convert_raw_crash_to_processed_crash
pa.destination.save_processed = mock.Mock()
pa.transform(17)
pa.source.get_raw_crash.assert_called_with(17)
pa.source.get_raw_dump.assert_called_with(17)
pa.processor.convert_raw_crash_to_processed_crash.assert_called_with(
fake_raw_crash,
fake_dump
)
pa.destination.save_processed.assert_called_with(7)
def test_transform_polystorage_error_without_sentry_configured(self, caplogpp):
caplogpp.set_level('DEBUG')
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*args, **kwargs):
exception = PolyStorageError()
exception.exceptions.append(NameError('waldo'))
raise exception
pa.destination.save_raw_and_processed.side_effect = mocked_save_raw_and_processed
# The important thing is that this is the exception that
# is raised and not something from the sentry error handling.
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
logging_msgs = [rec.message for rec in caplogpp.records]
assert 'Sentry DSN is not configured and an exception happened' in logging_msgs
def test_transform_get_error_with_raven_configured_successful(self, mock_raven):
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.return_value = 'someidentifier'
mock_raven.Client.return_value = raven_mock_client
# Set up a processor and mock .get_raw_crash() to raise an exception
config = self.get_standard_config(sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
expected_exception = ValueError('simulated error')
pa.source.get_raw_crash.side_effect = expected_exception
# The processor catches all exceptions from .get_raw_crash() and
# .get_raw_dumps_as_files(), so there's nothing we need to catch here
pa.transform('mycrashid')
# Assert that the processor sent something to Sentry
raven_mock_client.captureException.assert_called_once_with(expected_exception)
# Assert that the logger logged the appropriate thing
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier'
)
def test_transform_unexpected_exception(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
mocked_get_raw_crash = mock.Mock(side_effect=Exception('bummer'))
pa.source.get_raw_crash = mocked_get_raw_crash
pa.transform(17)
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.reject_raw_crash.assert_called_with(
17, 'error in loading: bummer')
def test_transform_crash_id_missing(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
mocked_get_raw_crash = mock.Mock(side_effect=CrashIDNotFound(17))
pa.source.get_raw_crash = mocked_get_raw_crash
pa.transform(17)
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.reject_raw_crash.assert_called_with(
17, 'this crash cannot be found in raw crash storage')
def test_transform_polystorage_error_with_sentry_configured_failing(
self, is_enabled, mock_get_hub, caplogpp):
caplogpp.set_level("DEBUG")
mock_hub = mock.MagicMock()
# Mock this to throw an error if it's called because it shouldn't get called
mock_hub.capture_exception.side_effect = ValueError("sentry error")
mock_get_hub.return_value = mock_hub
# Set up processor and mock .save_processed_crash() to raise an exception
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({"raw": "crash"})
pa.source.get_raw_dumps_as_files.return_value = {}
try:
raise NameError("waldo")
except NameError:
first_exc_info = sys.exc_info()
try:
raise AssertionError(False)
except AssertionError:
second_exc_info = sys.exc_info()
expected_exception = PolyStorageError()
expected_exception.exceptions.append(first_exc_info)
expected_exception.exceptions.append(second_exc_info)
pa.destination.save_processed_crash.side_effect = expected_exception
# Make sure the PolyStorageError is raised and not the error from
# .captureException()
with pytest.raises(PolyStorageError):
pa.transform("mycrashid")
# Assert logs for failing to process crash
expected = [
# Logs for failed Sentry reporting for first exception
("Unable to report error with Sentry", mock.ANY),
("Sentry DSN is not configured and an exception happened", None),
("Exception occurred", first_exc_info),
# Logs for failed Sentry reporting for second exception
("Unable to report error with Sentry", mock.ANY),
("Sentry DSN is not configured and an exception happened", None),
("Exception occurred", second_exc_info),
# Log for failing to process or save the crash
("error in processing or saving crash mycrashid", None),
]
actual = [(rec.message, rec.exc_info) for rec in caplogpp.records]
assert actual == expected
def test_transform_polystorage_error_with_sentry_configured_failing(
self, is_enabled, mock_get_hub, caplogpp):
caplogpp.set_level('DEBUG')
mock_hub = mock.MagicMock()
# Mock this to throw an error if it's called because it shouldn't get called
mock_hub.capture_exception.side_effect = ValueError('sentry error')
mock_get_hub.return_value = mock_hub
# Set up processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
first_exc_info = (NameError, NameError('waldo'), 'fake tb 1')
second_exc_info = (AssertionError, AssertionError(False), 'fake tb 2')
expected_exception = PolyStorageError()
expected_exception.exceptions.append(first_exc_info)
expected_exception.exceptions.append(second_exc_info)
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Make sure the PolyStorageError is raised and not the error from
# .captureException()
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
# Assert logs for failing to process crash
expected = [
# Logs for failed Sentry reporting for first exception
('Unable to report error with Sentry', WHATEVER),
('Sentry DSN is not configured and an exception happened', None),
('Exception occurred', first_exc_info),
# Logs for failed Sentry reporting for second exception
('Unable to report error with Sentry', WHATEVER),
('Sentry DSN is not configured and an exception happened', None),
('Exception occurred', second_exc_info),
# Log for failing to process or save the crash
('error in processing or saving crash mycrashid', None)
]
actual = [(rec.message, rec.exc_info) for rec in caplogpp.records]
assert actual == expected
def test_transform_polystorage_error_with_raven_configured_failing(
self, mock_raven, caplogpp):
caplogpp.set_level('DEBUG')
raven_mock_client = mock.MagicMock()
# Mock this to throw an error if it's called because it shouldn't get called
raven_mock_client.captureException.side_effect = ValueError(
'raven error')
mock_raven.Client.return_value = raven_mock_client
# Set up processor and mock .save_raw_and_processed() to raise an exception
config = self.get_standard_config(
sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
first_exc_info = (NameError, NameError('waldo'), 'fake tb 1')
second_exc_info = (AssertionError, AssertionError(False), 'fake tb 2')
expected_exception = PolyStorageError()
expected_exception.exceptions.append(first_exc_info)
expected_exception.exceptions.append(second_exc_info)
pa.destination.save_raw_and_processed.side_effect = expected_exception
# Make sure the PolyStorageError is raised and not the error from
# .captureException()
with pytest.raises(PolyStorageError):
pa.transform('mycrashid')
# Assert calls to logger--one set for each of the errors in
# PolyStorageError
expected = [
('Unable to report error with Raven', WHATEVER),
('Sentry DSN is not configured and an exception happened', None),
('Exception occurred', first_exc_info),
('error in processing or saving crash mycrashid', None),
('Unable to report error with Raven', WHATEVER),
('Sentry DSN is not configured and an exception happened', None),
('Exception occurred', second_exc_info),
('error in processing or saving crash mycrashid', None)
]
actual = [(rec.message, rec.exc_info) for rec in caplogpp.records]
assert actual == expected
def test_transform_success(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
fake_raw_crash = DotDict()
mocked_get_raw_crash = mock.Mock(return_value=fake_raw_crash)
pa.source.get_raw_crash = mocked_get_raw_crash
fake_dump = 'fake dump'
mocked_get_raw_dump = mock.Mock(return_value=fake_dump)
pa.source.get_raw_dump = mocked_get_raw_dump
mocked_convert_raw_crash_to_processed_crash = mock.Mock(return_value=7)
pa.processor.convert_raw_crash_to_processed_crash = \
mocked_convert_raw_crash_to_processed_crash
pa.destination.save_processed = mock.Mock()
pa.transform(17)
pa.source.get_raw_crash.assert_called_with(17)
pa.source.get_raw_dump.assert_called_with(17)
pa.processor.convert_raw_crash_to_processed_crash.assert_called_with(
fake_raw_crash, fake_dump)
pa.destination.save_processed.assert_called_with(7)
def test_transform_misc_error_with_raven_configured_successful(
self,
mock_raven,
):
captured_exceptions = [] # a global
def mock_capture_exception(exc_info=None):
captured_exceptions.append(exc_info)
return 'someidentifier'
raven_mock_client = mock.MagicMock()
raven_mock_client.captureException.side_effect = mock_capture_exception
mock_raven.Client.return_value = raven_mock_client
config = self.get_standard_config(
sentry_dsn='https://[email protected]/project')
pa = ProcessorApp(config)
pa._setup_source_and_destination()
pa.source.get_raw_crash.return_value = DotDict({'raw': 'crash'})
pa.source.get_raw_dumps_as_files.return_value = {}
def mocked_save_raw_and_processed(*_):
raise ValueError('Someone is wrong on the Internet')
pa.destination.save_raw_and_processed.side_effect = (
mocked_save_raw_and_processed)
# The important thing is that this is the exception that
# is raised and not something from the raven error handling.
with pytest.raises(ValueError):
pa.transform('mycrashid')
config.logger.info.assert_called_with(
'Error captured in Sentry! Reference: someidentifier')
assert len(captured_exceptions) == 1
captured_exception, = captured_exceptions
assert captured_exception.__class__ == ValueError
assert captured_exception.message == 'Someone is wrong on the Internet'
def test_transform_success(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
fake_raw_crash = DotDict()
mocked_get_raw_crash = mock.Mock(return_value=fake_raw_crash)
pa.source.get_raw_crash = mocked_get_raw_crash
fake_dump = {'upload_file_minidump': 'fake dump'}
mocked_get_raw_dumps_as_files = mock.Mock(return_value=fake_dump)
pa.source.get_raw_dumps_as_files = mocked_get_raw_dumps_as_files
mocked_convert_raw_crash_to_processed_crash = mock.Mock(return_value=7)
pa.processor.convert_raw_crash_to_processed_crash = \
mocked_convert_raw_crash_to_processed_crash
pa.destination.save_processed = mock.Mock()
# the call being tested
pa.transform(17)
# test results
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.convert_raw_crash_to_processed_crash.assert_called_with(
fake_raw_crash,
fake_dump
)
pa.destination.save_processed.assert_called_with(7)
def test_transform_success(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
fake_raw_crash = DotDict({"raw": "1"})
mocked_get_raw_crash = mock.Mock(return_value=fake_raw_crash)
pa.source.get_raw_crash = mocked_get_raw_crash
fake_dumps = {"upload_file_minidump": "fake_dump_TEMPORARY.dump"}
mocked_get_raw_dumps_as_files = mock.Mock(return_value=fake_dumps)
pa.source.get_raw_dumps_as_files = mocked_get_raw_dumps_as_files
fake_processed_crash = DotDict(
{"uuid": "9d8e7127-9d98-4d92-8ab1-065982200317"})
mocked_get_unredacted_processed = mock.Mock(
return_value=fake_processed_crash)
pa.source.get_unredacted_processed = mocked_get_unredacted_processed
mocked_process_crash = mock.Mock(
return_value=DotDict({"processed": "1"}))
pa.processor.process_crash = mocked_process_crash
pa.destination.save_processed_crash = mock.Mock()
finished_func = mock.Mock()
patch_path = "socorro.processor.processor_app.os.unlink"
with mock.patch(patch_path) as mocked_unlink:
# the call being tested
pa.transform(17, finished_func)
# test results
mocked_unlink.assert_called_with("fake_dump_TEMPORARY.dump")
pa.source.get_raw_crash.assert_called_with(17)
pa.processor.process_crash.assert_called_with(fake_raw_crash,
fake_dumps,
fake_processed_crash)
pa.destination.save_processed_crash.assert_called_with(
{"raw": "1"}, {"processed": "1"})
assert finished_func.call_count == 1
def test_setup(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
def test_setup(self):
config = self.get_standard_config()
pa = ProcessorApp(config)
pa._setup_source_and_destination()
