def test_basic_hbase_crashstorage(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'hbase_timeout':
100,
'hbase_host':
commonconfig.hbaseHost.default,
'hbase_port':
commonconfig.hbasePort.default,
}],
argv_source=[])
with config_manager.context() as config:
crashstorage = HBaseCrashStorage(config)
eq_(list(crashstorage.new_crashes()), [])
crash_id = '86b58ff2-9708-487d-bfc4-9dac32121214'
raw = ('{"name":"Peter", '
'"submitted_timestamp":"%d"}' % time.time())
fake_raw_dump_1 = 'peter is a swede'
fake_raw_dump_2 = 'lars is a norseman'
fake_raw_dump_3 = 'adrian is a frenchman'
fake_dumps = {
'upload_file_minidump': fake_raw_dump_1,
'lars': fake_raw_dump_2,
'adrian': fake_raw_dump_3
}
crashstorage.save_raw_crash(json.loads(raw), fake_dumps,
crash_id)
assert config.logger.info.called
assert config.logger.info.call_count > 1
msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0]
# ie logging.info(<template>, <arg>)
msg = msg_tmpl % msg_arg
ok_('saved' in msg)
ok_(crash_id in msg)
raw_crash = crashstorage.get_raw_crash(crash_id)
assert isinstance(raw_crash, dict)
eq_(raw_crash['name'], 'Peter')
dump = crashstorage.get_raw_dump(crash_id)
assert isinstance(dump, basestring)
ok_('peter is a swede' in dump)
dumps = crashstorage.get_raw_dumps(crash_id)
assert isinstance(dumps, dict)
ok_('upload_file_minidump' in dumps)
ok_('lars' in dumps)
ok_('adrian' in dumps)
eq_(dumps['upload_file_minidump'],
fake_dumps['upload_file_minidump'])
eq_(dumps['lars'], fake_dumps['lars'])
eq_(dumps['adrian'], fake_dumps['adrian'])
# hasn't been processed yet
assert_raises(CrashIDNotFound, crashstorage.get_processed,
crash_id)
pro = ('{"name":"Peter",'
'"uuid":"86b58ff2-9708-487d-bfc4-9dac32121214", '
'"submitted_timestamp":"%d", '
'"completeddatetime": "%d"}' %
(time.time(), time.time()))
crashstorage.save_processed(json.loads(pro))
data = crashstorage.get_processed(crash_id)
eq_(data['name'], u'Peter')
hb_connection = crashstorage.hbaseConnectionPool.connection()
ok_(hb_connection.transport.isOpen())
crashstorage.close()
ok_(not hb_connection.transport.isOpen())
def test_select_host_mode_success(self):
a_date = datetime(year=2012,
month=5,
day=4,
hour=15,
minute=10,
tzinfo=UTC)
frequency = timedelta(0, 300)
threshold = a_date - frequency
mock_logging = mock.Mock()
mock_connection = mock.MagicMock()
mock_postgres = mock.MagicMock(return_value=mock_connection)
mock_connection.return_value = mock_connection
mock_connection.__enter__.return_value = mock_connection
mock_cursor = mock.Mock()
mock_connection.cursor.return_value = mock_cursor
mock_execute = mock.Mock()
mock_cursor.execute = mock_execute
fetchall_returns = sequencer(((threshold, ), ), ((17, ), ))
mock_fetchall = mock.Mock(side_effect=fetchall_returns)
mock_cursor.fetchall = mock_fetchall
mock_fetchone = mock.Mock()
mock_cursor.fetchone = mock_fetchone
required_config = ProcessorAppRegistrationClient.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database': mock_postgres,
}])
with config_manager.context() as config:
mock_os_uname_str = 'os.uname'
with mock.patch(mock_os_uname_str) as mock_uname:
mock_uname.return_value = (0, 'wilma')
registrar = ProcessorAppRegistrationClient(config)
name = registrar.processor_name
self.assertEqual(mock_execute.call_count, 4)
expected_execute_args = (
(("select now() - interval %s", (frequency, )), ),
((("select id from processors"
" where lastseendatetime < %s"
" and name like %s limit 1"), (threshold, 'wilma%')), ),
((("update processors set name = %s, "
"startdatetime = now(), lastseendatetime = now()"
" where id = %s"), (name, 17)), ),
((("update jobs set"
" starteddatetime = NULL,"
" completeddatetime = NULL,"
" success = NULL "
"where"
" owner = %s"), (17, )), ),
)
actual_execute_args = mock_execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
self.assertEqual(expected, actual)
def _do_run(klass, config_path=None, values_source_list=None):
# while this method is defined here, only derived classes are allowed
# to call it.
if klass is SocorroApp:
raise NotImplementedError(
"The SocorroApp class has no useable 'main' method"
)
if config_path is None:
config_path = os.environ.get(
'DEFAULT_SOCORRO_CONFIG_PATH',
'./config'
)
if values_source_list is None:
values_source_list = [
# pull in the application defaults from the 'application'
# configman option, once it has been defined
application_defaults_proxy,
# pull in any configuration file
ConfigFileFutureProxy,
# get values from the environment
environment,
# use the command line to get the final overriding values
command_line
]
elif application_defaults_proxy not in values_source_list:
values_source_list = (
[application_defaults_proxy] + values_source_list
)
config_definition = klass.get_required_config()
if 'application' not in config_definition:
# the application option has not been defined. This means that
# the we're likely trying to run one of the applications directly
# rather than through the SocorroWelocomApp. Add the 'application'
# option initialized with the target application as the default.
application_config = Namespace()
application_config.add_option(
'application',
doc=(
'the fully qualified classname of the app to run'
),
default=klass_to_pypath(klass),
# the following setting means this option will NOT be
# commented out when configman generates a config file
likely_to_be_changed=True,
from_string_converter=(
application_defaults_proxy.str_to_application_class
),
)
config_definition = application_config
config_manager = ConfigurationManager(
config_definition,
app_name=klass.app_name,
app_version=klass.app_version,
app_description=klass.app_description,
values_source_list=values_source_list,
options_banned_from_help=[],
config_pathname=config_path
)
def fix_exit_code(code):
# some apps don't return a code so you might get None
# which isn't good enough to send to sys.exit()
if code is None:
return 0
return code
with config_manager.context() as config:
config.executor_identity = (
lambda: threading.currentThread().getName()
)
try:
config_manager.log_config(config.logger)
respond_to_SIGHUP_with_logging = functools.partial(
respond_to_SIGHUP,
logger=config.logger
)
# install the signal handler with logging
signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging)
except KeyError:
# config apparently doesn't have 'logger'
# install the signal handler without logging
signal.signal(signal.SIGHUP, respond_to_SIGHUP)
# we finally know what app to actually run, instantiate it
app_to_run = klass(config)
app_to_run.config_manager = config_manager
# whew, finally run the app that we wanted
return_code = fix_exit_code(app_to_run.main())
return return_code
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}],
argv_source=[])
with config_manager.context() as config:
config.executor_identity = lambda: 'dwight' # bogus thread id
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
expected_dump_2 = fake_binary_dump + " number 2"
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump,
"abc123")
eq_(klass.put_json_dump.call_count, 1)
a = klass.put_json_dump.call_args
eq_(len(a[0]), 4)
#eq_(a[0][1], "abc123")
eq_(a[0][2], expected_raw_crash)
eq_(a[0][3], expected_dump)
eq_(a[1], {'number_of_retries': 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {
"name": "Peter",
"uuid": "abc123",
}
expected_unredacted_processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
eq_(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
eq_(len(a[0]), 3)
eq_(a[0][1], "abc123")
eq_(a[0][2], expected_unredacted_processed_crash)
eq_(a[1], {'number_of_retries': 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
ok_(isinstance(r, DotDict))
a = klass.get_json.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_json.call_count, 1)
eq_(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
eq_(len(a[0]), 3)
eq_(a[0][1], "abc123")
eq_(klass.get_dump.call_count, 1)
eq_(r, expected_dump)
# test get_raw_dumps
m = mock.Mock(
return_value={'upload_file_minidump': fake_binary_dump})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_dumps.call_count, 1)
eq_(r, {'upload_file_minidump': expected_dump})
# test get_raw_dumps 2
m = mock.Mock(
return_value={
'upload_file_minidump': fake_binary_dump,
'aux_1': expected_dump_2
})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_dumps.call_count, 1)
eq_(
r, {
'upload_file_minidump': fake_binary_dump,
'aux_1': expected_dump_2
})
# test get_processed
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed("abc123")
ok_(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_processed_json.call_count, 1)
eq_(r, expected_processed_crash)
def test_migration_crash_storage(self):
n = Namespace()
n.add_option(
'storage',
default=MigrationCrashStorage,
)
n.add_option(
'logger',
default=mock.Mock(),
)
value = {
'primary.storage_class': (
'socorro.unittest.external.test_crashstorage_base.A'
),
'fallback.storage_class': (
'socorro.unittest.external.test_crashstorage_base.B'
),
'date_threshold': '150315'
}
cm = ConfigurationManager(
n,
values_source_list=[value],
argv_source=[]
)
with cm.context() as config:
raw_crash = {'ooid': ''}
before_crash_id = '1498dee9-9a45-45cc-8ec8-71bb62150314'
after_crash_id = '1498dee9-9a45-45cc-8ec8-71bb62150315'
dump = '12345'
processed_crash = {'ooid': '', 'product': 17}
migration_store = config.storage(config)
# save_raw tests
# save to primary
migration_store.primary_store.save_raw_crash = Mock()
migration_store.fallback_store.save_raw_crash = Mock()
migration_store.save_raw_crash(raw_crash, dump, after_crash_id)
migration_store.primary_store.save_raw_crash.assert_called_with(
raw_crash,
dump,
after_crash_id
)
eq_(migration_store.fallback_store.save_raw_crash.call_count, 0)
# save to fallback
migration_store.primary_store.save_raw_crash = Mock()
migration_store.fallback_store.save_raw_crash = Mock()
migration_store.save_raw_crash(raw_crash, dump, before_crash_id)
eq_(migration_store.primary_store.save_raw_crash.call_count, 0)
migration_store.fallback_store.save_raw_crash.assert_called_with(
raw_crash,
dump,
before_crash_id
)
# save_processed tests
# save to primary
processed_crash['crash_id'] = after_crash_id
migration_store.primary_store.save_processed = Mock()
migration_store.fallback_store.save_processed = Mock()
migration_store.save_processed(processed_crash)
migration_store.primary_store.save_processed.assert_called_with(
processed_crash
)
eq_(migration_store.fallback_store.save_processed.call_count, 0)
# save to fallback
processed_crash['crash_id'] = before_crash_id
migration_store.primary_store.save_processed = Mock()
migration_store.fallback_store.save_processed = Mock()
migration_store.save_processed(processed_crash)
eq_(migration_store.primary_store.save_processed.call_count, 0)
migration_store.fallback_store.save_processed.assert_called_with(
processed_crash
)
# close tests
migration_store.primary_store.close = Mock()
migration_store.fallback_store.close = Mock()
migration_store.close()
migration_store.primary_store.close.assert_called_with()
migration_store.fallback_store.close.assert_called_with()
migration_store.primary_store.close = Mock()
migration_store.fallback_store.close = Mock()
migration_store.fallback_store.close.side_effect = (
NotImplementedError()
)
migration_store.close()
migration_store.primary_store.close.assert_called_with()
migration_store.fallback_store.close.assert_called_with()
migration_store.primary_store.close = Mock()
migration_store.primary_store.close.side_effect = Exception('!')
migration_store.close()
migration_store.primary_store.close.assert_called_with()
migration_store.fallback_store.close.assert_called_with()
migration_store.fallback_store.close = Mock()
migration_store.fallback_store.close.side_effect = Exception('!')
assert_raises(PolyStorageError, migration_store.close)
migration_store.primary_store.close.assert_called_with()
migration_store.fallback_store.close.assert_called_with()
def test_processed_crash_storage(self):
n = Namespace()
n.add_option(
'storage',
default=PrimaryDeferredProcessedStorage,
)
n.add_option(
'logger',
default=mock.Mock(),
)
value = {
'primary.storage_class': (
'socorro.unittest.external.test_crashstorage_base.A'
),
'deferred.storage_class': (
'socorro.unittest.external.test_crashstorage_base.B'
),
'processed.storage_class': (
'socorro.unittest.external.test_crashstorage_base.B'
),
'deferral_criteria': lambda x: x.get('foo') == 'foo'
}
cm = ConfigurationManager(
n,
values_source_list=[value],
argv_source=[]
)
with cm.context() as config:
eq_(config.primary.storage_class.foo, 'a')
eq_(config.deferred.storage_class.foo, 'b')
eq_(config.processed.storage_class.foo, 'b')
raw_crash = {'ooid': ''}
crash_id = '1498dee9-9a45-45cc-8ec8-71bb62121203'
dump = '12345'
deferred_crash = {'ooid': '', 'foo': 'foo'}
processed_crash = {'ooid': '', 'product': 17}
pd_store = config.storage(config)
# save_raw tests
pd_store.primary_store.save_raw_crash = Mock()
pd_store.deferred_store.save_raw_crash = Mock()
pd_store.processed_store.save_raw_crash = Mock()
pd_store.save_raw_crash(raw_crash, dump, crash_id)
pd_store.primary_store.save_raw_crash.assert_called_with(
raw_crash,
dump,
crash_id
)
eq_(pd_store.deferred_store.save_raw_crash.call_count, 0)
pd_store.save_raw_crash(deferred_crash, dump, crash_id)
pd_store.deferred_store.save_raw_crash.assert_called_with(
deferred_crash,
dump,
crash_id
)
# save_processed tests
pd_store.primary_store.save_processed = Mock()
pd_store.deferred_store.save_processed = Mock()
pd_store.processed_store.save_processed = Mock()
pd_store.save_processed(processed_crash)
pd_store.processed_store.save_processed.assert_called_with(
processed_crash
)
eq_(pd_store.primary_store.save_processed.call_count, 0)
pd_store.save_processed(deferred_crash)
pd_store.processed_store.save_processed.assert_called_with(
deferred_crash
)
# close tests
pd_store.primary_store.close = Mock()
pd_store.deferred_store.close = Mock()
pd_store.close()
pd_store.primary_store.close.assert_called_with()
pd_store.deferred_store.close.assert_called_with()
pd_store.primary_store.close = Mock()
pd_store.deferred_store.close = Mock()
pd_store.deferred_store.close.side_effect = NotImplementedError()
pd_store.close()
pd_store.primary_store.close.assert_called_with()
pd_store.deferred_store.close.assert_called_with()
pd_store.primary_store.close = Mock()
pd_store.primary_store.close.side_effect = Exception('!')
pd_store.close()
pd_store.primary_store.close.assert_called_with()
pd_store.deferred_store.close.assert_called_with()
pd_store.deferred_store.close = Mock()
pd_store.deferred_store.close.side_effect = Exception('!')
assert_raises(PolyStorageError, pd_store.close)
pd_store.primary_store.close.assert_called_with()
pd_store.deferred_store.close.assert_called_with()
def test_poly_crash_storage(self):
n = Namespace()
n.add_option(
'storage',
default=PolyCrashStorage,
)
n.add_option(
'logger',
default=mock.Mock(),
)
value = {
'storage_namespaces': 'A,A2,B',
'A.crashstorage_class':
'socorro.unittest.external.test_crashstorage_base.A',
'A2.crashstorage_class':
'socorro.unittest.external.test_crashstorage_base.A',
'B.crashstorage_class':
'socorro.unittest.external.test_crashstorage_base.B',
'A2.y': 37
}
cm = ConfigurationManager(n, values_source_list=[value])
with cm.context() as config:
assert config.A.crashstorage_class.foo == 'a'
assert config.A2.crashstorage_class.foo == 'a'
assert config.A2.y == 37
assert config.B.crashstorage_class.foo == 'b'
poly_store = config.storage(config)
assert len(poly_store.storage_namespaces) == 3
assert poly_store.storage_namespaces[0] == 'A'
assert poly_store.storage_namespaces[1] == 'A2'
assert poly_store.storage_namespaces[2] == 'B'
assert len(poly_store.stores) == 3
assert poly_store.stores.A.foo == 'a'
assert poly_store.stores.A2.foo == 'a'
assert poly_store.stores.B.foo == 'b'
raw_crash = {'ooid': ''}
dump = '12345'
processed_crash = {'ooid': '', 'product': 17}
for v in poly_store.stores.itervalues():
v.save_raw_crash = Mock()
v.save_processed = Mock()
v.close = Mock()
poly_store.save_raw_crash(raw_crash, dump, '')
for v in poly_store.stores.itervalues():
v.save_raw_crash.assert_called_once_with(raw_crash, dump, '')
poly_store.save_processed(processed_crash)
for v in poly_store.stores.itervalues():
v.save_processed.assert_called_once_with(processed_crash)
poly_store.save_raw_and_processed(raw_crash, dump, processed_crash,
'n')
for v in poly_store.stores.itervalues():
v.save_raw_crash.assert_called_with(raw_crash, dump, 'n')
v.save_processed.assert_called_with(processed_crash)
raw_crash = {'ooid': 'oaeu'}
dump = '5432'
processed_crash = {'ooid': 'aoeu', 'product': 33}
expected = Exception('this is messed up')
poly_store.stores['A2'].save_raw_crash = Mock()
poly_store.stores['A2'].save_raw_crash.side_effect = expected
poly_store.stores['B'].save_processed = Mock()
poly_store.stores['B'].save_processed.side_effect = expected
with pytest.raises(PolyStorageError):
poly_store.save_raw_crash(raw_crash, dump, '')
for v in poly_store.stores.itervalues():
v.save_raw_crash.assert_called_with(raw_crash, dump, '')
with pytest.raises(PolyStorageError):
poly_store.save_processed(processed_crash)
for v in poly_store.stores.itervalues():
v.save_processed.assert_called_with(processed_crash)
with pytest.raises(PolyStorageError):
poly_store.save_raw_and_processed(raw_crash, dump,
processed_crash, 'n')
for v in poly_store.stores.itervalues():
v.save_raw_crash.assert_called_with(raw_crash, dump, 'n')
v.save_processed.assert_called_with(processed_crash)
poly_store.stores['B'].close.side_effect = Exception
with pytest.raises(PolyStorageError):
poly_store.close()
for v in poly_store.stores.itervalues():
v.close.assert_called_with()
def test_poly_crash_storage(self):
n = Namespace()
n.add_option(
'storage',
default=PolyCrashStorage,
)
n.add_option(
'logger',
default=mock.Mock(),
)
value = {
'storage_classes': (
'socorro.unittest.external.test_crashstorage_base.A,'
'socorro.unittest.external.test_crashstorage_base.A,'
'socorro.unittest.external.test_crashstorage_base.B'
),
'storage1.y': 37,
}
cm = ConfigurationManager(n, values_source_list=[value])
with cm.context() as config:
eq_(config.storage0.crashstorage_class.foo, 'a')
eq_(config.storage1.crashstorage_class.foo, 'a')
eq_(config.storage1.y, 37)
eq_(config.storage2.crashstorage_class.foo, 'b')
poly_store = config.storage(config)
l = len(poly_store.storage_namespaces)
eq_(
l, 3,
'expected poly_store to have lenth of 3, '
'but %d was found instead' % l
)
eq_(poly_store.storage_namespaces[0], 'storage0')
eq_(poly_store.storage_namespaces[1], 'storage1')
eq_(poly_store.storage_namespaces[2], 'storage2')
l = len(poly_store.stores)
eq_(
l, 3,
'expected poly_store.store to have lenth of 3, '
'but %d was found instead' % l
)
eq_(poly_store.stores.storage0.foo, 'a')
eq_(poly_store.stores.storage1.foo, 'a')
eq_(poly_store.stores.storage2.foo, 'b')
raw_crash = {'ooid': ''}
dump = '12345'
processed_crash = {'ooid': '', 'product': 17}
for v in poly_store.stores.itervalues():
v.save_raw_crash = Mock()
v.save_processed = Mock()
v.close = Mock()
poly_store.save_raw_crash(raw_crash, dump, '')
for v in poly_store.stores.itervalues():
v.save_raw_crash.assert_called_once_with(raw_crash, dump, '')
poly_store.save_processed(processed_crash)
for v in poly_store.stores.itervalues():
v.save_processed.assert_called_once_with(processed_crash)
poly_store.save_raw_and_processed(
raw_crash,
dump,
processed_crash,
'n'
)
for v in poly_store.stores.itervalues():
v.save_raw_crash.assert_called_with(raw_crash, dump, 'n')
v.save_processed.assert_called_with(processed_crash)
raw_crash = {'ooid': 'oaeu'}
dump = '5432'
processed_crash = {'ooid': 'aoeu', 'product': 33}
poly_store.stores['storage1'].save_raw_crash = Mock()
poly_store.stores['storage1'].save_raw_crash.side_effect = \
Exception('this is messed up')
poly_store.stores['storage2'].save_processed = Mock()
poly_store.stores['storage2'].save_processed.side_effect = \
Exception('this is messed up')
assert_raises(
PolyStorageError,
poly_store.save_raw_crash,
raw_crash,
dump,
''
)
for v in poly_store.stores.itervalues():
v.save_raw_crash.assert_called_with(raw_crash, dump, '')
assert_raises(
PolyStorageError,
poly_store.save_processed,
processed_crash
)
for v in poly_store.stores.itervalues():
v.save_processed.assert_called_with(processed_crash)
assert_raises(
PolyStorageError,
poly_store.save_raw_and_processed,
raw_crash,
dump,
processed_crash,
'n'
)
for v in poly_store.stores.itervalues():
v.save_raw_crash.assert_called_with(raw_crash, dump, 'n')
v.save_processed.assert_called_with(processed_crash)
poly_store.stores['storage2'].close.side_effect = Exception
assert_raises(PolyStorageError, poly_store.close)
for v in poly_store.stores.itervalues():
v.close.assert_called_with()
def _do_run(klass, config_path=None, values_source_list=None):
# while this method is defined here, only derived classes are allowed
# to call it.
if klass is SocorroApp:
raise NotImplementedError(
"The SocorroApp class has no useable 'main' method")
if config_path is None:
config_path = os.environ.get('DEFAULT_SOCORRO_CONFIG_PATH',
'./config')
if values_source_list is None:
values_source_list = [
# pull in any configuration file
ConfigFileFutureProxy,
# get values from the environment
environment,
# use the command line to get the final overriding values
command_line
]
# Pull base set of defaults from the config module if it is specified
if klass.config_defaults is not None:
values_source_list.insert(0, klass.config_defaults)
config_definition = klass.get_required_config()
if 'application' not in config_definition:
# FIXME(mkelly): We used to have a SocorroWelcomeApp that defined an
# "application" option. We no longer have that. This section should
# get reworked possibly as part of getting rid of application
# defaults.
application_config = Namespace()
application_config.add_option(
'application',
doc=('the fully qualified classname of the app to run'),
default=klass_to_pypath(klass),
# the following setting means this option will NOT be
# commented out when configman generates a config file
likely_to_be_changed=True,
from_string_converter=str_to_python_object,
)
config_definition = application_config
config_manager = ConfigurationManager(
config_definition,
app_name=klass.app_name,
app_version=klass.app_version,
app_description=klass.app_description,
values_source_list=values_source_list,
options_banned_from_help=[],
config_pathname=config_path)
def fix_exit_code(code):
# some apps don't return a code so you might get None
# which isn't good enough to send to sys.exit()
if code is None:
return 0
return code
with config_manager.context() as config:
config.executor_identity = (
lambda: threading.currentThread().getName())
# Log revision information
revision_data = get_revision_data()
revision_items = sorted(revision_data.items())
config.logger.info(
'version.json: {%s}', ', '.join(
['%r: %r' % (key, val) for key, val in revision_items]))
try:
config_manager.log_config(config.logger)
respond_to_SIGHUP_with_logging = functools.partial(
respond_to_SIGHUP, logger=config.logger)
# install the signal handler with logging
signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging)
except KeyError:
# config apparently doesn't have 'logger'
# install the signal handler without logging
signal.signal(signal.SIGHUP, respond_to_SIGHUP)
# we finally know what app to actually run, instantiate it
app_to_run = klass(config)
app_to_run.config_manager = config_manager
# whew, finally run the app that we wanted
return_code = fix_exit_code(app_to_run.main())
return return_code
# create an iterable collection of definition sources
# internally, this list will be appended to, so a tuple won't do.
# the definitions are in the json file listed below.
definition_source = 'demo1j.json'
# set up the manager with the option definitions along with the 'app_name' and
# 'app_description'. They will both be used later to create the output of the
# automatically created '--help' command line switch.
# By default, when assigning values to the options loaded from the json file,
# the ConfigurationManager will take, in turn: the default from the definition,
# any values loaded from a config file specified by the --admin.conf command
# line switch, values from the os environment and finally overrides from the
# commandline.
c = ConfigurationManager(definition_source,
app_name='demo1j',
app_description=__doc__)
# fetch the DOM-like instance that gives access to the configuration info
config = c.get_config()
# use the config
if config.action == 'echo':
echo(config.text)
elif config.action == 'backwards':
backwards(config.text)
elif config.action == 'upper':
upper(config.text)
else:
print >> sys.stderr, config.action, "is not a valid action"
def test_basic_postgres_save_raw_crash(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres
}],
argv_source=[])
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
database = crashstorage.database.return_value = mock.MagicMock()
ok_(isinstance(database, mock.Mock))
ok_('submitted_timestamp' in a_raw_crash)
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
crashstorage.save_raw_crash(
a_raw_crash, '', "936ce666-ff3b-4c7a-9674-367fe2120408")
eq_(m.cursor.call_count, 1)
eq_(
m.cursor.return_value.__enter__.return_value.execute.
call_count, 1)
expected_execute_args = ((("""
WITH update_raw_crash AS (
UPDATE raw_crashes_20120402 SET
raw_crash = %(raw_crash)s,
date_processed = %(date_processed)s
WHERE uuid = %(crash_id)s
RETURNING 1
),
insert_raw_crash AS (
INSERT into raw_crashes_20120402
(uuid, raw_crash, date_processed)
( SELECT
%(crash_id)s as uuid,
%(raw_crash)s as raw_crash,
%(date_processed)s as date_processed
WHERE NOT EXISTS (
SELECT uuid from raw_crashes_20120402
WHERE
uuid = %(crash_id)s
LIMIT 1
)
)
RETURNING 2
)
SELECT * from update_raw_crash
UNION ALL
SELECT * from insert_raw_crash
""", {
'crash_id':
'936ce666-ff3b-4c7a-9674-367fe2120408',
'raw_crash':
'{"submitted_timestamp": "2012-04-08 10:52:42.0", "Version": "6.02E23", "ProductName": "Fennicky"}',
'date_processed':
"2012-04-08 10:52:42.0"
}), ), )
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
expeceted_sql, expected_params = expected[0]
expeceted_sql = remove_whitespace(expeceted_sql)
actual_sql, actual_params = actual[0]
actual_sql = remove_whitespace(actual_sql)
eq_(expeceted_sql, actual_sql)
eq_(expected_params, actual_params)
def test_success_after_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
'timeout':
0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
esindex_results = [
pyelasticsearch.exceptions.Timeout,
pyelasticsearch.exceptions.Timeout
]
def esindex_fn(*args, **kwargs):
try:
r = esindex_results.pop(0)
raise r
except IndexError:
return mock_es.index
mock_es.index.side_effect = esindex_fn
crash_id = a_processed_crash['uuid']
es_storage.save_raw_and_processed(
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = ('socorro201214', 'crash_reports',
expected_crash)
expected_request_kwargs = {
'replication': 'async',
'id': crash_id,
}
mock_es.index.assert_called_with(*expected_request_args,
**expected_request_kwargs)
def test_basic_postgres_save_processed_succeed_after_failures(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}])
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.database.operational_exceptions = (OperationalError, )
database = crashstorage.database.return_value = mock.MagicMock()
self.assertTrue(isinstance(database, mock.Mock))
fetch_all_returns = [
((666, ), ),
None,
((23, ), ),
]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
fetch_mock = mock.Mock()
fetch_mock.fetchall.side_effect = fetch_all_func
connection_trouble = [
OperationalError('bad'),
OperationalError('worse'),
]
def broken_connection(*args):
try:
result = connection_trouble.pop(0)
raise result
except IndexError:
return fetch_mock
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.side_effect = broken_connection
crashstorage.save_processed(a_processed_crash)
self.assertEqual(m.cursor.call_count, 9)
self.assertEqual(m.cursor().fetchall.call_count, 3)
self.assertEqual(m.cursor().execute.call_count, 7)
expected_execute_args = (
(('savepoint MainThread', None), ),
(('insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id',
[
None, '0x1c', '...', '20120309050057',
'2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884',
'None | 0', 'arm', '2012-04-08 10:56:41.558922', None,
None, '*****@*****.**', 'high', '[blank]', None, 22385,
None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ',
'SignatureTool: signature truncated due to length',
'plugin', 'FennecAndroid', 'SIGSEGV', 'default',
'libxul.so@0x117441c', '2012-04-08 10:56:50.440752',
True, [], False, 170, None, None,
'http://embarasing.p**n.com',
'936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1'
]), ),
(('release savepoint MainThread', None), ),
(('select id from plugins where filename = %s and name = %s',
('dwight.txt', 'wilma')), ),
(('insert into plugins (filename, name) values (%s, %s) returning id',
('dwight.txt', 'wilma')), ),
(('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)',
(666, 23, '2012-04-08 10:56:41.558922', '69')), ),
(('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)',
(666, '2012-04-08 10:56:41.558922', 0,
'{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')), ),
)
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
self.assertEqual(expected, actual)
def test_select_host_mode_not_dead_fail(self):
a_date = datetime(year=2012,
month=5,
day=4,
hour=15,
minute=10,
tzinfo=UTC)
frequency = timedelta(0, 300)
threshold = a_date - frequency
mock_logging = mock.Mock()
mock_connection = mock.MagicMock()
mock_postgres = mock.MagicMock(return_value=mock_connection)
mock_connection.return_value = mock_connection
mock_connection.__enter__.return_value = mock_connection
mock_cursor = mock.Mock()
mock_connection.cursor.return_value = mock_cursor
mock_execute = mock.Mock()
mock_cursor.execute = mock_execute
fetchall_returns = sequencer(
((threshold, ), ),
SQLDidNotReturnSingleValue(),
((92, ), ),
)
mock_fetchall = mock.Mock(side_effect=fetchall_returns)
mock_cursor.fetchall = mock_fetchall
mock_fetchone = mock.Mock()
mock_cursor.fetchone = mock_fetchone
required_config = ProcessorAppRegistrationClient.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database': mock_postgres,
}])
with config_manager.context() as config:
mock_os_uname_str = 'os.uname'
with mock.patch(mock_os_uname_str) as mock_uname:
mock_uname.return_value = (0, 'wilma')
self.assertRaises(RegistrationError,
ProcessorAppRegistrationClient, config)
self.assertEqual(mock_execute.call_count, 3)
expected_execute_args = (
(("select now() - interval %s", (frequency, )), ),
((("select id from processors"
" where lastseendatetime < %s"
" and name like %s limit 1"), (threshold, 'wilma%')), ),
((("select id from processors"
" where name like 'wilma%'"), None), ),
)
actual_execute_args = mock_execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
self.assertEqual(expected, actual)
def test_programming_error_with_postgres_with_backoff_with_rollback(self):
required_config = Namespace()
required_config.add_option(
'transaction_executor_class',
default=TransactionExecutorWithInfiniteBackoff,
doc='a class that will execute transactions')
required_config.add_option('database_class',
default=MockConnectionContext,
from_string_converter=class_converter)
mock_logging = MockLogging()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'backoff_delays': [2, 4, 6, 10, 15]
}],
argv_source=[])
with config_manager.context() as config:
mocked_context = config.database_class(config)
executor = config.transaction_executor_class(
config, mocked_context)
_function_calls = [] # some mutable
_sleep_count = []
def mock_function_struggling(connection):
assert isinstance(connection, MockConnection)
connection.transaction_status = psycopg2.extensions.TRANSACTION_STATUS_INTRANS
_function_calls.append(connection)
# the default sleep times are going to be,
# 2, 4, 6, 10, 15
# so after 2 + 4 + 6 + 10 + 15 seconds
# all will be exhausted
if sum(_sleep_count) < sum([2, 4, 6, 10, 15]):
raise psycopg2.ProgrammingError(
'SSL SYSCALL error: EOF detected')
def mock_sleep(n):
_sleep_count.append(n)
# monkey patch the sleep function from inside transaction_executor
_orig_sleep = socorro.database.transaction_executor.time.sleep
socorro.database.transaction_executor.time.sleep = mock_sleep
try:
executor(mock_function_struggling)
assert _function_calls
assert commit_count == 1
assert rollback_count == 5
assert mock_logging.criticals
assert len(mock_logging.criticals) == 5
assert len(_sleep_count) > 10
finally:
socorro.database.transaction_executor.time.sleep = _orig_sleep
# this time, simulate an actual code bug where a callable function
# raises a ProgrammingError() exception by, for example, a syntax error
with config_manager.context() as config:
mocked_context = config.database_class(config)
executor = config.transaction_executor_class(
config, mocked_context)
def mock_function_developer_mistake(connection):
assert isinstance(connection, MockConnection)
connection.transaction_status = psycopg2.extensions.TRANSACTION_STATUS_INTRANS
raise psycopg2.ProgrammingError("syntax error")
with pytest.raises(psycopg2.ProgrammingError):
executor(mock_function_developer_mistake)
def get_config_manager():
pubsub_config = PubSubCrashQueue.get_required_config()
return ConfigurationManager([pubsub_config],
app_name='test-pubsub',
app_description='',
argv_source=[])
def test_write_with_imported_module(self):
import os
from configman.tests.values_for_module_tests_1 import Alpha, foo, a
definitions = {
'os_module': os,
'a': 17,
'imported_class': Alpha,
'imported_function': foo,
'xxx': {
'yyy': a,
}
}
cm = ConfigurationManager(
definitions,
values_source_list=[],
)
s = StringIO()
@contextlib.contextmanager
def s_opener():
yield s
cm.write_conf('py', s_opener)
generated_python_module_text = s.getvalue()
expected = """# generated Python configman file
from configman.dotdict import DotDict
from configman.tests.values_for_module_tests_1 import (
Alpha,
foo,
)
import os
# the following symbols will be ignored by configman when
# this module is used as a value source. This will
# suppress the mismatch warning since these symbols are
# values for options, not option names themselves.
ignore_symbol_list = [
"Alpha",
"DotDict",
"foo",
"os",
]
# a
a = 17
# imported_class
imported_class = Alpha
# imported_function
imported_function = foo
# os_module
os_module = os
# Namespace: xxx
xxx = DotDict()
# yyy
xxx.yyy = 18
"""
if six.PY2:
expected = six.binary_type(expected)
self.assertEqual(generated_python_module_text, expected)
def test_fallback_crash_storage(self):
n = Namespace()
n.add_option(
'storage',
default=FallbackCrashStorage,
)
n.add_option(
'logger',
default=mock.Mock(),
)
value = {
'primary.storage_class': (
'socorro.unittest.external.test_crashstorage_base.A'
),
'fallback.storage_class': (
'socorro.unittest.external.test_crashstorage_base.B'
),
}
cm = ConfigurationManager(
n,
values_source_list=[value],
argv_source=[]
)
with cm.context() as config:
eq_(config.primary.storage_class.foo, 'a')
eq_(config.fallback.storage_class.foo, 'b')
raw_crash = {'ooid': ''}
crash_id = '1498dee9-9a45-45cc-8ec8-71bb62121203'
dump = '12345'
processed_crash = {'ooid': '', 'product': 17}
fb_store = config.storage(config)
# save_raw tests
fb_store.primary_store.save_raw_crash = Mock()
fb_store.fallback_store.save_raw_crash = Mock()
fb_store.save_raw_crash(raw_crash, dump, crash_id)
fb_store.primary_store.save_raw_crash.assert_called_with(
raw_crash,
dump,
crash_id
)
eq_(fb_store.fallback_store.save_raw_crash.call_count, 0)
fb_store.primary_store.save_raw_crash = Mock()
fb_store.primary_store.save_raw_crash.side_effect = Exception('!')
fb_store.save_raw_crash(raw_crash, dump, crash_id)
fb_store.primary_store.save_raw_crash.assert_called_with(
raw_crash,
dump,
crash_id
)
fb_store.fallback_store.save_raw_crash.assert_called_with(
raw_crash,
dump,
crash_id
)
fb_store.fallback_store.save_raw_crash = Mock()
fb_store.fallback_store.save_raw_crash.side_effect = Exception('!')
assert_raises(
PolyStorageError,
fb_store.save_raw_crash,
raw_crash,
dump,
crash_id
)
fb_store.primary_store.save_raw_crash.assert_called_with(
raw_crash,
dump,
crash_id
)
fb_store.fallback_store.save_raw_crash.assert_called_with(
raw_crash,
dump,
crash_id
)
# save_processed tests
fb_store.primary_store.save_processed = Mock()
fb_store.fallback_store.save_processed = Mock()
fb_store.save_processed(processed_crash)
fb_store.primary_store.save_processed.assert_called_with(
processed_crash
)
eq_(fb_store.fallback_store.save_processed.call_count, 0)
fb_store.primary_store.save_processed = Mock()
fb_store.primary_store.save_processed.side_effect = Exception('!')
fb_store.save_processed(processed_crash)
fb_store.primary_store.save_processed.assert_called_with(
processed_crash
)
fb_store.fallback_store.save_processed.assert_called_with(
processed_crash
)
fb_store.fallback_store.save_processed = Mock()
fb_store.fallback_store.save_processed.side_effect = Exception('!')
assert_raises(
PolyStorageError,
fb_store.save_processed,
processed_crash
)
fb_store.primary_store.save_processed.assert_called_with(
processed_crash
)
fb_store.fallback_store.save_processed.assert_called_with(
processed_crash
)
# close tests
fb_store.primary_store.close = Mock()
fb_store.fallback_store.close = Mock()
fb_store.close()
fb_store.primary_store.close.assert_called_with()
fb_store.fallback_store.close.assert_called_with()
fb_store.primary_store.close = Mock()
fb_store.fallback_store.close = Mock()
fb_store.fallback_store.close.side_effect = NotImplementedError()
fb_store.close()
fb_store.primary_store.close.assert_called_with()
fb_store.fallback_store.close.assert_called_with()
fb_store.primary_store.close = Mock()
fb_store.primary_store.close.side_effect = Exception('!')
fb_store.close()
fb_store.primary_store.close.assert_called_with()
fb_store.fallback_store.close.assert_called_with()
fb_store.fallback_store.close = Mock()
fb_store.fallback_store.close.side_effect = Exception('!')
assert_raises(PolyStorageError, fb_store.close)
fb_store.primary_store.close.assert_called_with()
fb_store.fallback_store.close.assert_called_with()
def test_write_with_imported_module_with_internal_mappings(self):
import os
from configman.tests.values_for_module_tests_1 import Alpha, foo
d = {
'a': 18,
'b': 'hello',
'c': [1, 2, 3],
'd': {
'host': 'localhost',
'port': 5432,
}
}
definitions = {
'os_module': os,
'a': 17,
'imported_class': Alpha,
'imported_function': foo,
'xxx': {
'yyy': Option('yyy', default=d)
},
'e': None,
}
required_config = Namespace()
required_config.add_option(
'minimal_version_for_understanding_refusal',
doc='ignore the Thottleable protocol',
default={'Firefox': '3.5.4'},
)
cm = ConfigurationManager(
[definitions, required_config],
values_source_list=[],
)
cm.get_config()
s = StringIO()
@contextlib.contextmanager
def s_opener():
yield s
cm.write_conf('py', s_opener)
generated_python_module_text = s.getvalue()
expected = """# generated Python configman file
from configman.dotdict import DotDict
from configman.tests.values_for_module_tests_1 import (
Alpha,
foo,
)
import os
# the following symbols will be ignored by configman when
# this module is used as a value source. This will
# suppress the mismatch warning since these symbols are
# values for options, not option names themselves.
ignore_symbol_list = [
"Alpha",
"DotDict",
"foo",
"os",
]
# a
a = 17
# e
e = None
# imported_class
imported_class = Alpha
# imported_function
imported_function = foo
# ignore the Thottleable protocol
minimal_version_for_understanding_refusal = {
"Firefox": "3.5.4"
}
# os_module
os_module = os
# Namespace: xxx
xxx = DotDict()
xxx.yyy = {
"a": 18,
"b": "hello",
"c": [
1,
2,
3
],
"d": {
"host": "localhost",
"port": 5432
}
}
"""
self.assertEqual(generated_python_module_text, expected)
def test_basic_crashstorage(self):
required_config = Namespace()
mock_logging = Mock()
required_config.add_option('logger', default=mock_logging)
required_config.update(CrashStorageBase.required_config)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
}],
argv_source=[]
)
with config_manager.context() as config:
crashstorage = CrashStorageBase(
config,
quit_check_callback=fake_quit_check
)
crashstorage.save_raw_crash({}, 'payload', 'ooid')
crashstorage.save_processed({})
assert_raises(
NotImplementedError,
crashstorage.get_raw_crash, 'ooid'
)
assert_raises(
NotImplementedError,
crashstorage.get_raw_dump, 'ooid'
)
assert_raises(
NotImplementedError,
crashstorage.get_unredacted_processed, 'ooid'
)
assert_raises(
NotImplementedError,
crashstorage.remove, 'ooid'
)
eq_(crashstorage.new_crashes(), [])
crashstorage.close()
with config_manager.context() as config:
class MyCrashStorageTest(CrashStorageBase):
def save_raw_crash(self, raw_crash, dumps, crash_id):
eq_(crash_id, "fake_id")
eq_(raw_crash, "fake raw crash")
eq_(
sorted(dumps.keys()),
sorted(['one', 'two', 'three'])
)
eq_(
sorted(dumps.values()),
sorted(['eins', 'zwei', 'drei'])
)
values = ['eins', 'zwei', 'drei']
def open_function(*args, **kwargs):
return values.pop(0)
crashstorage = MyCrashStorageTest(
config,
quit_check_callback=fake_quit_check
)
with mock.patch("__builtin__.open") as open_mock:
open_mock.return_value = mock.MagicMock()
(
open_mock.return_value.__enter__
.return_value.read.side_effect
) = open_function
crashstorage.save_raw_crash_with_file_dumps(
"fake raw crash",
FileDumpsMapping({
'one': 'eins',
'two': 'zwei',
'three': 'drei'
}),
'fake_id'
)
#------------------------------------------------------------------------------
def query2(conn):
"""another transaction to be executed by the database"""
raise Exception("not a database related error")
#==============================================================================
if __name__ == "__main__":
definition_source = Namespace()
definition_source.add_option('transaction_executor_class',
default=TransactionExecutorWithBackoff,
doc='a class that will execute transactions')
c = ConfigurationManager(definition_source,
app_name='advanced_demo_3',
app_description=__doc__)
with c.context() as config:
# the configuration has a class that can execute transactions
# we instantiate it here.
executor = config.transaction_executor_class(config)
# this first query has a 50% probability of failing due to a database
# connectivity problem. If the transaction_executor_class is a class
# with backing off retry, you'll see the transaction tried over and
# over until it succeeds.
print "\n**** First query"
executor.do_transaction(query1)
# this second query has a 50% probability of failing due to a non-
def test_benchmarking_crashstore(self):
required_config = Namespace()
mock_logging = Mock()
required_config.add_option('logger', default=mock_logging)
required_config.update(BenchmarkingCrashStorage.get_required_config())
fake_crash_store = Mock()
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'wrapped_crashstore': fake_crash_store,
'benchmark_tag': 'test'
}],
argv_source=[]
)
with config_manager.context() as config:
crashstorage = BenchmarkingCrashStorage(
config,
quit_check_callback=fake_quit_check
)
crashstorage.start_timer = lambda: 0
crashstorage.end_timer = lambda: 1
fake_crash_store.assert_called_with(config, fake_quit_check)
crashstorage.save_raw_crash({}, 'payload', 'ooid')
crashstorage.wrapped_crashstore.save_raw_crash.assert_called_with(
{},
'payload',
'ooid'
)
mock_logging.debug.assert_called_with(
'%s save_raw_crash %s',
'test',
1
)
mock_logging.debug.reset_mock()
crashstorage.save_processed({})
crashstorage.wrapped_crashstore.save_processed.assert_called_with(
{}
)
mock_logging.debug.assert_called_with(
'%s save_processed %s',
'test',
1
)
mock_logging.debug.reset_mock()
crashstorage.save_raw_and_processed({}, 'payload', {}, 'ooid')
crashstorage.wrapped_crashstore.save_raw_and_processed \
.assert_called_with(
{},
'payload',
{},
'ooid'
)
mock_logging.debug.assert_called_with(
'%s save_raw_and_processed %s',
'test',
1
)
mock_logging.debug.reset_mock()
crashstorage.get_raw_crash('uuid')
crashstorage.wrapped_crashstore.get_raw_crash.assert_called_with(
'uuid'
)
mock_logging.debug.assert_called_with(
'%s get_raw_crash %s',
'test',
1
)
mock_logging.debug.reset_mock()
crashstorage.get_raw_dump('uuid')
crashstorage.wrapped_crashstore.get_raw_dump.assert_called_with(
'uuid'
)
mock_logging.debug.assert_called_with(
'%s get_raw_dump %s',
'test',
1
)
mock_logging.debug.reset_mock()
crashstorage.get_raw_dumps('uuid')
crashstorage.wrapped_crashstore.get_raw_dumps.assert_called_with(
'uuid'
)
mock_logging.debug.assert_called_with(
'%s get_raw_dumps %s',
'test',
1
)
mock_logging.debug.reset_mock()
crashstorage.get_raw_dumps_as_files('uuid')
crashstorage.wrapped_crashstore.get_raw_dumps_as_files \
.assert_called_with(
'uuid'
)
mock_logging.debug.assert_called_with(
'%s get_raw_dumps_as_files %s',
'test',
1
)
mock_logging.debug.reset_mock()
crashstorage.get_unredacted_processed('uuid')
crashstorage.wrapped_crashstore.get_unredacted_processed \
.assert_called_with(
'uuid'
)
mock_logging.debug.assert_called_with(
'%s get_unredacted_processed %s',
'test',
1
)
mock_logging.debug.reset_mock()
def test_basic_postgres_save_processed_succeed_after_failures(self):
mock_logging = mock.Mock()
mock_postgres = mock.Mock()
required_config = PostgreSQLCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'database_class': mock_postgres,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}],
argv_source=[])
with config_manager.context() as config:
crashstorage = PostgreSQLCrashStorage(config)
crashstorage.database.operational_exceptions = (OperationalError, )
database = crashstorage.database.return_value = mock.MagicMock()
ok_(isinstance(database, mock.Mock))
fetch_all_returns = [
((666, ), ),
None,
((23, ), ),
]
def fetch_all_func(*args):
result = fetch_all_returns.pop(0)
return result
fetch_mock = mock.Mock()
fetch_mock.fetchall.side_effect = fetch_all_func
connection_trouble = [
OperationalError('bad'),
OperationalError('worse'),
]
def broken_connection(*args):
try:
result = connection_trouble.pop(0)
raise result
except IndexError:
return fetch_mock
m = mock.MagicMock()
m.__enter__.return_value = m
database = crashstorage.database.return_value = m
m.cursor.side_effect = broken_connection
crashstorage.save_processed(a_processed_crash)
eq_(m.cursor.call_count, 10)
eq_(m.cursor().fetchall.call_count, 3)
eq_(m.cursor().execute.call_count, 8)
expected_execute_args = (
(('savepoint MainThread', None), ),
(('insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id',
[
None, '0x1c', '...', '20120309050057',
'2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884',
'None | 0', 'arm', '2012-04-08 10:56:41.558922', None,
None, '*****@*****.**', 'high', '[blank]', None, 22385,
None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ',
'SignatureTool: signature truncated due to length',
'plugin', 'FennecAndroid', 'FA-888888', 'SIGSEGV',
'default', 'libxul.so@0x117441c',
'2012-04-08 10:56:50.440752', True, [], False, 170, None,
None, 'http://embarrassing.p**n.com',
'936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1'
]), ),
(('release savepoint MainThread', None), ),
(('select id from plugins where filename = %s and name = %s',
('dwight.txt', 'wilma')), ),
(('insert into plugins (filename, name) values (%s, %s) returning id',
('dwight.txt', 'wilma')), ),
(('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)',
(666, 23, '2012-04-08 10:56:41.558922', '69')), ),
(('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)',
(666, '2012-04-08 10:56:41.558922', 0,
'{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')), ),
(("""WITH update_processed_crash AS ( UPDATE processed_crashes_20120402 SET processed_crash = %(processed_json)s, date_processed = %(date_processed)s WHERE uuid = %(uuid)s RETURNING 1), insert_processed_crash AS ( INSERT INTO processed_crashes_20120402 (uuid, processed_crash, date_processed) ( SELECT %(uuid)s as uuid, %(processed_json)s as processed_crash, %(date_processed)s as date_processed WHERE NOT EXISTS ( SELECT uuid from processed_crashes_20120402 WHERE uuid = %(uuid)s LIMIT 1)) RETURNING 2) SELECT * from update_processed_crash UNION ALL SELECT * from insert_processed_crash """,
{
'uuid': '936ce666-ff3b-4c7a-9674-367fe2120408',
'processed_json':
'{"startedDateTime": "2012-04-08 10:56:50.440752", "crashedThread": 8, "cpu_info": "None | 0", "PluginName": "wilma", "install_age": 22385, "topmost_filenames": [], "user_comments": null, "user_id": null, "uuid": "936ce666-ff3b-4c7a-9674-367fe2120408", "flash_version": "[blank]", "os_version": "0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ", "PluginVersion": "69", "addons_checked": null, "completeddatetime": "2012-04-08 10:56:50.902884", "productid": "FA-888888", "success": true, "exploitability": "high", "client_crash_date": "2012-04-08 10:52:42.0", "PluginFilename": "dwight.txt", "dump": "...", "truncated": false, "product": "FennecAndroid", "distributor": null, "processor_notes": "SignatureTool: signature truncated due to length", "uptime": 170, "release_channel": "default", "distributor_version": null, "process_type": "plugin", "id": 361399767, "hangid": null, "version": "13.0a1", "build": "20120309050057", "ReleaseChannel": "default", "email": "*****@*****.**", "app_notes": "...", "os_name": "Linux", "last_crash": null, "date_processed": "2012-04-08 10:56:41.558922", "cpu_name": "arm", "reason": "SIGSEGV", "address": "0x1c", "url": "http://embarrassing.p**n.com", "signature": "libxul.so@0x117441c", "addons": [["{1a5dabbd-0e74-41da-b532-a364bb552cab}", "1.0.4.1"]]}',
'date_processed': '2012-04-08 10:56:41.558922'
}), ),
)
actual_execute_args = m.cursor().execute.call_args_list
for expected, actual in zip(expected_execute_args,
actual_execute_args):
expected_sql, expected_params = expected[0]
expected_sql = remove_whitespace(expected_sql)
actual_sql, actual_params = actual[0]
actual_sql = remove_whitespace(actual_sql)
eq_(expected_sql, actual_sql)
eq_(expected_params, actual_params)
