def test_start1(self):
config = DotDict()
config.logger = self.logger
config.number_of_threads = 1
config.maximum_queue_size = 1
ttm = ThreadedTaskManager(config)
try:
ttm.start()
time.sleep(0.2)
assert ttm.queuing_thread.isAlive(
), "the queing thread is not running"
assert len(ttm.thread_list) == 1, "where's the worker thread?"
assert ttm.thread_list[0].isAlive(
), "the worker thread is stillborn"
ttm.stop()
assert not ttm.queuing_thread.isAlive(
), "the queuing thread did not stop"
except Exception:
# we got threads to join
ttm.wait_for_completion()
def test_extract_frame_info_frames_missing(self):
info = ['4', '12', 'msvcr100.dll', '_callthreadstartex',
'f:\\src\\threadex.c', '314', '0x6']
d = DotDict()
bpj._extract_frame_info(info, d)
assert 'threads' in d
assert len(d.threads) == 5
expected = {
"frame_count": 1,
"frames": [
{
"frame": 12,
"module": 'msvcr100.dll',
"function": '_callthreadstartex',
"file": 'f:\\src\\threadex.c',
"line": 314,
}
]
}
assert d.threads[4] == expected
def test_extract_module_info_not_main(self):
info = ['Module', 'firefloosy.exe', '24.0.0.4925', 'firefox.pdb',
'9FFDDF56AADE45988C759EF5ABAE53862', '0x00400000',
'0x004e0fff', '0']
d = DotDict()
bpj._extract_module_info(info, d, 17)
ok_('modules' in d)
ok_(len(d.modules), 1)
ok_('main_module' not in d)
eq_(
d.modules[0],
{
"filename": 'firefloosy.exe',
"version": '24.0.0.4925',
"debug_file": 'firefox.pdb',
"debug_id": '9FFDDF56AADE45988C759EF5ABAE53862',
"base_addr": '0x00400000',
"end_addr": '0x004e0fff',
}
)
def test_blocking_start_with_quit_on_empty(self):
config = DotDict()
config.logger = self.logger
config.number_of_threads = 2
config.maximum_queue_size = 2
config.quit_on_empty_queue = True
tm = ThreadedTaskManager(
config,
task_func=Mock()
)
waiting_func = Mock()
tm.blocking_start(waiting_func=waiting_func)
eq_(
tm.task_func.call_count,
10
)
def test_extract_module_info(self):
info = ['Module', 'firefox.exe', '24.0.0.4925', 'firefox.pdb',
'9FFDDF56AADE45988C759EF5ABAE53862', '0x00400000',
'0x004e0fff', '1']
d = DotDict()
bpj._extract_module_info(info, d, 17)
self.assertTrue('modules' in d)
self.assertTrue(len(d.modules), 1)
self.assertEqual(d.main_module, 17)
self.assertEqual(
d.modules[0],
{
"filename": 'firefox.exe',
"version": '24.0.0.4925',
"debug_file": 'firefox.pdb',
"debug_id": '9FFDDF56AADE45988C759EF5ABAE53862',
"base_addr": '0x00400000',
"end_addr": '0x004e0fff',
}
)
def test_task_raises_unexpected_exception(self):
global count
count = 0
def new_iter():
for x in xrange(10):
yield (x, )
my_list = []
def insert_into_list(anItem):
global count
count += 1
if count == 4:
raise Exception('Unexpected')
my_list.append(anItem)
config = DotDict()
config.logger = self.logger
config.number_of_threads = 1
config.maximum_queue_size = 1
config.job_source_iterator = new_iter
config.task_func = insert_into_list
ttm = ThreadedTaskManagerWithConfigSetup(config)
try:
ttm.start()
time.sleep(0.2)
ok_(
len(ttm.thread_list) == 1,
"expected 1 threads, but found %d" % len(ttm.thread_list))
ok_(
sorted(my_list) == [0, 1, 2, 4, 5, 6, 7, 8,
9], 'expected %s, but got %s' %
([0, 1, 2, 5, 6, 7, 8, 9], sorted(my_list)))
ok_(
len(my_list) == 9, 'expected to do 9 inserts, '
'but %d were done instead' % len(my_list))
except Exception:
# we got threads to join
ttm.wait_for_completion()
raise
def test_hbase_usage_with_transaction(self):
local_config = DotDict({
'hbase_host': 'host',
'database_name': 'name',
'hbase_port': 9090,
'hbase_timeout': 9000,
'number_of_retries': 2,
'logger': SilentFakeLogger(),
'executor_identity': lambda: 'dwight' # bogus thread id
})
a_fake_hbase_connection = FakeHB_Connection(local_config)
with mock.patch.object(
connection_context, 'HBaseConnection',
mock.Mock(return_value=a_fake_hbase_connection)):
hb_context = connection_context.HBasePooledConnectionContext(
local_config)
def all_ok(connection, dummy):
eq_(dummy, 'hello')
return True
transaction = TransactionExecutor(local_config, hb_context)
result = transaction(all_ok, 'hello')
ok_(result)
eq_(a_fake_hbase_connection.close_counter, 0)
eq_(a_fake_hbase_connection.rollback_counter, 0)
eq_(a_fake_hbase_connection.commit_counter, 1)
def bad_deal(connection, dummy):
raise KeyError('fred')
assert_raises(KeyError, transaction, bad_deal, 'hello')
# at this point, the underlying connection has been deleted from
# the pool, because it was considered to be a bad connection.
eq_(a_fake_hbase_connection.close_counter, 0)
eq_(a_fake_hbase_connection.commit_counter, 1)
hb_context.close()
# because the connection was previously deleted from the pool,
# no connection gets closed at this point.
eq_(a_fake_hbase_connection.close_counter, 0)
def test_doing_work_with_one_worker(self):
config = DotDict()
config.logger = self.logger
config.number_of_threads = 1
config.maximum_queue_size = 1
my_list = []
def insert_into_list(anItem):
my_list.append(anItem)
ttm = ThreadedTaskManager(config, task_func=insert_into_list)
try:
ttm.start()
time.sleep(0.2)
assert len(my_list) == 10
assert my_list == range(10)
ttm.stop()
except Exception:
# we got threads to join
ttm.wait_for_completion()
raise
def test_external_fails(self, mocked_subprocess_module):
config = self.get_basic_config()
raw_crash = copy.copy(canonical_standard_raw_crash)
raw_dumps = {config.dump_field: 'a_fake_dump.dump'}
processed_crash = DotDict()
processor_meta = self.get_basic_processor_meta()
mocked_subprocess_handle = \
mocked_subprocess_module.Popen.return_value
mocked_subprocess_handle.stdout.read.return_value = '{}'
mocked_subprocess_handle.wait.return_value = 124
rule = ExternalProcessRule(config)
# the call to be tested
rule.act(raw_crash, raw_dumps, processed_crash, processor_meta)
assert processed_crash.bogus_command_result == {}
assert processed_crash.bogus_command_return_code == 124
assert processor_meta.processor_notes == []
def test_constructor(self):
faked_connection_object = Mock()
config = DotDict()
conn = Connection(config, faked_connection_object)
self.assertTrue(conn.config is config)
self.assertTrue(conn.connection is faked_connection_object)
faked_connection_object.channel.called_once_with()
self.assertEqual(
faked_connection_object.channel.return_value.queue_declare.
call_count, 3)
expected_queue_declare_call_args = [
call(queue='socorro.normal', durable=True),
call(queue='socorro.priority', durable=True),
call(queue='socorro.reprocessing', durable=True),
]
self.assertEqual(
faked_connection_object.channel.return_value.queue_declare \
.call_args_list,
expected_queue_declare_call_args
)
def test_stuff_missing(self):
config = self.get_basic_config()
raw_crash = copy.copy(canonical_standard_raw_crash)
del raw_crash.uuid
expected_raw_crash = copy.copy(raw_crash)
raw_dumps = {}
processed_crash = DotDict()
processor_meta = self.get_basic_processor_meta()
rule = IdentifierRule(config)
# the call to be tested
result = rule.act(raw_crash, raw_dumps, processed_crash,
processor_meta)
eq_(result, (True, False))
# raw crash should be unchanged
eq_(raw_crash, expected_raw_crash)
def test_basic_hbase_usage(self, mocked_hbcl):
local_config = DotDict({
'hbase_host': 'host',
'database_name': 'name',
'hbase_port': 9090,
'hbase_timeout': 9000,
'number_of_retries': 2,
'logger': SilentFakeLogger(),
})
a_fake_hbase_connection = FakeHB_Connection()
mocked_hbcl.HBaseConnectionForCrashReports = \
mock.Mock(return_value=a_fake_hbase_connection)
hb_context = HBaseConnectionContextPooled(local_config, local_config)
self.assertEqual(mocked_hbcl.HBaseConnectionForCrashReports.call_count,
1)
self.assertEqual(a_fake_hbase_connection.close_counter, 1)
# open a connection
with hb_context() as conn:
self.assertEqual(
mocked_hbcl.HBaseConnectionForCrashReports.call_count, 2)
self.assertEqual(a_fake_hbase_connection.close_counter, 1)
# get that same connection again
with hb_context() as conn:
self.assertEqual(
mocked_hbcl.HBaseConnectionForCrashReports.call_count, 2)
self.assertEqual(a_fake_hbase_connection.close_counter, 1)
# get a named connection
with hb_context('fred') as conn:
self.assertEqual(
mocked_hbcl.HBaseConnectionForCrashReports.call_count, 3)
self.assertEqual(a_fake_hbase_connection.close_counter, 1)
self.assertEqual(len(hb_context.pool), 2)
# get that original same connection again
with hb_context() as conn:
self.assertEqual(
mocked_hbcl.HBaseConnectionForCrashReports.call_count, 3)
self.assertEqual(a_fake_hbase_connection.close_counter, 1)
# close all connections
hb_context.close()
self.assertEqual(a_fake_hbase_connection.close_counter, 3)
def _fake_unredacted_processed_crash(self):
d = self._fake_processed_crash()
# these keys do not survive redaction
d['url'] = 'http://very.embarassing.com'
d['email'] = '*****@*****.**'
d['user_id'] = '3333'
d['exploitability'] = 'yep'
d.json_dump = DotDict()
d.json_dump.sensitive = 22
d.upload_file_minidump_flash1 = DotDict()
d.upload_file_minidump_flash1.json_dump = DotDict()
d.upload_file_minidump_flash1.json_dump.sensitive = 33
d.upload_file_minidump_flash2 = DotDict()
d.upload_file_minidump_flash2.json_dump = DotDict()
d.upload_file_minidump_flash2.json_dump.sensitive = 33
d.upload_file_minidump_browser = DotDict()
d.upload_file_minidump_browser.json_dump = DotDict()
d.upload_file_minidump_browser.json_dump.sensitive = DotDict()
d.upload_file_minidump_browser.json_dump.sensitive.exploitable = 55
d.upload_file_minidump_browser.json_dump.sensitive.secret = 66
return d
def test_reprocessing(self):
""" Simple test of reprocessing"""
config_manager = self._setup_config_manager()
c = config_manager.get_config()
cursor = self.conn.cursor()
# Create partitions to support the status query
# Load report_partition_info data
cursor.execute("""
INSERT into reprocessing_jobs
(crash_id)
VALUES
('13c4a348-5d04-11e3-8118-d231feb1dc81')
""")
# We have to do this here to accommodate separate crontabber processes
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = tab.job_state_database['reprocessing-jobs']
assert not information['last_error']
assert information['last_success']
cursor = self.conn.cursor()
cursor.execute('select count(*) from reprocessing_jobs')
res_expected = 0
res, = cursor.fetchone()
eq_(res, res_expected)
self.rabbit_queue_mocked.return_value.save_raw_crash \
.assert_called_once_with(
DotDict({'legacy_processing': 0}),
[],
'13c4a348-5d04-11e3-8118-d231feb1dc81'
)
def test_missing_cpu_count(self):
config = self.get_basic_config()
raw_crash = copy.copy(canonical_standard_raw_crash)
raw_dumps = {}
system_info = copy.copy(
canonical_processed_crash['json_dump']['system_info'])
del system_info['cpu_count']
processed_crash = DotDict()
processed_crash.json_dump = {'system_info': system_info}
processor_meta = self.get_basic_processor_meta()
rule = CPUInfoRule(config)
# the call to be tested
rule.act(raw_crash, raw_dumps, processed_crash, processor_meta)
assert processed_crash.cpu_info == 'GenuineIntel family 6 model 42 stepping 7'
assert processed_crash.cpu_name == 'x86'
# raw crash should be unchanged
assert raw_crash == canonical_standard_raw_crash
def test_hbase_usage_with_transaction(self, mocked_hbcl):
local_config = DotDict({
'hbase_host': 'host',
'database_name': 'name',
'hbase_port': 9090,
'hbase_timeout': 9000,
'number_of_retries': 2,
'logger': SilentFakeLogger(),
})
a_fake_hbase_connection = FakeHB_Connection()
mocked_hbcl.HBaseConnectionForCrashReports = \
mock.Mock(return_value=a_fake_hbase_connection)
hb_context = HBaseConnectionContextPooled(local_config, local_config)
def all_ok(connection, dummy):
self.assertEqual(dummy, 'hello')
return True
transaction = TransactionExecutor(local_config, hb_context)
result = transaction(all_ok, 'hello')
self.assertTrue(result)
self.assertEqual(mocked_hbcl.HBaseConnectionForCrashReports.call_count,
2)
self.assertEqual(a_fake_hbase_connection.close_counter, 1)
self.assertEqual(a_fake_hbase_connection.rollback_counter, 0)
self.assertEqual(a_fake_hbase_connection.commit_counter, 1)
def bad_deal(connection, dummy):
raise KeyError('fred')
self.assertRaises(KeyError, transaction, bad_deal, 'hello')
self.assertEqual(mocked_hbcl.HBaseConnectionForCrashReports.call_count,
2)
self.assertEqual(a_fake_hbase_connection.close_counter, 1)
self.assertEqual(a_fake_hbase_connection.rollback_counter, 1)
self.assertEqual(a_fake_hbase_connection.commit_counter, 1)
hb_context.close()
self.assertEqual(a_fake_hbase_connection.close_counter, 2)
def test_extract_frame_inf(self):
info = ['0', '12', 'msvcr100.dll', '_callthreadstartex',
'f:\\src\\threadex.c', '314', '0x6']
d = DotDict()
bpj._extract_frame_info(info, d)
self.assertTrue('threads' in d)
self.assertEqual(len(d.threads), 1)
self.assertEqual(
d.threads[0],
{
"frame_count": 1,
"frames": [
{
"frame": 12,
"module": 'msvcr100.dll',
"function": '_callthreadstartex',
"file": 'f:\\src\\threadex.c',
"line": 314,
}
]
}
)
def test_extract_frame_info_frames_missing(self):
info = ['4', '12', 'msvcr100.dll', '_callthreadstartex',
'f:\\src\\threadex.c', '314', '0x6']
d = DotDict()
bpj._extract_frame_info(info, d)
ok_('threads' in d)
eq_(len(d.threads), 5)
eq_(
d.threads[4],
{
"frame_count": 1,
"frames": [
{
"frame": 12,
"module": 'msvcr100.dll',
"function": '_callthreadstartex',
"file": 'f:\\src\\threadex.c',
"line": 314,
}
]
}
)
def test_generate_signature_15_replace_address(self):
config = DotDict()
j = JavaSignatureTool(config)
java_stack_trace = """java.lang.IllegalArgumentException: Receiver not registered: org.mozilla.gecko.GeckoNetworkManager@405afea8
at android.app.LoadedApk.forgetReceiverDispatcher(LoadedApk.java:610)
at android.app.ContextImpl.unregisterReceiver(ContextImpl.java:883)
at android.content.ContextWrapper.unregisterReceiver(ContextWrapper.java:331)
at org.mozilla.gecko.GeckoNetworkManager.stopListening(GeckoNetworkManager.java:141)
at org.mozilla.gecko.GeckoNetworkManager.stop(GeckoNetworkManager.java:136)
at org.mozilla.gecko.GeckoApp.onApplicationPause(GeckoApp.java:2130)
at org.mozilla.gecko.GeckoApplication.onActivityPause(GeckoApplication.java:55)
at org.mozilla.gecko.GeckoActivity.onPause(GeckoActivity.java:22)
at org.mozilla.gecko.GeckoApp.onPause(GeckoApp.java:1948)
at android.app.Activity.performPause(Activity.java:3877)
at android.app.Instrumentation.callActivityOnPause(Instrumentation.java:1191)
at android.app.ActivityThread.performPauseActivity(ActivityThread.java:2345)
at android.app.ActivityThread.performPauseActivity(ActivityThread.java:2315)
at android.app.ActivityThread.handlePauseActivity(ActivityThread.java:2295)
at android.app.ActivityThread.access$1700(ActivityThread.java:117)
at android.app.ActivityThread$H.handleMessage(ActivityThread.java:942)
at android.os.Handler.dispatchMessage(Handler.java:99)
at android.os.Looper.loop(Looper.java:130)
at android.app.ActivityThread.main(ActivityThread.java:3691)
at java.lang.reflect.Method.invokeNative(Native Method)
at java.lang.reflect.Method.invoke(Method.java:507)
at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:907)
at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:665)
at dalvik.system.NativeStart.main(Native Method)
"""
sig, notes = j.generate(java_stack_trace, delimiter=': ')
e = (
'java.lang.IllegalArgumentException: '
'Receiver not registered: '
'org.mozilla.gecko.GeckoNetworkManager@<addr>: '
'at android.app.LoadedApk.forgetReceiverDispatcher(LoadedApk.java)'
)
self.assert_equal_with_nicer_output(e, sig)
e = []
self.assert_equal_with_nicer_output(e, notes)
def test_statistics_all_good(self):
d = DotDict()
d.statsd_host = 'localhost'
d.statsd_port = 666
d.prefix = 'a.b'
d.active_counters_list = ['x', 'y', 'z']
with patch('socorro.lib.statistics.StatsClient') as StatsClientMocked:
s = StatisticsForStatsd(d, 'processor')
StatsClientMocked.assert_called_with('localhost', 666,
'a.b.processor')
s.incr('x')
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls,
[call.incr('a.b.processor.x')])
s.incr('y')
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls,
[call.incr('a.b.processor.y')])
s.incr('z')
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls,
[call.incr('a.b.processor.z')])
s.incr('w')
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls, [
call.incr('a.b.processor.y'),
call.incr('a.b.processor.x'),
call.incr('a.b.processor.y')
])
s.incr(None)
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls, [
call.incr('a.b.processor.y'),
call.incr('a.b.processor.x'),
call.incr('a.b.processor.y'),
call.incr('a.b.processor.unknown')
])
def test_stackwalker_fails_2(self, mocked_subprocess_module):
config = self.get_basic_config()
raw_crash = copy.copy(canonical_standard_raw_crash)
raw_dumps = {config.dump_field: 'a_fake_dump.dump'}
processed_crash = DotDict()
processor_meta = self.get_basic_processor_meta()
mocked_subprocess_handle = (
mocked_subprocess_module.Popen.return_value
)
mocked_subprocess_handle.stdout.read.return_value = int
mocked_subprocess_handle.wait.return_value = -1
rule = BreakpadStackwalkerRule2015(config)
# the call to be tested
rule.act(raw_crash, raw_dumps, processed_crash, processor_meta)
eq_(processed_crash.json_dump, {})
eq_(processed_crash.mdsw_return_code, -1)
eq_(processed_crash.mdsw_status_string, "unknown error")
ok_(not processed_crash.success)
eq_(
processor_meta.processor_notes,
[
"{command_pathname} output failed in json: Expected String "
"or Unicode".format(
**config
),
"MDSW failed on 'timeout -s KILL 30 /bin/stackwalker "
"--raw-json /tmp/00000000-0000-0000-0000-000002140504."
"MainThread.TEMPORARY.json --symbols-url https://localhost "
"--symbols-url https://localhost "
"--symbols-cache /mnt/socorro/symbols a_fake_dump.dump "
"2>/dev/null': unknown error"
]
)
def setup_mocked_s3_storage(self, executor=TransactionExecutor):
config = DotDict({
'source': {
'dump_field': 'dump'
},
'transaction_executor_class':
executor,
'backoff_delays': [0, 0, 0],
'redactor_class':
Redactor,
'forbidden_keys':
Redactor.required_config.forbidden_keys.default,
'logger':
mock.Mock(),
'host':
's3.is.out.here.somewhere',
'port':
38080,
'access_key':
'this is the access key',
'secret_access_key':
'secrets',
'buckets':
'daily',
'temporary_file_system_storage_path':
'/i/am/hiding/junk/files/here',
'dump_file_suffix':
'.dump',
})
s3 = BotoS3CrashStorage(config)
s3._connect_to_endpoint = mock.Mock()
s3._mocked_connection = s3._connect_to_endpoint.return_value
s3._calling_format = mock.Mock()
s3._calling_format.return_value = mock.Mock()
s3._CreateError = mock.Mock()
s3._S3ResponseError = mock.Mock()
s3._open = mock.MagicMock()
return s3
class SubmitterFileSystemWalkerSource(CrashStorageBase):
"""This is a crashstorage derivative that can walk an arbitrary file
system path looking for crashes. The new_crashes generator yields
pathnames rather than crash_ids - so it is not compatible with other
instances of the CrashStorageSystem.
"""
required_config = Namespace()
required_config.add_option(
'search_root',
doc="a filesystem location to begin a search for raw crash/dump sets",
short_form='s',
default=None)
required_config.add_option('dump_suffix',
doc="the standard file extension for dumps",
default='.dump')
required_config.add_option('dump_field',
doc="the default name for the main dump",
default='upload_file_minidump')
def __init__(self, config, quit_check_callback=None):
if isinstance(quit_check_callback, basestring):
# this class is being used as a 'new_crash_source' and the name
# of the app has been passed - we can ignore it
quit_check_callback = None
super(SubmitterFileSystemWalkerSource,
self).__init__(config, quit_check_callback)
def get_raw_crash(self, (prefix, path_tuple)):
"""default implemntation of fetching a raw_crash
parameters:
path_tuple - a tuple of paths. the first element is the raw_crash
pathname
"""
with open(path_tuple[0]) as raw_crash_fp:
return DotDict(json.load(raw_crash_fp))
def test_everything_we_hoped_for(self):
config = get_basic_config()
raw_crash = {}
processed_crash = DotDict({
'json_dump': {
'system_info': {
'os': 'Windows NT',
'os_ver': '6.1.7601 Service Pack 1'
}
}
})
processor_meta = get_basic_processor_meta()
rule = OSInfoRule(config)
# the call to be tested
rule.act(raw_crash, {}, processed_crash, processor_meta)
assert processed_crash['os_name'] == "Windows NT"
assert processed_crash['os_version'] == "6.1.7601 Service Pack 1"
# raw crash should be unchanged
assert raw_crash == {}
def test_everything_we_hoped_for(self, mocked_subprocess_module):
config = self.get_basic_config()
raw_crash = copy.copy(canonical_standard_raw_crash)
raw_dumps = {config.dump_field: 'a_fake_dump.dump'}
processed_crash = DotDict()
processor_meta = self.get_basic_processor_meta()
mocked_subprocess_handle = (
mocked_subprocess_module.Popen.return_value)
mocked_subprocess_handle.stdout.read.return_value = (
cannonical_stackwalker_output_str)
mocked_subprocess_handle.wait.return_value = 0
rule = BreakpadStackwalkerRule2015(config)
# the call to be tested
rule.act(raw_crash, raw_dumps, processed_crash, processor_meta)
assert processed_crash.json_dump == cannonical_stackwalker_output
assert processed_crash.mdsw_return_code == 0
assert processed_crash.mdsw_status_string == "OK"
assert processed_crash.success
def test_statistics_all_missing_prefix_and_missing_name(self):
d = DotDict()
d.statsd_host = 'localhost'
d.statsd_port = 666
d.prefix = None
d.active_counters_list = ['x', 'y', 'z']
with patch('socorro.lib.statistics.StatsClient') as StatsClientMocked:
s = StatisticsForStatsd(d, None)
StatsClientMocked.assert_called_with('localhost', 666, '')
s.incr('x')
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls,
[call.incr('x')])
s.incr('y')
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls,
[call.incr('y')])
s.incr('z')
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls,
[call.incr('z')])
s.incr('w')
StatsClientMocked.assert_has_calls(
StatsClientMocked.mock_calls,
[call.incr('y'),
call.incr('x'),
call.incr('y')])
s.incr(None)
StatsClientMocked.assert_has_calls(StatsClientMocked.mock_calls, [
call.incr('y'),
call.incr('x'),
call.incr('y'),
call.incr('unknown')
])
def test_hbase_usage_with_transaction(self):
local_config = DotDict({
'hbase_host': 'host',
'database_name': 'name',
'hbase_port': 9090,
'hbase_timeout': 9000,
'number_of_retries': 2,
'logger': SilentFakeLogger(),
'executor_identity': lambda: 'dwight' # bogus thread id
})
a_fake_hbase_connection = FakeHB_Connection(local_config)
with mock.patch.object(
connection_context, 'HBaseConnection',
mock.Mock(return_value=a_fake_hbase_connection)):
hb_context = connection_context.HBaseConnectionContext(
local_config)
def all_ok(connection, dummy):
eq_(dummy, 'hello')
return True
transaction = TransactionExecutor(local_config, hb_context)
result = transaction(all_ok, 'hello')
ok_(result)
eq_(a_fake_hbase_connection.close_counter, 1)
eq_(a_fake_hbase_connection.rollback_counter, 0)
eq_(a_fake_hbase_connection.commit_counter, 1)
def bad_deal(connection, dummy):
raise KeyError('fred')
assert_raises(KeyError, transaction, bad_deal, 'hello')
eq_(a_fake_hbase_connection.close_counter, 2)
eq_(a_fake_hbase_connection.commit_counter, 1)
hb_context.close()
eq_(a_fake_hbase_connection.close_counter, 2)
def test_external_fails_2(self, mocked_subprocess_module):
config = self.get_basic_config()
raw_crash = copy.copy(canonical_standard_raw_crash)
raw_dumps = {config.dump_field: 'a_fake_dump.dump'}
processed_crash = DotDict()
processor_meta = self.get_basic_processor_meta()
mocked_subprocess_handle = (
mocked_subprocess_module.Popen.return_value)
mocked_subprocess_handle.stdout.read.return_value = int
mocked_subprocess_handle.wait.return_value = -1
rule = ExternalProcessRule(config)
# the call to be tested
rule.act(raw_crash, raw_dumps, processed_crash, processor_meta)
eq_(processed_crash.bogus_command_result, {})
eq_(processed_crash.bogus_command_return_code, -1)
eq_(processor_meta.processor_notes, [
'bogus_command output failed in '
'json: Expected String or Unicode',
])
def test_hbase_usage_with_transaction(self):
local_config = DotDict({
'hbase_host': 'host',
'database_name': 'name',
'hbase_port': 9090,
'hbase_timeout': 9000,
'number_of_retries': 2,
'logger': SilentFakeLogger(),
})
a_fake_hbase_connection = FakeHB_Connection2(local_config)
a_fake_hbase_pool = mock.MagicMock()
a_fake_hbase_pool.connection = a_fake_hbase_connection
with mock.patch.object(happybase, 'ConnectionPool',
mock.Mock(return_value=a_fake_hbase_pool)):
hb_context = HappyBasePooledConnectionContextMock(local_config)
def all_ok(connection, dummy):
self.assertEqual(dummy, 'hello')
return True
transaction = TransactionExecutor(local_config, hb_context)
result = transaction(all_ok, 'hello')
self.assertTrue(result)
self.assertEqual(a_fake_hbase_connection.close_counter, 0)
self.assertEqual(a_fake_hbase_connection.rollback_counter, 0)
self.assertEqual(a_fake_hbase_connection.commit_counter, 1)
def bad_deal(connection, dummy):
raise KeyError('fred')
self.assertRaises(KeyError, transaction, bad_deal, 'hello')
self.assertEqual(a_fake_hbase_connection.close_counter, 0)
self.assertEqual(a_fake_hbase_connection.commit_counter, 1)
hb_context.close()
self.assertEqual(a_fake_hbase_connection.close_counter, 0)
def test_hbase_crashstorage_error_after_retries(self):
cshbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
cchbaseclient_ = \
'socorro.external.hbase.connection_context.hbase_client'
with nested(mock.patch(cshbaseclient_),
mock.patch(cchbaseclient_)) as (cshclient, cchclient):
fake_hbase_client_connection = mock.MagicMock()
cshclient.HBaseConnectionForCrashReports.return_value = \
fake_hbase_client_connection
fake_put_json_method = mock.MagicMock()
cshclient.HBaseConnectionForCrashReports.put_json_dump = \
fake_put_json_method
cchclient.HBaseConnectionForCrashReports.return_value = \
fake_hbase_client_connection
fake_hbase_client_connection.hbaseThriftExceptions = \
(SomeThriftError,)
fake_put_json_method.side_effect = SomeThriftError('try again')
config = DotDict({
'logger': mock.MagicMock(),
'hbase_timeout': 0,
'hbase_host': 'somehost',
'hbase_port': 9090,
'number_of_retries': 2,
'hbase_connection_pool_class': HBaseConnectionContextPooled,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0]
})
crashstorage = HBaseCrashStorage(config)
raw = ('{"name":"Peter", '
'"submitted_timestamp":"%d"}' % time.time())
self.assertRaises(SomeThriftError, crashstorage.save_raw_crash,
json.loads(raw), raw, {})
self.assertEqual(fake_put_json_method.call_count, 3)
