def setUp(self):
super(TestCrashStorage, self).setUp()
self.context = mock.MagicMock()
self.context.__enter__.return_value = self.context
config = DotDict({
'hbase_host':
'host',
'database_name':
'name',
'hbase_port':
9090,
'hbase_timeout':
9000,
'number_of_retries':
2,
'logger':
SilentFakeLogger(),
'hbase_connection_context_class':
mock.Mock(return_value=self.context),
'transaction_executor_class':
TransactionExecutor,
'new_crash_limit':
10**6,
'redactor_class':
Redactor,
'forbidden_keys':
Redactor.required_config.forbidden_keys.default,
})
self.storage = HBaseCrashStorage(config)
def setUp(self):
self.context = mock.MagicMock()
self.context.__enter__.return_value = self.context
config = DotDict({
'hbase_host': 'host',
'database_name': 'name',
'hbase_port': 9090,
'hbase_timeout': 9000,
'number_of_retries': 2,
'logger': SilentFakeLogger(),
'hbase_connection_context_class': mock.Mock(
return_value=self.context
),
'transaction_executor_class': TransactionExecutor,
'new_crash_limit': 10 ** 6,
'redactor_class': Redactor,
'forbidden_keys': Redactor.required_config.forbidden_keys.default,
})
self.storage = HBaseCrashStorage(config)
class TestCrashStorage(TestCase):
def setUp(self):
super(TestCrashStorage, self).setUp()
self.context = mock.MagicMock()
self.context.__enter__.return_value = self.context
config = DotDict({
'hbase_host': 'host',
'database_name': 'name',
'hbase_port': 9090,
'hbase_timeout': 9000,
'number_of_retries': 2,
'logger': SilentFakeLogger(),
'hbase_connection_context_class': mock.Mock(
return_value=self.context
),
'transaction_executor_class': TransactionExecutor,
'new_crash_limit': 10 ** 6,
'redactor_class': Redactor,
'forbidden_keys': Redactor.required_config.forbidden_keys.default,
})
self.storage = HBaseCrashStorage(config)
def _fake_processed_crash(self):
d = DotDict()
# these keys survive redaction
d.a = DotDict()
d.a.b = DotDict()
d.a.b.c = 11
d.sensitive = DotDict()
d.sensitive.x = 2
d.not_url = 'not a url'
return d
def _fake_redacted_processed_crash(self):
d = self._fake_unredacted_processed_crash()
del d.url
del d.email
del d.user_id
del d.exploitability
del d.json_dump.sensitive
del d.upload_file_minidump_flash1.json_dump.sensitive
del d.upload_file_minidump_flash2.json_dump.sensitive
del d.upload_file_minidump_browser.json_dump.sensitive
return d
def _fake_unredacted_processed_crash(self):
d = self._fake_processed_crash()
# these keys do not survive redaction
d['url'] = 'http://very.embarassing.com'
d['email'] = '*****@*****.**'
d['user_id'] = '3333'
d['exploitability'] = 'yep'
d.json_dump = DotDict()
d.json_dump.sensitive = 22
d.upload_file_minidump_flash1 = DotDict()
d.upload_file_minidump_flash1.json_dump = DotDict()
d.upload_file_minidump_flash1.json_dump.sensitive = 33
d.upload_file_minidump_flash2 = DotDict()
d.upload_file_minidump_flash2.json_dump = DotDict()
d.upload_file_minidump_flash2.json_dump.sensitive = 33
d.upload_file_minidump_browser = DotDict()
d.upload_file_minidump_browser.json_dump = DotDict()
d.upload_file_minidump_browser.json_dump.sensitive = DotDict()
d.upload_file_minidump_browser.json_dump.sensitive.exploitable = 55
d.upload_file_minidump_browser.json_dump.sensitive.secret = 66
return d
def _fake_unredacted_processed_crash_as_string(self):
d = self._fake_unredacted_processed_crash()
s = json.dumps(d)
return s
def test_close(self):
self.storage.close()
self.assertEqual(self.storage.hbase.close.call_count, 0)
def test_save_processed(self):
self.storage.save_processed({
"uuid": "936ce666-ff3b-4c7a-9674-367fe2120408",
"completeddatetime": "2012-04-08 10:56:50.902884",
"signature": 'now_this_is_a_signature'
})
with self.storage.hbase() as conn:
self.assertEqual(conn.table.call_count, 1)
#self.assertEqual(conn.client.mutateRow.call_count, 2)
def test_save_raw_crash(self):
self.storage.save_raw_crash({
"submitted_timestamp": "2013-01-09T22:21:18.646733+00:00"
}, MemoryDumpsMapping(), "0bba929f-8721-460c-dead-a43c20071027")
with self.storage.hbase() as conn:
self.assertEqual(conn.table.call_count, 1)
self.assertEqual(conn.table.return_value.put.call_count, 1)
def test_save_raw_crash_hang(self):
self.storage.save_raw_crash({
"submitted_timestamp": "2013-01-09T22:21:18.646733+00:00",
"HangID": "?"
}, MemoryDumpsMapping(), "0bba929f-8721-460c-dead-a43c20071027")
with self.storage.hbase() as conn:
self.assertEqual(conn.table.call_count, 1)
self.assertEqual(conn.table.return_value.put.call_count, 1)
def test_get_raw_dumps(self):
self.storage.get_raw_dumps("936ce666-ff3b-4c7a-9674-367fe2120408")
with self.storage.hbase() as conn:
self.assertEqual(conn.table.return_value.row.call_count, 1)
def test_get_raw_dumps_as_files(self):
self.storage.get_raw_dumps_as_files(
"936ce666-ff3b-4c7a-9674-367fe2120408")
with self.storage.hbase() as conn:
self.assertEqual(conn.table.return_value.row.call_count, 1)
def test_get_unredacted_processed(self):
processed_crash = DotDict()
with self.storage.hbase() as conn:
conn.table.return_value.row.return_value = {
'processed_data:json':
self._fake_unredacted_processed_crash_as_string()
}
processed_crash = self.storage.get_unredacted_processed(
"936ce666-ff3b-4c7a-9674-367fe2120408"
)
self.assertEqual(
processed_crash,
self._fake_unredacted_processed_crash()
)
def test_get_processed(self):
faked_hb_row_object = DotDict()
faked_hb_row_object.columns = DotDict()
faked_hb_row_object.columns['processed_data:json'] = DotDict()
faked_hb_row_object.columns['processed_data:json'].value = \
self._fake_unredacted_processed_crash_as_string()
processed_crash = DotDict()
with self.storage.hbase() as conn:
conn.table.return_value.row.return_value = {
'processed_data:json':
self._fake_unredacted_processed_crash_as_string()
}
processed_crash = self.storage.get_processed(
"936ce666-ff3b-4c7a-9674-367fe2120408"
)
self.assertEqual(
processed_crash,
self._fake_redacted_processed_crash()
)
def test_get_processed_failure(self):
with self.storage.hbase() as conn:
conn.table.return_value.row.return_value = {}
self.assertRaises(
CrashIDNotFound,
self.storage.get_processed,
"936ce666-ff3b-4c7a-9674-367fe2120408"
)
def test_new_crashes(self):
self.storage._salted_scanner_iterable = mock.Mock(
return_value=iter([])
)
self.assertEqual(list(self.storage.new_crashes()), [])
