def test_hbase_crashstorage_basic_error(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
}])
with config_manager.context() as config:
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
klass = hclient.HBaseConnectionForCrashReports
def retry_raiser(*args, **kwargs):
raise SomeThriftError('try again')
klass.put_json_dump.side_effect = ValueError('crap!')
crashstorage = HBaseCrashStorage(config)
raw = ('{"name":"Peter", '
'"submitted_timestamp":"%d"}' % time.time())
# Note, we're not expect it to raise an error
self.assertRaises(ValueError, crashstorage.save_raw_crash,
json.loads(raw), raw, "abc123")
def test_hbase_crashstorage_basic_error(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
}],
argv_source=[]
)
with config_manager.context() as config:
config.executor_identity = lambda: 'dwight' # bogus thread id
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
klass = hclient.HBaseConnectionForCrashReports
def retry_raiser(*args, **kwargs):
raise SomeThriftError('try again')
klass.put_json_dump.side_effect = ValueError('crap!')
crashstorage = HBaseCrashStorage(config)
raw = ('{"name":"Peter", '
'"submitted_timestamp":"%d"}' % time.time())
# Note, we're not expect it to raise an error
assert_raises(ValueError,
crashstorage.save_raw_crash,
json.loads(raw),
raw,
"abc123"
)
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}])
with config_manager.context() as config:
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
expected_dump_2 = fake_binary_dump + " number 2"
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump,
"abc123")
self.assertEqual(klass.put_json_dump.call_count, 1)
a = klass.put_json_dump.call_args
self.assertEqual(len(a[0]), 4)
#self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_raw_crash)
self.assertEqual(a[0][3], expected_dump)
self.assertEqual(a[1], {'number_of_retries': 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {
"name": "Peter",
"uuid": "abc123",
}
expected_unredacted_processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
self.assertEqual(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_unredacted_processed_crash)
self.assertEqual(a[1], {'number_of_retries': 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_json.call_count, 1)
self.assertEqual(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dump.call_count, 1)
self.assertEqual(r, expected_dump)
# test get_raw_dumps
m = mock.Mock(
return_value={'upload_file_minidump': fake_binary_dump})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dumps.call_count, 1)
self.assertEqual(r, {'upload_file_minidump': expected_dump})
# test get_raw_dumps 2
m = mock.Mock(
return_value={
'upload_file_minidump': fake_binary_dump,
'aux_1': expected_dump_2
})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dumps.call_count, 1)
self.assertEqual(
r, {
'upload_file_minidump': fake_binary_dump,
'aux_1': expected_dump_2
})
# test get_processed
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_processed_json.call_count, 1)
self.assertEqual(r, expected_processed_crash)
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0],
}]
)
with config_manager.context() as config:
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
expected_dump_2 = fake_binary_dump + " number 2"
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump,
"abc123")
self.assertEqual(
klass.put_json_dump.call_count,
1
)
a = klass.put_json_dump.call_args
self.assertEqual(len(a[0]), 4)
#self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_raw_crash)
self.assertEqual(a[0][3], expected_dump)
self.assertEqual(a[1], {'number_of_retries': 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {
"name": "Peter",
"uuid": "abc123",
}
expected_unredacted_processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
self.assertEqual(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_unredacted_processed_crash)
self.assertEqual(a[1], {'number_of_retries': 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_json.call_count, 1)
self.assertEqual(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dump.call_count, 1)
self.assertEqual(r, expected_dump)
# test get_raw_dumps
m = mock.Mock(return_value={'upload_file_minidump':
fake_binary_dump})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dumps.call_count, 1)
self.assertEqual(r, {'upload_file_minidump': expected_dump})
# test get_raw_dumps 2
m = mock.Mock(return_value={'upload_file_minidump':
fake_binary_dump,
'aux_1':
expected_dump_2})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dumps.call_count, 1)
self.assertEqual(r, {'upload_file_minidump':
fake_binary_dump,
'aux_1':
expected_dump_2})
# test get_processed
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_processed_json.call_count, 1)
self.assertEqual(r, expected_processed_crash)
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.get_required_config()
required_config.add_option("logger", default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name="testapp",
app_version="1.0",
app_description="app description",
values_source_list=[
{
"logger": mock_logging,
"hbase_timeout": 100,
"hbase_host": commonconfig.hbaseHost.default,
"hbase_port": commonconfig.hbasePort.default,
"transaction_executor_class": TransactionExecutorWithLimitedBackoff,
"backoff_delays": [0, 0, 0],
}
],
argv_source=[],
)
with config_manager.context() as config:
config.executor_identity = lambda: "dwight" # bogus thread id
hbaseclient_ = "socorro.external.hbase.crashstorage.hbase_client"
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
expected_dump_2 = fake_binary_dump + " number 2"
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump, "abc123")
eq_(klass.put_json_dump.call_count, 1)
a = klass.put_json_dump.call_args
eq_(len(a[0]), 4)
# eq_(a[0][1], "abc123")
eq_(a[0][2], expected_raw_crash)
eq_(a[0][3], expected_dump)
eq_(a[1], {"number_of_retries": 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {"name": "Peter", "uuid": "abc123"}
expected_unredacted_processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
eq_(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
eq_(len(a[0]), 3)
eq_(a[0][1], "abc123")
eq_(a[0][2], expected_unredacted_processed_crash)
eq_(a[1], {"number_of_retries": 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
ok_(isinstance(r, DotDict))
a = klass.get_json.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_json.call_count, 1)
eq_(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
eq_(len(a[0]), 3)
eq_(a[0][1], "abc123")
eq_(klass.get_dump.call_count, 1)
eq_(r, expected_dump)
# test get_raw_dumps
m = mock.Mock(return_value={"upload_file_minidump": fake_binary_dump})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_dumps.call_count, 1)
eq_(r, {"upload_file_minidump": expected_dump})
# test get_raw_dumps 2
m = mock.Mock(return_value={"upload_file_minidump": fake_binary_dump, "aux_1": expected_dump_2})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_dumps.call_count, 1)
eq_(r, {"upload_file_minidump": fake_binary_dump, "aux_1": expected_dump_2})
# test get_processed
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed("abc123")
ok_(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_processed_json.call_count, 1)
eq_(r, expected_processed_crash)
