def test_basic_hbase_crashstorage(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
}]
)
with config_manager.context() as config:
crashstorage = HBaseCrashStorage(config)
self.assertEqual(list(crashstorage.new_crashes()), [])
crash_id = '86b58ff2-9708-487d-bfc4-9dac32121214'
raw = ('{"name":"Peter", '
'"submitted_timestamp":"%d"}' % time.time())
fake_raw_dump_1 = 'peter is a swede'
fake_raw_dump_2 = 'lars is a norseman'
fake_raw_dump_3 = 'adrian is a frenchman'
fake_dumps = {'upload_file_minidump': fake_raw_dump_1,
'lars': fake_raw_dump_2,
'adrian': fake_raw_dump_3}
crashstorage.save_raw_crash(json.loads(raw),
fake_dumps,
crash_id)
assert config.logger.info.called
assert config.logger.info.call_count > 1
msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0]
# ie logging.info(<template>, <arg>)
msg = msg_tmpl % msg_arg
self.assertTrue('saved' in msg)
self.assertTrue(crash_id in msg)
raw_crash = crashstorage.get_raw_crash(crash_id)
assert isinstance(raw_crash, dict)
self.assertEqual(raw_crash['name'], 'Peter')
dump = crashstorage.get_raw_dump(crash_id)
assert isinstance(dump, basestring)
self.assertTrue('peter is a swede' in dump)
dumps = crashstorage.get_raw_dumps(crash_id)
assert isinstance(dumps, dict)
self.assertTrue('upload_file_minidump' in dumps)
self.assertTrue('lars' in dumps)
self.assertTrue('adrian' in dumps)
self.assertEqual(dumps['upload_file_minidump'],
fake_dumps['upload_file_minidump'])
self.assertEqual(dumps['lars'],
fake_dumps['lars'])
self.assertEqual(dumps['adrian'],
fake_dumps['adrian'])
# hasn't been processed yet
self.assertRaises(CrashIDNotFound,
crashstorage.get_processed,
crash_id)
pro = ('{"name":"Peter",'
'"uuid":"86b58ff2-9708-487d-bfc4-9dac32121214", '
'"submitted_timestamp":"%d", '
'"completeddatetime": "%d"}' %
(time.time(), time.time()))
crashstorage.save_processed(json.loads(pro))
data = crashstorage.get_processed(crash_id)
self.assertEqual(data['name'], u'Peter')
hb_connection = crashstorage.hbaseConnectionPool.connection()
self.assertTrue(hb_connection.transport.isOpen())
crashstorage.close()
self.assertFalse(hb_connection.transport.isOpen())
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0]
}]
)
with config_manager.context() as config:
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
expected_dump_2 = fake_binary_dump + " number 2"
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump,
"abc123")
self.assertEqual(
klass.put_json_dump.call_count,
1
)
a = klass.put_json_dump.call_args
self.assertEqual(len(a[0]), 4)
#self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_raw_crash)
self.assertEqual(a[0][3], expected_dump)
self.assertEqual(a[1], {'number_of_retries': 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {
"name": "Peter",
"uuid": "abc123",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
self.assertEqual(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_processed_crash)
self.assertEqual(a[1], {'number_of_retries': 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_json.call_count, 1)
self.assertEqual(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dump.call_count, 1)
self.assertEqual(r, expected_dump)
# test get_raw_dumps
m = mock.Mock(return_value={'upload_file_minidump':
fake_binary_dump})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dumps.call_count, 1)
self.assertEqual(r, {'upload_file_minidump': expected_dump})
# test get_raw_dumps 2
m = mock.Mock(return_value={'upload_file_minidump':
fake_binary_dump,
'aux_1':
expected_dump_2})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dumps.call_count, 1)
self.assertEqual(r, {'upload_file_minidump':
fake_binary_dump,
'aux_1':
expected_dump_2})
# test get_processed
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_processed_json.call_count, 1)
self.assertEqual(r, expected_processed_crash)
def test_basic_hbase_crashstorage(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'hbase_timeout':
100,
'hbase_host':
commonconfig.hbaseHost.default,
'hbase_port':
commonconfig.hbasePort.default,
}])
with config_manager.context() as config:
crashstorage = HBaseCrashStorage(config)
self.assertEqual(list(crashstorage.new_crashes()), [])
crash_id = '86b58ff2-9708-487d-bfc4-9dac32121214'
raw = ('{"name":"Peter", '
'"submitted_timestamp":"%d"}' % time.time())
fake_raw_dump_1 = 'peter is a swede'
fake_raw_dump_2 = 'lars is a norseman'
fake_raw_dump_3 = 'adrian is a frenchman'
fake_dumps = {
'upload_file_minidump': fake_raw_dump_1,
'lars': fake_raw_dump_2,
'adrian': fake_raw_dump_3
}
crashstorage.save_raw_crash(json.loads(raw), fake_dumps,
crash_id)
assert config.logger.info.called
assert config.logger.info.call_count > 1
msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0]
# ie logging.info(<template>, <arg>)
msg = msg_tmpl % msg_arg
self.assertTrue('saved' in msg)
self.assertTrue(crash_id in msg)
raw_crash = crashstorage.get_raw_crash(crash_id)
assert isinstance(raw_crash, dict)
self.assertEqual(raw_crash['name'], 'Peter')
dump = crashstorage.get_raw_dump(crash_id)
assert isinstance(dump, basestring)
self.assertTrue('peter is a swede' in dump)
dumps = crashstorage.get_raw_dumps(crash_id)
assert isinstance(dumps, dict)
self.assertTrue('upload_file_minidump' in dumps)
self.assertTrue('lars' in dumps)
self.assertTrue('adrian' in dumps)
self.assertEqual(dumps['upload_file_minidump'],
fake_dumps['upload_file_minidump'])
self.assertEqual(dumps['lars'], fake_dumps['lars'])
self.assertEqual(dumps['adrian'], fake_dumps['adrian'])
# hasn't been processed yet
self.assertRaises(CrashIDNotFound, crashstorage.get_processed,
crash_id)
pro = ('{"name":"Peter",'
'"uuid":"86b58ff2-9708-487d-bfc4-9dac32121214", '
'"submitted_timestamp":"%d", '
'"completeddatetime": "%d"}' %
(time.time(), time.time()))
crashstorage.save_processed(json.loads(pro))
data = crashstorage.get_processed(crash_id)
self.assertEqual(data['name'], u'Peter')
hb_connection = crashstorage.hbaseConnectionPool.connection()
self.assertTrue(hb_connection.transport.isOpen())
crashstorage.close()
self.assertFalse(hb_connection.transport.isOpen())
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0]
}])
with config_manager.context() as config:
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
expected_dump_2 = fake_binary_dump + " number 2"
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump,
"abc123")
self.assertEqual(klass.put_json_dump.call_count, 1)
a = klass.put_json_dump.call_args
self.assertEqual(len(a[0]), 4)
#self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_raw_crash)
self.assertEqual(a[0][3], expected_dump)
self.assertEqual(a[1], {'number_of_retries': 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {
"name": "Peter",
"uuid": "abc123",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
self.assertEqual(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_processed_crash)
self.assertEqual(a[1], {'number_of_retries': 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_json.call_count, 1)
self.assertEqual(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dump.call_count, 1)
self.assertEqual(r, expected_dump)
# test get_raw_dumps
m = mock.Mock(
return_value={'upload_file_minidump': fake_binary_dump})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dumps.call_count, 1)
self.assertEqual(r, {'upload_file_minidump': expected_dump})
# test get_raw_dumps 2
m = mock.Mock(
return_value={
'upload_file_minidump': fake_binary_dump,
'aux_1': expected_dump_2
})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dumps.call_count, 1)
self.assertEqual(
r, {
'upload_file_minidump': fake_binary_dump,
'aux_1': expected_dump_2
})
# test get_processed
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_processed_json.call_count, 1)
self.assertEqual(r, expected_processed_crash)
def test_basic_hbase_crashstorage(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option("logger", default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name="testapp",
app_version="1.0",
app_description="app description",
values_source_list=[
{
"logger": mock_logging,
"hbase_timeout": 100,
"hbase_host": commonconfig.hbaseHost.default,
"hbase_port": commonconfig.hbasePort.default,
}
],
argv_source=[],
)
with config_manager.context() as config:
crashstorage = HBaseCrashStorage(config)
eq_(list(crashstorage.new_crashes()), [])
crash_id = "86b58ff2-9708-487d-bfc4-9dac32121214"
raw = '{"name":"Peter", ' '"submitted_timestamp":"%d"}' % time.time()
fake_raw_dump_1 = "peter is a swede"
fake_raw_dump_2 = "lars is a norseman"
fake_raw_dump_3 = "adrian is a frenchman"
fake_dumps = MemoryDumpsMapping(
{"upload_file_minidump": fake_raw_dump_1, "lars": fake_raw_dump_2, "adrian": fake_raw_dump_3}
)
crashstorage.save_raw_crash(json.loads(raw), fake_dumps, crash_id)
assert config.logger.info.called
assert config.logger.info.call_count > 1
msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0]
# ie logging.info(<template>, <arg>)
msg = msg_tmpl % msg_arg
ok_("saved" in msg)
ok_(crash_id in msg)
raw_crash = crashstorage.get_raw_crash(crash_id)
assert isinstance(raw_crash, dict)
eq_(raw_crash["name"], "Peter")
dump = crashstorage.get_raw_dump(crash_id)
assert isinstance(dump, basestring)
ok_("peter is a swede" in dump)
dumps = crashstorage.get_raw_dumps(crash_id)
assert isinstance(dumps, dict)
ok_("upload_file_minidump" in dumps)
ok_("lars" in dumps)
ok_("adrian" in dumps)
eq_(dumps["upload_file_minidump"], fake_dumps["upload_file_minidump"])
eq_(dumps["lars"], fake_dumps["lars"])
eq_(dumps["adrian"], fake_dumps["adrian"])
# hasn't been processed yet
assert_raises(CrashIDNotFound, crashstorage.get_processed, crash_id)
pro = (
'{"name":"Peter",'
'"uuid":"86b58ff2-9708-487d-bfc4-9dac32121214", '
'"submitted_timestamp":"%d", '
'"completeddatetime": "%d"}' % (time.time(), time.time())
)
crashstorage.save_processed(json.loads(pro))
data = crashstorage.get_processed(crash_id)
eq_(data["name"], u"Peter")
hb_connection = crashstorage.hbaseConnectionPool.connection()
ok_(hb_connection.transport.isOpen())
crashstorage.close()
ok_(not hb_connection.transport.isOpen())
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.get_required_config()
required_config.add_option("logger", default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name="testapp",
app_version="1.0",
app_description="app description",
values_source_list=[
{
"logger": mock_logging,
"hbase_timeout": 100,
"hbase_host": commonconfig.hbaseHost.default,
"hbase_port": commonconfig.hbasePort.default,
"transaction_executor_class": TransactionExecutorWithLimitedBackoff,
"backoff_delays": [0, 0, 0],
}
],
argv_source=[],
)
with config_manager.context() as config:
config.executor_identity = lambda: "dwight" # bogus thread id
hbaseclient_ = "socorro.external.hbase.crashstorage.hbase_client"
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
expected_dump_2 = fake_binary_dump + " number 2"
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump, "abc123")
eq_(klass.put_json_dump.call_count, 1)
a = klass.put_json_dump.call_args
eq_(len(a[0]), 4)
# eq_(a[0][1], "abc123")
eq_(a[0][2], expected_raw_crash)
eq_(a[0][3], expected_dump)
eq_(a[1], {"number_of_retries": 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {"name": "Peter", "uuid": "abc123"}
expected_unredacted_processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
eq_(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
eq_(len(a[0]), 3)
eq_(a[0][1], "abc123")
eq_(a[0][2], expected_unredacted_processed_crash)
eq_(a[1], {"number_of_retries": 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
ok_(isinstance(r, DotDict))
a = klass.get_json.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_json.call_count, 1)
eq_(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
eq_(len(a[0]), 3)
eq_(a[0][1], "abc123")
eq_(klass.get_dump.call_count, 1)
eq_(r, expected_dump)
# test get_raw_dumps
m = mock.Mock(return_value={"upload_file_minidump": fake_binary_dump})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_dumps.call_count, 1)
eq_(r, {"upload_file_minidump": expected_dump})
# test get_raw_dumps 2
m = mock.Mock(return_value={"upload_file_minidump": fake_binary_dump, "aux_1": expected_dump_2})
klass.get_dumps = m
r = crashstorage.get_raw_dumps("abc123")
a = klass.get_dumps.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_dumps.call_count, 1)
eq_(r, {"upload_file_minidump": fake_binary_dump, "aux_1": expected_dump_2})
# test get_processed
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed("abc123")
ok_(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
eq_(len(a[0]), 2)
eq_(a[0][1], "abc123")
eq_(klass.get_processed_json.call_count, 1)
eq_(r, expected_processed_crash)
