def test_basic_hbase_crashstorage(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'hbase_timeout':
100,
'hbase_host':
commonconfig.hbaseHost.default,
'hbase_port':
commonconfig.hbasePort.default,
}])
with config_manager.context() as config:
crashstorage = HBaseCrashStorage(config)
self.assertEqual(list(crashstorage.new_crashes()), [])
# data doesn't contain an 'ooid' key
#raw = '{"name": "Peter"}'
#self.assertRaises(
#CrashIDNotFound,
#crashstorage.save_raw_crash,
#json.loads(raw),
#raw
#)
#raw = '{"name":"Peter","ooid":"abc123"}'
#self.assertRaises(
#ValueError, # missing the 'submitted_timestamp' key
#crashstorage.save_raw_crash,
#json.loads(raw),
#raw
#)
raw = ('{"name":"Peter", '
'"submitted_timestamp":"%d"}' % time.time())
crashstorage.save_raw_crash(json.loads(raw), raw, "abc123")
assert config.logger.info.called
assert config.logger.info.call_count > 1
msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0]
# ie logging.info(<template>, <arg>)
msg = msg_tmpl % msg_arg
self.assertTrue('saved' in msg)
self.assertTrue('abc123' in msg)
meta = crashstorage.get_raw_crash('abc123')
assert isinstance(meta, dict)
self.assertEqual(meta['name'], 'Peter')
dump = crashstorage.get_raw_dump('abc123')
assert isinstance(dump, basestring)
self.assertTrue('"name":"Peter"' in dump)
# hasn't been processed yet
self.assertRaises(CrashIDNotFound,
crashstorage.get_processed_crash, 'abc123')
pro = ('{"name":"Peter","uuid":"abc123", '
'"submitted_timestamp":"%d", '
'"completeddatetime": "%d"}' %
(time.time(), time.time()))
crashstorage.save_processed(json.loads(pro))
data = crashstorage.get_processed_crash('abc123')
self.assertEqual(data['name'], u'Peter')
assert crashstorage.hbaseConnection.transport.isOpen()
crashstorage.close()
transport = crashstorage.hbaseConnection.transport
self.assertTrue(not transport.isOpen())
def test_basic_hbase_crashstorage(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
}]
)
with config_manager.context() as config:
crashstorage = HBaseCrashStorage(config)
self.assertEqual(list(crashstorage.new_crashes()), [])
# data doesn't contain an 'ooid' key
#raw = '{"name": "Peter"}'
#self.assertRaises(
#CrashIDNotFound,
#crashstorage.save_raw_crash,
#json.loads(raw),
#raw
#)
#raw = '{"name":"Peter","ooid":"abc123"}'
#self.assertRaises(
#ValueError, # missing the 'submitted_timestamp' key
#crashstorage.save_raw_crash,
#json.loads(raw),
#raw
#)
raw = ('{"name":"Peter", '
'"submitted_timestamp":"%d"}' % time.time())
crashstorage.save_raw_crash(json.loads(raw), raw, "abc123")
assert config.logger.info.called
assert config.logger.info.call_count > 1
msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0]
# ie logging.info(<template>, <arg>)
msg = msg_tmpl % msg_arg
self.assertTrue('saved' in msg)
self.assertTrue('abc123' in msg)
meta = crashstorage.get_raw_crash('abc123')
assert isinstance(meta, dict)
self.assertEqual(meta['name'], 'Peter')
dump = crashstorage.get_raw_dump('abc123')
assert isinstance(dump, basestring)
self.assertTrue('"name":"Peter"' in dump)
# hasn't been processed yet
self.assertRaises(CrashIDNotFound,
crashstorage.get_processed_crash,
'abc123')
pro = ('{"name":"Peter","uuid":"abc123", '
'"submitted_timestamp":"%d", '
'"completeddatetime": "%d"}' %
(time.time(), time.time()))
crashstorage.save_processed(json.loads(pro))
data = crashstorage.get_processed_crash('abc123')
self.assertEqual(data['name'], u'Peter')
assert crashstorage.hbaseConnection.transport.isOpen()
crashstorage.close()
transport = crashstorage.hbaseConnection.transport
self.assertTrue(not transport.isOpen())
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0]
}])
with config_manager.context() as config:
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump,
"abc123")
self.assertEqual(klass.put_json_dump.call_count, 1)
a = klass.put_json_dump.call_args
self.assertEqual(len(a[0]), 4)
#self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_raw_crash)
self.assertEqual(a[0][3], expected_dump)
self.assertEqual(a[1], {'number_of_retries': 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {
"name": "Peter",
"uuid": "abc123",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
self.assertEqual(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_processed_crash)
self.assertEqual(a[1], {'number_of_retries': 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_json.call_count, 1)
self.assertEqual(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dump.call_count, 1)
self.assertEqual(r, expected_dump)
# test get_processed_crash
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed_crash("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_processed_json.call_count, 1)
self.assertEqual(r, expected_processed_crash)
def test_hbase_crashstorage_puts_and_gets(self):
mock_logging = mock.Mock()
required_config = HBaseCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'hbase_timeout': 100,
'hbase_host': commonconfig.hbaseHost.default,
'hbase_port': commonconfig.hbasePort.default,
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff,
'backoff_delays': [0, 0, 0]
}]
)
with config_manager.context() as config:
hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client'
with mock.patch(hbaseclient_) as hclient:
# test save_raw_crash
raw_crash = {
"name": "Peter",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"submitted_timestamp": "2012-05-04T15:10:00",
"user_id": "000-00-0000",
}
fake_binary_dump = "this a bogus binary dump"
expected_raw_crash = raw_crash
expected_dump = fake_binary_dump
# saves us from loooong lines
klass = hclient.HBaseConnectionForCrashReports
crashstorage = HBaseCrashStorage(config)
crashstorage.save_raw_crash(raw_crash, fake_binary_dump,
"abc123")
self.assertEqual(
klass.put_json_dump.call_count,
1
)
a = klass.put_json_dump.call_args
self.assertEqual(len(a[0]), 4)
#self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_raw_crash)
self.assertEqual(a[0][3], expected_dump)
self.assertEqual(a[1], {'number_of_retries': 0})
# test save_processed
processed_crash = {
"name": "Peter",
"uuid": "abc123",
"email": "*****@*****.**",
"url": "http://embarassing.xxx",
"user_id": "000-00-0000",
}
expected_processed_crash = {
"name": "Peter",
"uuid": "abc123",
}
crashstorage = HBaseCrashStorage(config)
crashstorage.save_processed(processed_crash)
self.assertEqual(klass.put_processed_json.call_count, 1)
a = klass.put_processed_json.call_args
self.assertEqual(len(a[0]), 3)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(a[0][2], expected_processed_crash)
self.assertEqual(a[1], {'number_of_retries': 0})
# test get_raw_crash
m = mock.Mock(return_value=raw_crash)
klass.get_json = m
r = crashstorage.get_raw_crash("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_json.call_count, 1)
self.assertEqual(r, expected_raw_crash)
# test get_raw_dump
m = mock.Mock(return_value=fake_binary_dump)
klass.get_dump = m
r = crashstorage.get_raw_dump("abc123")
a = klass.get_dump.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_dump.call_count, 1)
self.assertEqual(r, expected_dump)
# test get_processed_crash
m = mock.Mock(return_value=expected_processed_crash)
klass.get_processed_json = m
r = crashstorage.get_processed_crash("abc123")
self.assertTrue(isinstance(r, DotDict))
a = klass.get_processed_json.call_args
self.assertEqual(len(a[0]), 2)
self.assertEqual(a[0][1], "abc123")
self.assertEqual(klass.get_processed_json.call_count, 1)
self.assertEqual(r, expected_processed_crash)
