def test_basic_hbase_crashstorage(self): mock_logging = mock.Mock() required_config = HBaseCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'hbase_timeout': 100, 'hbase_host': commonconfig.hbaseHost.default, 'hbase_port': commonconfig.hbasePort.default, }]) with config_manager.context() as config: crashstorage = HBaseCrashStorage(config) self.assertEqual(list(crashstorage.new_crashes()), []) # data doesn't contain an 'ooid' key #raw = '{"name": "Peter"}' #self.assertRaises( #CrashIDNotFound, #crashstorage.save_raw_crash, #json.loads(raw), #raw #) #raw = '{"name":"Peter","ooid":"abc123"}' #self.assertRaises( #ValueError, # missing the 'submitted_timestamp' key #crashstorage.save_raw_crash, #json.loads(raw), #raw #) raw = ('{"name":"Peter", ' '"submitted_timestamp":"%d"}' % time.time()) crashstorage.save_raw_crash(json.loads(raw), raw, "abc123") assert config.logger.info.called assert config.logger.info.call_count > 1 msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0] # ie logging.info(<template>, <arg>) msg = msg_tmpl % msg_arg self.assertTrue('saved' in msg) self.assertTrue('abc123' in msg) meta = crashstorage.get_raw_crash('abc123') assert isinstance(meta, dict) self.assertEqual(meta['name'], 'Peter') dump = crashstorage.get_raw_dump('abc123') assert isinstance(dump, basestring) self.assertTrue('"name":"Peter"' in dump) # hasn't been processed yet self.assertRaises(CrashIDNotFound, crashstorage.get_processed_crash, 'abc123') pro = ('{"name":"Peter","uuid":"abc123", ' '"submitted_timestamp":"%d", ' '"completeddatetime": "%d"}' % (time.time(), time.time())) crashstorage.save_processed(json.loads(pro)) data = crashstorage.get_processed_crash('abc123') self.assertEqual(data['name'], u'Peter') assert crashstorage.hbaseConnection.transport.isOpen() crashstorage.close() transport = crashstorage.hbaseConnection.transport self.assertTrue(not transport.isOpen())
def test_basic_hbase_crashstorage(self): mock_logging = mock.Mock() required_config = HBaseCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'hbase_timeout': 100, 'hbase_host': commonconfig.hbaseHost.default, 'hbase_port': commonconfig.hbasePort.default, }] ) with config_manager.context() as config: crashstorage = HBaseCrashStorage(config) self.assertEqual(list(crashstorage.new_crashes()), []) # data doesn't contain an 'ooid' key #raw = '{"name": "Peter"}' #self.assertRaises( #CrashIDNotFound, #crashstorage.save_raw_crash, #json.loads(raw), #raw #) #raw = '{"name":"Peter","ooid":"abc123"}' #self.assertRaises( #ValueError, # missing the 'submitted_timestamp' key #crashstorage.save_raw_crash, #json.loads(raw), #raw #) raw = ('{"name":"Peter", ' '"submitted_timestamp":"%d"}' % time.time()) crashstorage.save_raw_crash(json.loads(raw), raw, "abc123") assert config.logger.info.called assert config.logger.info.call_count > 1 msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0] # ie logging.info(<template>, <arg>) msg = msg_tmpl % msg_arg self.assertTrue('saved' in msg) self.assertTrue('abc123' in msg) meta = crashstorage.get_raw_crash('abc123') assert isinstance(meta, dict) self.assertEqual(meta['name'], 'Peter') dump = crashstorage.get_raw_dump('abc123') assert isinstance(dump, basestring) self.assertTrue('"name":"Peter"' in dump) # hasn't been processed yet self.assertRaises(CrashIDNotFound, crashstorage.get_processed_crash, 'abc123') pro = ('{"name":"Peter","uuid":"abc123", ' '"submitted_timestamp":"%d", ' '"completeddatetime": "%d"}' % (time.time(), time.time())) crashstorage.save_processed(json.loads(pro)) data = crashstorage.get_processed_crash('abc123') self.assertEqual(data['name'], u'Peter') assert crashstorage.hbaseConnection.transport.isOpen() crashstorage.close() transport = crashstorage.hbaseConnection.transport self.assertTrue(not transport.isOpen())
def test_hbase_crashstorage_puts_and_gets(self): mock_logging = mock.Mock() required_config = HBaseCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'hbase_timeout': 100, 'hbase_host': commonconfig.hbaseHost.default, 'hbase_port': commonconfig.hbasePort.default, 'transaction_executor_class': TransactionExecutorWithLimitedBackoff, 'backoff_delays': [0, 0, 0] }]) with config_manager.context() as config: hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client' with mock.patch(hbaseclient_) as hclient: # test save_raw_crash raw_crash = { "name": "Peter", "email": "*****@*****.**", "url": "http://embarassing.xxx", "submitted_timestamp": "2012-05-04T15:10:00", "user_id": "000-00-0000", } fake_binary_dump = "this a bogus binary dump" expected_raw_crash = raw_crash expected_dump = fake_binary_dump # saves us from loooong lines klass = hclient.HBaseConnectionForCrashReports crashstorage = HBaseCrashStorage(config) crashstorage.save_raw_crash(raw_crash, fake_binary_dump, "abc123") self.assertEqual(klass.put_json_dump.call_count, 1) a = klass.put_json_dump.call_args self.assertEqual(len(a[0]), 4) #self.assertEqual(a[0][1], "abc123") self.assertEqual(a[0][2], expected_raw_crash) self.assertEqual(a[0][3], expected_dump) self.assertEqual(a[1], {'number_of_retries': 0}) # test save_processed processed_crash = { "name": "Peter", "uuid": "abc123", "email": "*****@*****.**", "url": "http://embarassing.xxx", "user_id": "000-00-0000", } expected_processed_crash = { "name": "Peter", "uuid": "abc123", } crashstorage = HBaseCrashStorage(config) crashstorage.save_processed(processed_crash) self.assertEqual(klass.put_processed_json.call_count, 1) a = klass.put_processed_json.call_args self.assertEqual(len(a[0]), 3) self.assertEqual(a[0][1], "abc123") self.assertEqual(a[0][2], expected_processed_crash) self.assertEqual(a[1], {'number_of_retries': 0}) # test get_raw_crash m = mock.Mock(return_value=raw_crash) klass.get_json = m r = crashstorage.get_raw_crash("abc123") self.assertTrue(isinstance(r, DotDict)) a = klass.get_json.call_args self.assertEqual(len(a[0]), 2) self.assertEqual(a[0][1], "abc123") self.assertEqual(klass.get_json.call_count, 1) self.assertEqual(r, expected_raw_crash) # test get_raw_dump m = mock.Mock(return_value=fake_binary_dump) klass.get_dump = m r = crashstorage.get_raw_dump("abc123") a = klass.get_dump.call_args self.assertEqual(len(a[0]), 2) self.assertEqual(a[0][1], "abc123") self.assertEqual(klass.get_dump.call_count, 1) self.assertEqual(r, expected_dump) # test get_processed_crash m = mock.Mock(return_value=expected_processed_crash) klass.get_processed_json = m r = crashstorage.get_processed_crash("abc123") self.assertTrue(isinstance(r, DotDict)) a = klass.get_processed_json.call_args self.assertEqual(len(a[0]), 2) self.assertEqual(a[0][1], "abc123") self.assertEqual(klass.get_processed_json.call_count, 1) self.assertEqual(r, expected_processed_crash)
def test_hbase_crashstorage_puts_and_gets(self): mock_logging = mock.Mock() required_config = HBaseCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'hbase_timeout': 100, 'hbase_host': commonconfig.hbaseHost.default, 'hbase_port': commonconfig.hbasePort.default, 'transaction_executor_class': TransactionExecutorWithLimitedBackoff, 'backoff_delays': [0, 0, 0] }] ) with config_manager.context() as config: hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client' with mock.patch(hbaseclient_) as hclient: # test save_raw_crash raw_crash = { "name": "Peter", "email": "*****@*****.**", "url": "http://embarassing.xxx", "submitted_timestamp": "2012-05-04T15:10:00", "user_id": "000-00-0000", } fake_binary_dump = "this a bogus binary dump" expected_raw_crash = raw_crash expected_dump = fake_binary_dump # saves us from loooong lines klass = hclient.HBaseConnectionForCrashReports crashstorage = HBaseCrashStorage(config) crashstorage.save_raw_crash(raw_crash, fake_binary_dump, "abc123") self.assertEqual( klass.put_json_dump.call_count, 1 ) a = klass.put_json_dump.call_args self.assertEqual(len(a[0]), 4) #self.assertEqual(a[0][1], "abc123") self.assertEqual(a[0][2], expected_raw_crash) self.assertEqual(a[0][3], expected_dump) self.assertEqual(a[1], {'number_of_retries': 0}) # test save_processed processed_crash = { "name": "Peter", "uuid": "abc123", "email": "*****@*****.**", "url": "http://embarassing.xxx", "user_id": "000-00-0000", } expected_processed_crash = { "name": "Peter", "uuid": "abc123", } crashstorage = HBaseCrashStorage(config) crashstorage.save_processed(processed_crash) self.assertEqual(klass.put_processed_json.call_count, 1) a = klass.put_processed_json.call_args self.assertEqual(len(a[0]), 3) self.assertEqual(a[0][1], "abc123") self.assertEqual(a[0][2], expected_processed_crash) self.assertEqual(a[1], {'number_of_retries': 0}) # test get_raw_crash m = mock.Mock(return_value=raw_crash) klass.get_json = m r = crashstorage.get_raw_crash("abc123") self.assertTrue(isinstance(r, DotDict)) a = klass.get_json.call_args self.assertEqual(len(a[0]), 2) self.assertEqual(a[0][1], "abc123") self.assertEqual(klass.get_json.call_count, 1) self.assertEqual(r, expected_raw_crash) # test get_raw_dump m = mock.Mock(return_value=fake_binary_dump) klass.get_dump = m r = crashstorage.get_raw_dump("abc123") a = klass.get_dump.call_args self.assertEqual(len(a[0]), 2) self.assertEqual(a[0][1], "abc123") self.assertEqual(klass.get_dump.call_count, 1) self.assertEqual(r, expected_dump) # test get_processed_crash m = mock.Mock(return_value=expected_processed_crash) klass.get_processed_json = m r = crashstorage.get_processed_crash("abc123") self.assertTrue(isinstance(r, DotDict)) a = klass.get_processed_json.call_args self.assertEqual(len(a[0]), 2) self.assertEqual(a[0][1], "abc123") self.assertEqual(klass.get_processed_json.call_count, 1) self.assertEqual(r, expected_processed_crash)