def test_basic_hbase_crashstorage(self): mock_logging = mock.Mock() required_config = HBaseCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'hbase_timeout': 100, 'hbase_host': commonconfig.hbaseHost.default, 'hbase_port': commonconfig.hbasePort.default, }], argv_source=[]) with config_manager.context() as config: crashstorage = HBaseCrashStorage(config) eq_(list(crashstorage.new_crashes()), []) crash_id = '86b58ff2-9708-487d-bfc4-9dac32121214' raw = ('{"name":"Peter", ' '"submitted_timestamp":"%d"}' % time.time()) fake_raw_dump_1 = 'peter is a swede' fake_raw_dump_2 = 'lars is a norseman' fake_raw_dump_3 = 'adrian is a frenchman' fake_dumps = { 'upload_file_minidump': fake_raw_dump_1, 'lars': fake_raw_dump_2, 'adrian': fake_raw_dump_3 } crashstorage.save_raw_crash(json.loads(raw), fake_dumps, crash_id) assert config.logger.info.called assert config.logger.info.call_count > 1 msg_tmpl, msg_arg = config.logger.info.call_args_list[1][0] # ie logging.info(<template>, <arg>) msg = msg_tmpl % msg_arg ok_('saved' in msg) ok_(crash_id in msg) raw_crash = crashstorage.get_raw_crash(crash_id) assert isinstance(raw_crash, dict) eq_(raw_crash['name'], 'Peter') dump = crashstorage.get_raw_dump(crash_id) assert isinstance(dump, basestring) ok_('peter is a swede' in dump) dumps = crashstorage.get_raw_dumps(crash_id) assert isinstance(dumps, dict) ok_('upload_file_minidump' in dumps) ok_('lars' in dumps) ok_('adrian' in dumps) eq_(dumps['upload_file_minidump'], fake_dumps['upload_file_minidump']) eq_(dumps['lars'], fake_dumps['lars']) eq_(dumps['adrian'], fake_dumps['adrian']) # hasn't been processed yet assert_raises(CrashIDNotFound, crashstorage.get_processed, crash_id) pro = ('{"name":"Peter",' '"uuid":"86b58ff2-9708-487d-bfc4-9dac32121214", ' '"submitted_timestamp":"%d", ' '"completeddatetime": "%d"}' % (time.time(), time.time())) crashstorage.save_processed(json.loads(pro)) data = crashstorage.get_processed(crash_id) eq_(data['name'], u'Peter') hb_connection = crashstorage.hbaseConnectionPool.connection() ok_(hb_connection.transport.isOpen()) crashstorage.close() ok_(not hb_connection.transport.isOpen())
def test_select_host_mode_success(self): a_date = datetime(year=2012, month=5, day=4, hour=15, minute=10, tzinfo=UTC) frequency = timedelta(0, 300) threshold = a_date - frequency mock_logging = mock.Mock() mock_connection = mock.MagicMock() mock_postgres = mock.MagicMock(return_value=mock_connection) mock_connection.return_value = mock_connection mock_connection.__enter__.return_value = mock_connection mock_cursor = mock.Mock() mock_connection.cursor.return_value = mock_cursor mock_execute = mock.Mock() mock_cursor.execute = mock_execute fetchall_returns = sequencer(((threshold, ), ), ((17, ), )) mock_fetchall = mock.Mock(side_effect=fetchall_returns) mock_cursor.fetchall = mock_fetchall mock_fetchone = mock.Mock() mock_cursor.fetchone = mock_fetchone required_config = ProcessorAppRegistrationClient.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database': mock_postgres, }]) with config_manager.context() as config: mock_os_uname_str = 'os.uname' with mock.patch(mock_os_uname_str) as mock_uname: mock_uname.return_value = (0, 'wilma') registrar = ProcessorAppRegistrationClient(config) name = registrar.processor_name self.assertEqual(mock_execute.call_count, 4) expected_execute_args = ( (("select now() - interval %s", (frequency, )), ), ((("select id from processors" " where lastseendatetime < %s" " and name like %s limit 1"), (threshold, 'wilma%')), ), ((("update processors set name = %s, " "startdatetime = now(), lastseendatetime = now()" " where id = %s"), (name, 17)), ), ((("update jobs set" " starteddatetime = NULL," " completeddatetime = NULL," " success = NULL " "where" " owner = %s"), (17, )), ), ) actual_execute_args = mock_execute.call_args_list for expected, actual in zip(expected_execute_args, actual_execute_args): self.assertEqual(expected, actual)
def _do_run(klass, config_path=None, values_source_list=None): # while this method is defined here, only derived classes are allowed # to call it. if klass is SocorroApp: raise NotImplementedError( "The SocorroApp class has no useable 'main' method" ) if config_path is None: config_path = os.environ.get( 'DEFAULT_SOCORRO_CONFIG_PATH', './config' ) if values_source_list is None: values_source_list = [ # pull in the application defaults from the 'application' # configman option, once it has been defined application_defaults_proxy, # pull in any configuration file ConfigFileFutureProxy, # get values from the environment environment, # use the command line to get the final overriding values command_line ] elif application_defaults_proxy not in values_source_list: values_source_list = ( [application_defaults_proxy] + values_source_list ) config_definition = klass.get_required_config() if 'application' not in config_definition: # the application option has not been defined. This means that # the we're likely trying to run one of the applications directly # rather than through the SocorroWelocomApp. Add the 'application' # option initialized with the target application as the default. application_config = Namespace() application_config.add_option( 'application', doc=( 'the fully qualified classname of the app to run' ), default=klass_to_pypath(klass), # the following setting means this option will NOT be # commented out when configman generates a config file likely_to_be_changed=True, from_string_converter=( application_defaults_proxy.str_to_application_class ), ) config_definition = application_config config_manager = ConfigurationManager( config_definition, app_name=klass.app_name, app_version=klass.app_version, app_description=klass.app_description, values_source_list=values_source_list, options_banned_from_help=[], config_pathname=config_path ) def fix_exit_code(code): # some apps don't return a code so you might get None # which isn't good enough to send to sys.exit() if code is None: return 0 return code with config_manager.context() as config: config.executor_identity = ( lambda: threading.currentThread().getName() ) try: config_manager.log_config(config.logger) respond_to_SIGHUP_with_logging = functools.partial( respond_to_SIGHUP, logger=config.logger ) # install the signal handler with logging signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging) except KeyError: # config apparently doesn't have 'logger' # install the signal handler without logging signal.signal(signal.SIGHUP, respond_to_SIGHUP) # we finally know what app to actually run, instantiate it app_to_run = klass(config) app_to_run.config_manager = config_manager # whew, finally run the app that we wanted return_code = fix_exit_code(app_to_run.main()) return return_code
def test_hbase_crashstorage_puts_and_gets(self): mock_logging = mock.Mock() required_config = HBaseCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'hbase_timeout': 100, 'hbase_host': commonconfig.hbaseHost.default, 'hbase_port': commonconfig.hbasePort.default, 'transaction_executor_class': TransactionExecutorWithLimitedBackoff, 'backoff_delays': [0, 0, 0], }], argv_source=[]) with config_manager.context() as config: config.executor_identity = lambda: 'dwight' # bogus thread id hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client' with mock.patch(hbaseclient_) as hclient: # test save_raw_crash raw_crash = { "name": "Peter", "email": "*****@*****.**", "url": "http://embarassing.xxx", "submitted_timestamp": "2012-05-04T15:10:00", "user_id": "000-00-0000", } fake_binary_dump = "this a bogus binary dump" expected_raw_crash = raw_crash expected_dump = fake_binary_dump expected_dump_2 = fake_binary_dump + " number 2" # saves us from loooong lines klass = hclient.HBaseConnectionForCrashReports crashstorage = HBaseCrashStorage(config) crashstorage.save_raw_crash(raw_crash, fake_binary_dump, "abc123") eq_(klass.put_json_dump.call_count, 1) a = klass.put_json_dump.call_args eq_(len(a[0]), 4) #eq_(a[0][1], "abc123") eq_(a[0][2], expected_raw_crash) eq_(a[0][3], expected_dump) eq_(a[1], {'number_of_retries': 0}) # test save_processed processed_crash = { "name": "Peter", "uuid": "abc123", "email": "*****@*****.**", "url": "http://embarassing.xxx", "user_id": "000-00-0000", } expected_processed_crash = { "name": "Peter", "uuid": "abc123", } expected_unredacted_processed_crash = { "name": "Peter", "uuid": "abc123", "email": "*****@*****.**", "url": "http://embarassing.xxx", "user_id": "000-00-0000", } crashstorage = HBaseCrashStorage(config) crashstorage.save_processed(processed_crash) eq_(klass.put_processed_json.call_count, 1) a = klass.put_processed_json.call_args eq_(len(a[0]), 3) eq_(a[0][1], "abc123") eq_(a[0][2], expected_unredacted_processed_crash) eq_(a[1], {'number_of_retries': 0}) # test get_raw_crash m = mock.Mock(return_value=raw_crash) klass.get_json = m r = crashstorage.get_raw_crash("abc123") ok_(isinstance(r, DotDict)) a = klass.get_json.call_args eq_(len(a[0]), 2) eq_(a[0][1], "abc123") eq_(klass.get_json.call_count, 1) eq_(r, expected_raw_crash) # test get_raw_dump m = mock.Mock(return_value=fake_binary_dump) klass.get_dump = m r = crashstorage.get_raw_dump("abc123") a = klass.get_dump.call_args eq_(len(a[0]), 3) eq_(a[0][1], "abc123") eq_(klass.get_dump.call_count, 1) eq_(r, expected_dump) # test get_raw_dumps m = mock.Mock( return_value={'upload_file_minidump': fake_binary_dump}) klass.get_dumps = m r = crashstorage.get_raw_dumps("abc123") a = klass.get_dumps.call_args eq_(len(a[0]), 2) eq_(a[0][1], "abc123") eq_(klass.get_dumps.call_count, 1) eq_(r, {'upload_file_minidump': expected_dump}) # test get_raw_dumps 2 m = mock.Mock( return_value={ 'upload_file_minidump': fake_binary_dump, 'aux_1': expected_dump_2 }) klass.get_dumps = m r = crashstorage.get_raw_dumps("abc123") a = klass.get_dumps.call_args eq_(len(a[0]), 2) eq_(a[0][1], "abc123") eq_(klass.get_dumps.call_count, 1) eq_( r, { 'upload_file_minidump': fake_binary_dump, 'aux_1': expected_dump_2 }) # test get_processed m = mock.Mock(return_value=expected_processed_crash) klass.get_processed_json = m r = crashstorage.get_processed("abc123") ok_(isinstance(r, DotDict)) a = klass.get_processed_json.call_args eq_(len(a[0]), 2) eq_(a[0][1], "abc123") eq_(klass.get_processed_json.call_count, 1) eq_(r, expected_processed_crash)
def test_migration_crash_storage(self): n = Namespace() n.add_option( 'storage', default=MigrationCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'primary.storage_class': ( 'socorro.unittest.external.test_crashstorage_base.A' ), 'fallback.storage_class': ( 'socorro.unittest.external.test_crashstorage_base.B' ), 'date_threshold': '150315' } cm = ConfigurationManager( n, values_source_list=[value], argv_source=[] ) with cm.context() as config: raw_crash = {'ooid': ''} before_crash_id = '1498dee9-9a45-45cc-8ec8-71bb62150314' after_crash_id = '1498dee9-9a45-45cc-8ec8-71bb62150315' dump = '12345' processed_crash = {'ooid': '', 'product': 17} migration_store = config.storage(config) # save_raw tests # save to primary migration_store.primary_store.save_raw_crash = Mock() migration_store.fallback_store.save_raw_crash = Mock() migration_store.save_raw_crash(raw_crash, dump, after_crash_id) migration_store.primary_store.save_raw_crash.assert_called_with( raw_crash, dump, after_crash_id ) eq_(migration_store.fallback_store.save_raw_crash.call_count, 0) # save to fallback migration_store.primary_store.save_raw_crash = Mock() migration_store.fallback_store.save_raw_crash = Mock() migration_store.save_raw_crash(raw_crash, dump, before_crash_id) eq_(migration_store.primary_store.save_raw_crash.call_count, 0) migration_store.fallback_store.save_raw_crash.assert_called_with( raw_crash, dump, before_crash_id ) # save_processed tests # save to primary processed_crash['crash_id'] = after_crash_id migration_store.primary_store.save_processed = Mock() migration_store.fallback_store.save_processed = Mock() migration_store.save_processed(processed_crash) migration_store.primary_store.save_processed.assert_called_with( processed_crash ) eq_(migration_store.fallback_store.save_processed.call_count, 0) # save to fallback processed_crash['crash_id'] = before_crash_id migration_store.primary_store.save_processed = Mock() migration_store.fallback_store.save_processed = Mock() migration_store.save_processed(processed_crash) eq_(migration_store.primary_store.save_processed.call_count, 0) migration_store.fallback_store.save_processed.assert_called_with( processed_crash ) # close tests migration_store.primary_store.close = Mock() migration_store.fallback_store.close = Mock() migration_store.close() migration_store.primary_store.close.assert_called_with() migration_store.fallback_store.close.assert_called_with() migration_store.primary_store.close = Mock() migration_store.fallback_store.close = Mock() migration_store.fallback_store.close.side_effect = ( NotImplementedError() ) migration_store.close() migration_store.primary_store.close.assert_called_with() migration_store.fallback_store.close.assert_called_with() migration_store.primary_store.close = Mock() migration_store.primary_store.close.side_effect = Exception('!') migration_store.close() migration_store.primary_store.close.assert_called_with() migration_store.fallback_store.close.assert_called_with() migration_store.fallback_store.close = Mock() migration_store.fallback_store.close.side_effect = Exception('!') assert_raises(PolyStorageError, migration_store.close) migration_store.primary_store.close.assert_called_with() migration_store.fallback_store.close.assert_called_with()
def test_processed_crash_storage(self): n = Namespace() n.add_option( 'storage', default=PrimaryDeferredProcessedStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'primary.storage_class': ( 'socorro.unittest.external.test_crashstorage_base.A' ), 'deferred.storage_class': ( 'socorro.unittest.external.test_crashstorage_base.B' ), 'processed.storage_class': ( 'socorro.unittest.external.test_crashstorage_base.B' ), 'deferral_criteria': lambda x: x.get('foo') == 'foo' } cm = ConfigurationManager( n, values_source_list=[value], argv_source=[] ) with cm.context() as config: eq_(config.primary.storage_class.foo, 'a') eq_(config.deferred.storage_class.foo, 'b') eq_(config.processed.storage_class.foo, 'b') raw_crash = {'ooid': ''} crash_id = '1498dee9-9a45-45cc-8ec8-71bb62121203' dump = '12345' deferred_crash = {'ooid': '', 'foo': 'foo'} processed_crash = {'ooid': '', 'product': 17} pd_store = config.storage(config) # save_raw tests pd_store.primary_store.save_raw_crash = Mock() pd_store.deferred_store.save_raw_crash = Mock() pd_store.processed_store.save_raw_crash = Mock() pd_store.save_raw_crash(raw_crash, dump, crash_id) pd_store.primary_store.save_raw_crash.assert_called_with( raw_crash, dump, crash_id ) eq_(pd_store.deferred_store.save_raw_crash.call_count, 0) pd_store.save_raw_crash(deferred_crash, dump, crash_id) pd_store.deferred_store.save_raw_crash.assert_called_with( deferred_crash, dump, crash_id ) # save_processed tests pd_store.primary_store.save_processed = Mock() pd_store.deferred_store.save_processed = Mock() pd_store.processed_store.save_processed = Mock() pd_store.save_processed(processed_crash) pd_store.processed_store.save_processed.assert_called_with( processed_crash ) eq_(pd_store.primary_store.save_processed.call_count, 0) pd_store.save_processed(deferred_crash) pd_store.processed_store.save_processed.assert_called_with( deferred_crash ) # close tests pd_store.primary_store.close = Mock() pd_store.deferred_store.close = Mock() pd_store.close() pd_store.primary_store.close.assert_called_with() pd_store.deferred_store.close.assert_called_with() pd_store.primary_store.close = Mock() pd_store.deferred_store.close = Mock() pd_store.deferred_store.close.side_effect = NotImplementedError() pd_store.close() pd_store.primary_store.close.assert_called_with() pd_store.deferred_store.close.assert_called_with() pd_store.primary_store.close = Mock() pd_store.primary_store.close.side_effect = Exception('!') pd_store.close() pd_store.primary_store.close.assert_called_with() pd_store.deferred_store.close.assert_called_with() pd_store.deferred_store.close = Mock() pd_store.deferred_store.close.side_effect = Exception('!') assert_raises(PolyStorageError, pd_store.close) pd_store.primary_store.close.assert_called_with() pd_store.deferred_store.close.assert_called_with()
def test_poly_crash_storage(self): n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_namespaces': 'A,A2,B', 'A.crashstorage_class': 'socorro.unittest.external.test_crashstorage_base.A', 'A2.crashstorage_class': 'socorro.unittest.external.test_crashstorage_base.A', 'B.crashstorage_class': 'socorro.unittest.external.test_crashstorage_base.B', 'A2.y': 37 } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: assert config.A.crashstorage_class.foo == 'a' assert config.A2.crashstorage_class.foo == 'a' assert config.A2.y == 37 assert config.B.crashstorage_class.foo == 'b' poly_store = config.storage(config) assert len(poly_store.storage_namespaces) == 3 assert poly_store.storage_namespaces[0] == 'A' assert poly_store.storage_namespaces[1] == 'A2' assert poly_store.storage_namespaces[2] == 'B' assert len(poly_store.stores) == 3 assert poly_store.stores.A.foo == 'a' assert poly_store.stores.A2.foo == 'a' assert poly_store.stores.B.foo == 'b' raw_crash = {'ooid': ''} dump = '12345' processed_crash = {'ooid': '', 'product': 17} for v in poly_store.stores.itervalues(): v.save_raw_crash = Mock() v.save_processed = Mock() v.close = Mock() poly_store.save_raw_crash(raw_crash, dump, '') for v in poly_store.stores.itervalues(): v.save_raw_crash.assert_called_once_with(raw_crash, dump, '') poly_store.save_processed(processed_crash) for v in poly_store.stores.itervalues(): v.save_processed.assert_called_once_with(processed_crash) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, 'n') for v in poly_store.stores.itervalues(): v.save_raw_crash.assert_called_with(raw_crash, dump, 'n') v.save_processed.assert_called_with(processed_crash) raw_crash = {'ooid': 'oaeu'} dump = '5432' processed_crash = {'ooid': 'aoeu', 'product': 33} expected = Exception('this is messed up') poly_store.stores['A2'].save_raw_crash = Mock() poly_store.stores['A2'].save_raw_crash.side_effect = expected poly_store.stores['B'].save_processed = Mock() poly_store.stores['B'].save_processed.side_effect = expected with pytest.raises(PolyStorageError): poly_store.save_raw_crash(raw_crash, dump, '') for v in poly_store.stores.itervalues(): v.save_raw_crash.assert_called_with(raw_crash, dump, '') with pytest.raises(PolyStorageError): poly_store.save_processed(processed_crash) for v in poly_store.stores.itervalues(): v.save_processed.assert_called_with(processed_crash) with pytest.raises(PolyStorageError): poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, 'n') for v in poly_store.stores.itervalues(): v.save_raw_crash.assert_called_with(raw_crash, dump, 'n') v.save_processed.assert_called_with(processed_crash) poly_store.stores['B'].close.side_effect = Exception with pytest.raises(PolyStorageError): poly_store.close() for v in poly_store.stores.itervalues(): v.close.assert_called_with()
def test_poly_crash_storage(self): n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_classes': ( 'socorro.unittest.external.test_crashstorage_base.A,' 'socorro.unittest.external.test_crashstorage_base.A,' 'socorro.unittest.external.test_crashstorage_base.B' ), 'storage1.y': 37, } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: eq_(config.storage0.crashstorage_class.foo, 'a') eq_(config.storage1.crashstorage_class.foo, 'a') eq_(config.storage1.y, 37) eq_(config.storage2.crashstorage_class.foo, 'b') poly_store = config.storage(config) l = len(poly_store.storage_namespaces) eq_( l, 3, 'expected poly_store to have lenth of 3, ' 'but %d was found instead' % l ) eq_(poly_store.storage_namespaces[0], 'storage0') eq_(poly_store.storage_namespaces[1], 'storage1') eq_(poly_store.storage_namespaces[2], 'storage2') l = len(poly_store.stores) eq_( l, 3, 'expected poly_store.store to have lenth of 3, ' 'but %d was found instead' % l ) eq_(poly_store.stores.storage0.foo, 'a') eq_(poly_store.stores.storage1.foo, 'a') eq_(poly_store.stores.storage2.foo, 'b') raw_crash = {'ooid': ''} dump = '12345' processed_crash = {'ooid': '', 'product': 17} for v in poly_store.stores.itervalues(): v.save_raw_crash = Mock() v.save_processed = Mock() v.close = Mock() poly_store.save_raw_crash(raw_crash, dump, '') for v in poly_store.stores.itervalues(): v.save_raw_crash.assert_called_once_with(raw_crash, dump, '') poly_store.save_processed(processed_crash) for v in poly_store.stores.itervalues(): v.save_processed.assert_called_once_with(processed_crash) poly_store.save_raw_and_processed( raw_crash, dump, processed_crash, 'n' ) for v in poly_store.stores.itervalues(): v.save_raw_crash.assert_called_with(raw_crash, dump, 'n') v.save_processed.assert_called_with(processed_crash) raw_crash = {'ooid': 'oaeu'} dump = '5432' processed_crash = {'ooid': 'aoeu', 'product': 33} poly_store.stores['storage1'].save_raw_crash = Mock() poly_store.stores['storage1'].save_raw_crash.side_effect = \ Exception('this is messed up') poly_store.stores['storage2'].save_processed = Mock() poly_store.stores['storage2'].save_processed.side_effect = \ Exception('this is messed up') assert_raises( PolyStorageError, poly_store.save_raw_crash, raw_crash, dump, '' ) for v in poly_store.stores.itervalues(): v.save_raw_crash.assert_called_with(raw_crash, dump, '') assert_raises( PolyStorageError, poly_store.save_processed, processed_crash ) for v in poly_store.stores.itervalues(): v.save_processed.assert_called_with(processed_crash) assert_raises( PolyStorageError, poly_store.save_raw_and_processed, raw_crash, dump, processed_crash, 'n' ) for v in poly_store.stores.itervalues(): v.save_raw_crash.assert_called_with(raw_crash, dump, 'n') v.save_processed.assert_called_with(processed_crash) poly_store.stores['storage2'].close.side_effect = Exception assert_raises(PolyStorageError, poly_store.close) for v in poly_store.stores.itervalues(): v.close.assert_called_with()
def _do_run(klass, config_path=None, values_source_list=None): # while this method is defined here, only derived classes are allowed # to call it. if klass is SocorroApp: raise NotImplementedError( "The SocorroApp class has no useable 'main' method") if config_path is None: config_path = os.environ.get('DEFAULT_SOCORRO_CONFIG_PATH', './config') if values_source_list is None: values_source_list = [ # pull in any configuration file ConfigFileFutureProxy, # get values from the environment environment, # use the command line to get the final overriding values command_line ] # Pull base set of defaults from the config module if it is specified if klass.config_defaults is not None: values_source_list.insert(0, klass.config_defaults) config_definition = klass.get_required_config() if 'application' not in config_definition: # FIXME(mkelly): We used to have a SocorroWelcomeApp that defined an # "application" option. We no longer have that. This section should # get reworked possibly as part of getting rid of application # defaults. application_config = Namespace() application_config.add_option( 'application', doc=('the fully qualified classname of the app to run'), default=klass_to_pypath(klass), # the following setting means this option will NOT be # commented out when configman generates a config file likely_to_be_changed=True, from_string_converter=str_to_python_object, ) config_definition = application_config config_manager = ConfigurationManager( config_definition, app_name=klass.app_name, app_version=klass.app_version, app_description=klass.app_description, values_source_list=values_source_list, options_banned_from_help=[], config_pathname=config_path) def fix_exit_code(code): # some apps don't return a code so you might get None # which isn't good enough to send to sys.exit() if code is None: return 0 return code with config_manager.context() as config: config.executor_identity = ( lambda: threading.currentThread().getName()) # Log revision information revision_data = get_revision_data() revision_items = sorted(revision_data.items()) config.logger.info( 'version.json: {%s}', ', '.join( ['%r: %r' % (key, val) for key, val in revision_items])) try: config_manager.log_config(config.logger) respond_to_SIGHUP_with_logging = functools.partial( respond_to_SIGHUP, logger=config.logger) # install the signal handler with logging signal.signal(signal.SIGHUP, respond_to_SIGHUP_with_logging) except KeyError: # config apparently doesn't have 'logger' # install the signal handler without logging signal.signal(signal.SIGHUP, respond_to_SIGHUP) # we finally know what app to actually run, instantiate it app_to_run = klass(config) app_to_run.config_manager = config_manager # whew, finally run the app that we wanted return_code = fix_exit_code(app_to_run.main()) return return_code
# create an iterable collection of definition sources # internally, this list will be appended to, so a tuple won't do. # the definitions are in the json file listed below. definition_source = 'demo1j.json' # set up the manager with the option definitions along with the 'app_name' and # 'app_description'. They will both be used later to create the output of the # automatically created '--help' command line switch. # By default, when assigning values to the options loaded from the json file, # the ConfigurationManager will take, in turn: the default from the definition, # any values loaded from a config file specified by the --admin.conf command # line switch, values from the os environment and finally overrides from the # commandline. c = ConfigurationManager(definition_source, app_name='demo1j', app_description=__doc__) # fetch the DOM-like instance that gives access to the configuration info config = c.get_config() # use the config if config.action == 'echo': echo(config.text) elif config.action == 'backwards': backwards(config.text) elif config.action == 'upper': upper(config.text) else: print >> sys.stderr, config.action, "is not a valid action"
def test_basic_postgres_save_raw_crash(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[]) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) database = crashstorage.database.return_value = mock.MagicMock() ok_(isinstance(database, mock.Mock)) ok_('submitted_timestamp' in a_raw_crash) m = mock.MagicMock() m.__enter__.return_value = m database = crashstorage.database.return_value = m crashstorage.save_raw_crash( a_raw_crash, '', "936ce666-ff3b-4c7a-9674-367fe2120408") eq_(m.cursor.call_count, 1) eq_( m.cursor.return_value.__enter__.return_value.execute. call_count, 1) expected_execute_args = (((""" WITH update_raw_crash AS ( UPDATE raw_crashes_20120402 SET raw_crash = %(raw_crash)s, date_processed = %(date_processed)s WHERE uuid = %(crash_id)s RETURNING 1 ), insert_raw_crash AS ( INSERT into raw_crashes_20120402 (uuid, raw_crash, date_processed) ( SELECT %(crash_id)s as uuid, %(raw_crash)s as raw_crash, %(date_processed)s as date_processed WHERE NOT EXISTS ( SELECT uuid from raw_crashes_20120402 WHERE uuid = %(crash_id)s LIMIT 1 ) ) RETURNING 2 ) SELECT * from update_raw_crash UNION ALL SELECT * from insert_raw_crash """, { 'crash_id': '936ce666-ff3b-4c7a-9674-367fe2120408', 'raw_crash': '{"submitted_timestamp": "2012-04-08 10:52:42.0", "Version": "6.02E23", "ProductName": "Fennicky"}', 'date_processed': "2012-04-08 10:52:42.0" }), ), ) actual_execute_args = m.cursor().execute.call_args_list for expected, actual in zip(expected_execute_args, actual_execute_args): expeceted_sql, expected_params = expected[0] expeceted_sql = remove_whitespace(expeceted_sql) actual_sql, actual_params = actual[0] actual_sql = remove_whitespace(actual_sql) eq_(expeceted_sql, actual_sql) eq_(expected_params, actual_params)
def test_success_after_limited_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) esindex_results = [ pyelasticsearch.exceptions.Timeout, pyelasticsearch.exceptions.Timeout ] def esindex_fn(*args, **kwargs): try: r = esindex_results.pop(0) raise r except IndexError: return mock_es.index mock_es.index.side_effect = esindex_fn crash_id = a_processed_crash['uuid'] es_storage.save_raw_and_processed( a_raw_crash, None, a_processed_crash.copy(), crash_id, ) expected_crash = { 'crash_id': crash_id, 'processed_crash': a_processed_crash.copy(), 'raw_crash': a_raw_crash } expected_request_args = ('socorro201214', 'crash_reports', expected_crash) expected_request_kwargs = { 'replication': 'async', 'id': crash_id, } mock_es.index.assert_called_with(*expected_request_args, **expected_request_kwargs)
def test_basic_postgres_save_processed_succeed_after_failures(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres, 'transaction_executor_class': TransactionExecutorWithLimitedBackoff, 'backoff_delays': [0, 0, 0], }]) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) crashstorage.database.operational_exceptions = (OperationalError, ) database = crashstorage.database.return_value = mock.MagicMock() self.assertTrue(isinstance(database, mock.Mock)) fetch_all_returns = [ ((666, ), ), None, ((23, ), ), ] def fetch_all_func(*args): result = fetch_all_returns.pop(0) return result fetch_mock = mock.Mock() fetch_mock.fetchall.side_effect = fetch_all_func connection_trouble = [ OperationalError('bad'), OperationalError('worse'), ] def broken_connection(*args): try: result = connection_trouble.pop(0) raise result except IndexError: return fetch_mock m = mock.MagicMock() m.__enter__.return_value = m database = crashstorage.database.return_value = m m.cursor.side_effect = broken_connection crashstorage.save_processed(a_processed_crash) self.assertEqual(m.cursor.call_count, 9) self.assertEqual(m.cursor().fetchall.call_count, 3) self.assertEqual(m.cursor().execute.call_count, 7) expected_execute_args = ( (('savepoint MainThread', None), ), (('insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id', [ None, '0x1c', '...', '20120309050057', '2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884', 'None | 0', 'arm', '2012-04-08 10:56:41.558922', None, None, '*****@*****.**', 'high', '[blank]', None, 22385, None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ', 'SignatureTool: signature truncated due to length', 'plugin', 'FennecAndroid', 'SIGSEGV', 'default', 'libxul.so@0x117441c', '2012-04-08 10:56:50.440752', True, [], False, 170, None, None, 'http://embarasing.p**n.com', '936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1' ]), ), (('release savepoint MainThread', None), ), (('select id from plugins where filename = %s and name = %s', ('dwight.txt', 'wilma')), ), (('insert into plugins (filename, name) values (%s, %s) returning id', ('dwight.txt', 'wilma')), ), (('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)', (666, 23, '2012-04-08 10:56:41.558922', '69')), ), (('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)', (666, '2012-04-08 10:56:41.558922', 0, '{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')), ), ) actual_execute_args = m.cursor().execute.call_args_list for expected, actual in zip(expected_execute_args, actual_execute_args): self.assertEqual(expected, actual)
def test_select_host_mode_not_dead_fail(self): a_date = datetime(year=2012, month=5, day=4, hour=15, minute=10, tzinfo=UTC) frequency = timedelta(0, 300) threshold = a_date - frequency mock_logging = mock.Mock() mock_connection = mock.MagicMock() mock_postgres = mock.MagicMock(return_value=mock_connection) mock_connection.return_value = mock_connection mock_connection.__enter__.return_value = mock_connection mock_cursor = mock.Mock() mock_connection.cursor.return_value = mock_cursor mock_execute = mock.Mock() mock_cursor.execute = mock_execute fetchall_returns = sequencer( ((threshold, ), ), SQLDidNotReturnSingleValue(), ((92, ), ), ) mock_fetchall = mock.Mock(side_effect=fetchall_returns) mock_cursor.fetchall = mock_fetchall mock_fetchone = mock.Mock() mock_cursor.fetchone = mock_fetchone required_config = ProcessorAppRegistrationClient.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database': mock_postgres, }]) with config_manager.context() as config: mock_os_uname_str = 'os.uname' with mock.patch(mock_os_uname_str) as mock_uname: mock_uname.return_value = (0, 'wilma') self.assertRaises(RegistrationError, ProcessorAppRegistrationClient, config) self.assertEqual(mock_execute.call_count, 3) expected_execute_args = ( (("select now() - interval %s", (frequency, )), ), ((("select id from processors" " where lastseendatetime < %s" " and name like %s limit 1"), (threshold, 'wilma%')), ), ((("select id from processors" " where name like 'wilma%'"), None), ), ) actual_execute_args = mock_execute.call_args_list for expected, actual in zip(expected_execute_args, actual_execute_args): self.assertEqual(expected, actual)
def test_programming_error_with_postgres_with_backoff_with_rollback(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutorWithInfiniteBackoff, doc='a class that will execute transactions') required_config.add_option('database_class', default=MockConnectionContext, from_string_converter=class_converter) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'backoff_delays': [2, 4, 6, 10, 15] }], argv_source=[]) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class( config, mocked_context) _function_calls = [] # some mutable _sleep_count = [] def mock_function_struggling(connection): assert isinstance(connection, MockConnection) connection.transaction_status = psycopg2.extensions.TRANSACTION_STATUS_INTRANS _function_calls.append(connection) # the default sleep times are going to be, # 2, 4, 6, 10, 15 # so after 2 + 4 + 6 + 10 + 15 seconds # all will be exhausted if sum(_sleep_count) < sum([2, 4, 6, 10, 15]): raise psycopg2.ProgrammingError( 'SSL SYSCALL error: EOF detected') def mock_sleep(n): _sleep_count.append(n) # monkey patch the sleep function from inside transaction_executor _orig_sleep = socorro.database.transaction_executor.time.sleep socorro.database.transaction_executor.time.sleep = mock_sleep try: executor(mock_function_struggling) assert _function_calls assert commit_count == 1 assert rollback_count == 5 assert mock_logging.criticals assert len(mock_logging.criticals) == 5 assert len(_sleep_count) > 10 finally: socorro.database.transaction_executor.time.sleep = _orig_sleep # this time, simulate an actual code bug where a callable function # raises a ProgrammingError() exception by, for example, a syntax error with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class( config, mocked_context) def mock_function_developer_mistake(connection): assert isinstance(connection, MockConnection) connection.transaction_status = psycopg2.extensions.TRANSACTION_STATUS_INTRANS raise psycopg2.ProgrammingError("syntax error") with pytest.raises(psycopg2.ProgrammingError): executor(mock_function_developer_mistake)
def get_config_manager(): pubsub_config = PubSubCrashQueue.get_required_config() return ConfigurationManager([pubsub_config], app_name='test-pubsub', app_description='', argv_source=[])
def test_write_with_imported_module(self): import os from configman.tests.values_for_module_tests_1 import Alpha, foo, a definitions = { 'os_module': os, 'a': 17, 'imported_class': Alpha, 'imported_function': foo, 'xxx': { 'yyy': a, } } cm = ConfigurationManager( definitions, values_source_list=[], ) s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) generated_python_module_text = s.getvalue() expected = """# generated Python configman file from configman.dotdict import DotDict from configman.tests.values_for_module_tests_1 import ( Alpha, foo, ) import os # the following symbols will be ignored by configman when # this module is used as a value source. This will # suppress the mismatch warning since these symbols are # values for options, not option names themselves. ignore_symbol_list = [ "Alpha", "DotDict", "foo", "os", ] # a a = 17 # imported_class imported_class = Alpha # imported_function imported_function = foo # os_module os_module = os # Namespace: xxx xxx = DotDict() # yyy xxx.yyy = 18 """ if six.PY2: expected = six.binary_type(expected) self.assertEqual(generated_python_module_text, expected)
def test_fallback_crash_storage(self): n = Namespace() n.add_option( 'storage', default=FallbackCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'primary.storage_class': ( 'socorro.unittest.external.test_crashstorage_base.A' ), 'fallback.storage_class': ( 'socorro.unittest.external.test_crashstorage_base.B' ), } cm = ConfigurationManager( n, values_source_list=[value], argv_source=[] ) with cm.context() as config: eq_(config.primary.storage_class.foo, 'a') eq_(config.fallback.storage_class.foo, 'b') raw_crash = {'ooid': ''} crash_id = '1498dee9-9a45-45cc-8ec8-71bb62121203' dump = '12345' processed_crash = {'ooid': '', 'product': 17} fb_store = config.storage(config) # save_raw tests fb_store.primary_store.save_raw_crash = Mock() fb_store.fallback_store.save_raw_crash = Mock() fb_store.save_raw_crash(raw_crash, dump, crash_id) fb_store.primary_store.save_raw_crash.assert_called_with( raw_crash, dump, crash_id ) eq_(fb_store.fallback_store.save_raw_crash.call_count, 0) fb_store.primary_store.save_raw_crash = Mock() fb_store.primary_store.save_raw_crash.side_effect = Exception('!') fb_store.save_raw_crash(raw_crash, dump, crash_id) fb_store.primary_store.save_raw_crash.assert_called_with( raw_crash, dump, crash_id ) fb_store.fallback_store.save_raw_crash.assert_called_with( raw_crash, dump, crash_id ) fb_store.fallback_store.save_raw_crash = Mock() fb_store.fallback_store.save_raw_crash.side_effect = Exception('!') assert_raises( PolyStorageError, fb_store.save_raw_crash, raw_crash, dump, crash_id ) fb_store.primary_store.save_raw_crash.assert_called_with( raw_crash, dump, crash_id ) fb_store.fallback_store.save_raw_crash.assert_called_with( raw_crash, dump, crash_id ) # save_processed tests fb_store.primary_store.save_processed = Mock() fb_store.fallback_store.save_processed = Mock() fb_store.save_processed(processed_crash) fb_store.primary_store.save_processed.assert_called_with( processed_crash ) eq_(fb_store.fallback_store.save_processed.call_count, 0) fb_store.primary_store.save_processed = Mock() fb_store.primary_store.save_processed.side_effect = Exception('!') fb_store.save_processed(processed_crash) fb_store.primary_store.save_processed.assert_called_with( processed_crash ) fb_store.fallback_store.save_processed.assert_called_with( processed_crash ) fb_store.fallback_store.save_processed = Mock() fb_store.fallback_store.save_processed.side_effect = Exception('!') assert_raises( PolyStorageError, fb_store.save_processed, processed_crash ) fb_store.primary_store.save_processed.assert_called_with( processed_crash ) fb_store.fallback_store.save_processed.assert_called_with( processed_crash ) # close tests fb_store.primary_store.close = Mock() fb_store.fallback_store.close = Mock() fb_store.close() fb_store.primary_store.close.assert_called_with() fb_store.fallback_store.close.assert_called_with() fb_store.primary_store.close = Mock() fb_store.fallback_store.close = Mock() fb_store.fallback_store.close.side_effect = NotImplementedError() fb_store.close() fb_store.primary_store.close.assert_called_with() fb_store.fallback_store.close.assert_called_with() fb_store.primary_store.close = Mock() fb_store.primary_store.close.side_effect = Exception('!') fb_store.close() fb_store.primary_store.close.assert_called_with() fb_store.fallback_store.close.assert_called_with() fb_store.fallback_store.close = Mock() fb_store.fallback_store.close.side_effect = Exception('!') assert_raises(PolyStorageError, fb_store.close) fb_store.primary_store.close.assert_called_with() fb_store.fallback_store.close.assert_called_with()
def test_write_with_imported_module_with_internal_mappings(self): import os from configman.tests.values_for_module_tests_1 import Alpha, foo d = { 'a': 18, 'b': 'hello', 'c': [1, 2, 3], 'd': { 'host': 'localhost', 'port': 5432, } } definitions = { 'os_module': os, 'a': 17, 'imported_class': Alpha, 'imported_function': foo, 'xxx': { 'yyy': Option('yyy', default=d) }, 'e': None, } required_config = Namespace() required_config.add_option( 'minimal_version_for_understanding_refusal', doc='ignore the Thottleable protocol', default={'Firefox': '3.5.4'}, ) cm = ConfigurationManager( [definitions, required_config], values_source_list=[], ) cm.get_config() s = StringIO() @contextlib.contextmanager def s_opener(): yield s cm.write_conf('py', s_opener) generated_python_module_text = s.getvalue() expected = """# generated Python configman file from configman.dotdict import DotDict from configman.tests.values_for_module_tests_1 import ( Alpha, foo, ) import os # the following symbols will be ignored by configman when # this module is used as a value source. This will # suppress the mismatch warning since these symbols are # values for options, not option names themselves. ignore_symbol_list = [ "Alpha", "DotDict", "foo", "os", ] # a a = 17 # e e = None # imported_class imported_class = Alpha # imported_function imported_function = foo # ignore the Thottleable protocol minimal_version_for_understanding_refusal = { "Firefox": "3.5.4" } # os_module os_module = os # Namespace: xxx xxx = DotDict() xxx.yyy = { "a": 18, "b": "hello", "c": [ 1, 2, 3 ], "d": { "host": "localhost", "port": 5432 } } """ self.assertEqual(generated_python_module_text, expected)
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = Mock() required_config.add_option('logger', default=mock_logging) required_config.update(CrashStorageBase.required_config) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, }], argv_source=[] ) with config_manager.context() as config: crashstorage = CrashStorageBase( config, quit_check_callback=fake_quit_check ) crashstorage.save_raw_crash({}, 'payload', 'ooid') crashstorage.save_processed({}) assert_raises( NotImplementedError, crashstorage.get_raw_crash, 'ooid' ) assert_raises( NotImplementedError, crashstorage.get_raw_dump, 'ooid' ) assert_raises( NotImplementedError, crashstorage.get_unredacted_processed, 'ooid' ) assert_raises( NotImplementedError, crashstorage.remove, 'ooid' ) eq_(crashstorage.new_crashes(), []) crashstorage.close() with config_manager.context() as config: class MyCrashStorageTest(CrashStorageBase): def save_raw_crash(self, raw_crash, dumps, crash_id): eq_(crash_id, "fake_id") eq_(raw_crash, "fake raw crash") eq_( sorted(dumps.keys()), sorted(['one', 'two', 'three']) ) eq_( sorted(dumps.values()), sorted(['eins', 'zwei', 'drei']) ) values = ['eins', 'zwei', 'drei'] def open_function(*args, **kwargs): return values.pop(0) crashstorage = MyCrashStorageTest( config, quit_check_callback=fake_quit_check ) with mock.patch("__builtin__.open") as open_mock: open_mock.return_value = mock.MagicMock() ( open_mock.return_value.__enter__ .return_value.read.side_effect ) = open_function crashstorage.save_raw_crash_with_file_dumps( "fake raw crash", FileDumpsMapping({ 'one': 'eins', 'two': 'zwei', 'three': 'drei' }), 'fake_id' )
#------------------------------------------------------------------------------ def query2(conn): """another transaction to be executed by the database""" raise Exception("not a database related error") #============================================================================== if __name__ == "__main__": definition_source = Namespace() definition_source.add_option('transaction_executor_class', default=TransactionExecutorWithBackoff, doc='a class that will execute transactions') c = ConfigurationManager(definition_source, app_name='advanced_demo_3', app_description=__doc__) with c.context() as config: # the configuration has a class that can execute transactions # we instantiate it here. executor = config.transaction_executor_class(config) # this first query has a 50% probability of failing due to a database # connectivity problem. If the transaction_executor_class is a class # with backing off retry, you'll see the transaction tried over and # over until it succeeds. print "\n**** First query" executor.do_transaction(query1) # this second query has a 50% probability of failing due to a non-
def test_benchmarking_crashstore(self): required_config = Namespace() mock_logging = Mock() required_config.add_option('logger', default=mock_logging) required_config.update(BenchmarkingCrashStorage.get_required_config()) fake_crash_store = Mock() config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'wrapped_crashstore': fake_crash_store, 'benchmark_tag': 'test' }], argv_source=[] ) with config_manager.context() as config: crashstorage = BenchmarkingCrashStorage( config, quit_check_callback=fake_quit_check ) crashstorage.start_timer = lambda: 0 crashstorage.end_timer = lambda: 1 fake_crash_store.assert_called_with(config, fake_quit_check) crashstorage.save_raw_crash({}, 'payload', 'ooid') crashstorage.wrapped_crashstore.save_raw_crash.assert_called_with( {}, 'payload', 'ooid' ) mock_logging.debug.assert_called_with( '%s save_raw_crash %s', 'test', 1 ) mock_logging.debug.reset_mock() crashstorage.save_processed({}) crashstorage.wrapped_crashstore.save_processed.assert_called_with( {} ) mock_logging.debug.assert_called_with( '%s save_processed %s', 'test', 1 ) mock_logging.debug.reset_mock() crashstorage.save_raw_and_processed({}, 'payload', {}, 'ooid') crashstorage.wrapped_crashstore.save_raw_and_processed \ .assert_called_with( {}, 'payload', {}, 'ooid' ) mock_logging.debug.assert_called_with( '%s save_raw_and_processed %s', 'test', 1 ) mock_logging.debug.reset_mock() crashstorage.get_raw_crash('uuid') crashstorage.wrapped_crashstore.get_raw_crash.assert_called_with( 'uuid' ) mock_logging.debug.assert_called_with( '%s get_raw_crash %s', 'test', 1 ) mock_logging.debug.reset_mock() crashstorage.get_raw_dump('uuid') crashstorage.wrapped_crashstore.get_raw_dump.assert_called_with( 'uuid' ) mock_logging.debug.assert_called_with( '%s get_raw_dump %s', 'test', 1 ) mock_logging.debug.reset_mock() crashstorage.get_raw_dumps('uuid') crashstorage.wrapped_crashstore.get_raw_dumps.assert_called_with( 'uuid' ) mock_logging.debug.assert_called_with( '%s get_raw_dumps %s', 'test', 1 ) mock_logging.debug.reset_mock() crashstorage.get_raw_dumps_as_files('uuid') crashstorage.wrapped_crashstore.get_raw_dumps_as_files \ .assert_called_with( 'uuid' ) mock_logging.debug.assert_called_with( '%s get_raw_dumps_as_files %s', 'test', 1 ) mock_logging.debug.reset_mock() crashstorage.get_unredacted_processed('uuid') crashstorage.wrapped_crashstore.get_unredacted_processed \ .assert_called_with( 'uuid' ) mock_logging.debug.assert_called_with( '%s get_unredacted_processed %s', 'test', 1 ) mock_logging.debug.reset_mock()
def test_basic_postgres_save_processed_succeed_after_failures(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres, 'transaction_executor_class': TransactionExecutorWithLimitedBackoff, 'backoff_delays': [0, 0, 0], }], argv_source=[]) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) crashstorage.database.operational_exceptions = (OperationalError, ) database = crashstorage.database.return_value = mock.MagicMock() ok_(isinstance(database, mock.Mock)) fetch_all_returns = [ ((666, ), ), None, ((23, ), ), ] def fetch_all_func(*args): result = fetch_all_returns.pop(0) return result fetch_mock = mock.Mock() fetch_mock.fetchall.side_effect = fetch_all_func connection_trouble = [ OperationalError('bad'), OperationalError('worse'), ] def broken_connection(*args): try: result = connection_trouble.pop(0) raise result except IndexError: return fetch_mock m = mock.MagicMock() m.__enter__.return_value = m database = crashstorage.database.return_value = m m.cursor.side_effect = broken_connection crashstorage.save_processed(a_processed_crash) eq_(m.cursor.call_count, 10) eq_(m.cursor().fetchall.call_count, 3) eq_(m.cursor().execute.call_count, 8) expected_execute_args = ( (('savepoint MainThread', None), ), (('insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id', [ None, '0x1c', '...', '20120309050057', '2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884', 'None | 0', 'arm', '2012-04-08 10:56:41.558922', None, None, '*****@*****.**', 'high', '[blank]', None, 22385, None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ', 'SignatureTool: signature truncated due to length', 'plugin', 'FennecAndroid', 'FA-888888', 'SIGSEGV', 'default', 'libxul.so@0x117441c', '2012-04-08 10:56:50.440752', True, [], False, 170, None, None, 'http://embarrassing.p**n.com', '936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1' ]), ), (('release savepoint MainThread', None), ), (('select id from plugins where filename = %s and name = %s', ('dwight.txt', 'wilma')), ), (('insert into plugins (filename, name) values (%s, %s) returning id', ('dwight.txt', 'wilma')), ), (('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)', (666, 23, '2012-04-08 10:56:41.558922', '69')), ), (('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)', (666, '2012-04-08 10:56:41.558922', 0, '{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')), ), (("""WITH update_processed_crash AS ( UPDATE processed_crashes_20120402 SET processed_crash = %(processed_json)s, date_processed = %(date_processed)s WHERE uuid = %(uuid)s RETURNING 1), insert_processed_crash AS ( INSERT INTO processed_crashes_20120402 (uuid, processed_crash, date_processed) ( SELECT %(uuid)s as uuid, %(processed_json)s as processed_crash, %(date_processed)s as date_processed WHERE NOT EXISTS ( SELECT uuid from processed_crashes_20120402 WHERE uuid = %(uuid)s LIMIT 1)) RETURNING 2) SELECT * from update_processed_crash UNION ALL SELECT * from insert_processed_crash """, { 'uuid': '936ce666-ff3b-4c7a-9674-367fe2120408', 'processed_json': '{"startedDateTime": "2012-04-08 10:56:50.440752", "crashedThread": 8, "cpu_info": "None | 0", "PluginName": "wilma", "install_age": 22385, "topmost_filenames": [], "user_comments": null, "user_id": null, "uuid": "936ce666-ff3b-4c7a-9674-367fe2120408", "flash_version": "[blank]", "os_version": "0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ", "PluginVersion": "69", "addons_checked": null, "completeddatetime": "2012-04-08 10:56:50.902884", "productid": "FA-888888", "success": true, "exploitability": "high", "client_crash_date": "2012-04-08 10:52:42.0", "PluginFilename": "dwight.txt", "dump": "...", "truncated": false, "product": "FennecAndroid", "distributor": null, "processor_notes": "SignatureTool: signature truncated due to length", "uptime": 170, "release_channel": "default", "distributor_version": null, "process_type": "plugin", "id": 361399767, "hangid": null, "version": "13.0a1", "build": "20120309050057", "ReleaseChannel": "default", "email": "*****@*****.**", "app_notes": "...", "os_name": "Linux", "last_crash": null, "date_processed": "2012-04-08 10:56:41.558922", "cpu_name": "arm", "reason": "SIGSEGV", "address": "0x1c", "url": "http://embarrassing.p**n.com", "signature": "libxul.so@0x117441c", "addons": [["{1a5dabbd-0e74-41da-b532-a364bb552cab}", "1.0.4.1"]]}', 'date_processed': '2012-04-08 10:56:41.558922' }), ), ) actual_execute_args = m.cursor().execute.call_args_list for expected, actual in zip(expected_execute_args, actual_execute_args): expected_sql, expected_params = expected[0] expected_sql = remove_whitespace(expected_sql) actual_sql, actual_params = actual[0] actual_sql = remove_whitespace(actual_sql) eq_(expected_sql, actual_sql) eq_(expected_params, actual_params)