def test_basic_usage_with_postgres(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', #default=TransactionExecutorWithBackoff, default=TransactionExecutor, doc='a class that will execute transactions') required_config.add_option('database_class', default=MockConnectionContext, from_string_converter=class_converter) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[], argv_source=[]) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class( config, mocked_context) _function_calls = [] # some mutable def mock_function(connection): assert isinstance(connection, MockConnection) _function_calls.append(connection) executor(mock_function) ok_(_function_calls) eq_(commit_count, 1) eq_(rollback_count, 0)
def test_basic_postgres_save_processed_success_3_truncations(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[]) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) with mock.patch( 'socorro.external.postgresql.crashstorage.single_value_sql' ) as mocked_sql_execute: fake_connection = mock.Mock(), crashstorage._save_processed_report( fake_connection, a_processed_crash_with_everything_too_long) mocked_sql_execute.assert_called_with( fake_connection, "\n WITH\n update_report AS (\n UPDATE reports_20120402 SET\n addons_checked = %s, address = %s, app_notes = %s, build = %s, client_crash_date = %s, completed_datetime = %s, cpu_info = %s, cpu_name = %s, date_processed = %s, distributor = %s, distributor_version = %s, email = %s, exploitability = %s, flash_version = %s, hangid = %s, install_age = %s, last_crash = %s, os_name = %s, os_version = %s, processor_notes = %s, process_type = %s, product = %s, productid = %s, reason = %s, release_channel = %s, signature = %s, started_datetime = %s, success = %s, topmost_filenames = %s, truncated = %s, uptime = %s, user_comments = %s, user_id = %s, url = %s, uuid = %s, version = %s\n WHERE uuid = %s\n RETURNING id\n ),\n insert_report AS (\n INSERT INTO reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version)\n ( SELECT\n %s as addons_checked, %s as address, %s as app_notes, %s as build, %s as client_crash_date, %s as completed_datetime, %s as cpu_info, %s as cpu_name, %s as date_processed, %s as distributor, %s as distributor_version, %s as email, %s as exploitability, %s as flash_version, %s as hangid, %s as install_age, %s as last_crash, %s as os_name, %s as os_version, %s as processor_notes, %s as process_type, %s as product, %s as productid, %s as reason, %s as release_channel, %s as signature, %s as started_datetime, %s as success, %s as topmost_filenames, %s as truncated, %s as uptime, %s as user_comments, %s as user_id, %s as url, %s as uuid, %s as version\n WHERE NOT EXISTS (\n SELECT uuid from reports_20120402\n WHERE\n uuid = %s\n LIMIT 1\n )\n )\n RETURNING id\n )\n SELECT * from update_report\n UNION ALL\n SELECT * from insert_report\n ", a_processed_report_with_everything_truncated * 2)
def test_basic_key_error_on_save_processed(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[]) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) database = crashstorage.database.return_value = mock.MagicMock() ok_(isinstance(database, mock.Mock)) broken_processed_crash = { "product": "Peter", "version": "1.0B3", "ooid": "abc123", "submitted_timestamp": time.time(), "unknown_field": 'whatever' } assert_raises(KeyError, crashstorage.save_processed, broken_processed_crash)
def test_failure_limited_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }], argv_source=[] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) failure_exception = pyelasticsearch.exceptions.Timeout mock_es.index.side_effect = failure_exception crash_id = a_processed_crash['uuid'] assert_raises( pyelasticsearch.exceptions.Timeout, es_storage.save_raw_and_processed, a_raw_crash, None, a_processed_crash.copy(), crash_id, ) expected_crash = { 'crash_id': crash_id, 'processed_crash': a_processed_crash.copy(), 'raw_crash': a_raw_crash } expected_request_args = ( 'socorro201214', 'crash_reports', expected_crash ) expected_request_kwargs = { 'id': crash_id, } mock_es.index.assert_called_with( *expected_request_args, **expected_request_kwargs )
def test_basic_key_error_on_save_processed(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }] ) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) database = crashstorage.database.return_value = mock.MagicMock() self.assertTrue(isinstance(database, mock.Mock)) broken_processed_crash = { "product": "Peter", "version": "1.0B3", "ooid": "abc123", "submitted_timestamp": time.time(), "unknown_field": 'whatever' } self.assertRaises(KeyError, crashstorage.save_processed, broken_processed_crash)
def test_no_rollback_exception_with_postgres(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutor, doc='a class that will execute transactions') required_config.add_option('database_class', default=MockConnectionContext, from_string_converter=class_converter) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[], ) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class( config, mocked_context) def mock_function(connection): assert isinstance(connection, MockConnection) raise NameError('crap!') self.assertRaises(NameError, executor, mock_function) self.assertEqual(commit_count, 0) self.assertEqual(rollback_count, 0) self.assertTrue(mock_logging.errors)
def get_config_context(self, es_index=None): mock_logging = mock.Mock() storage_config = \ crashstorage.ElasticSearchCrashStorage.get_required_config() middleware_config = MiddlewareApp.get_required_config() middleware_config.add_option('logger', default=mock_logging) values_source = { 'logger': mock_logging, 'elasticsearch_index': 'socorro_integration_test', 'backoff_delays': [1], 'elasticsearch_timeout': 5, } if es_index: values_source['elasticsearch_index'] = es_index config_manager = ConfigurationManager( [storage_config, middleware_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[os.environ, values_source], argv_source=[], ) with config_manager.context() as config: # This is an ugly hack to compensate for a bug in configman. # See https://github.com/mozilla/configman/issues/103 config.backoff_delays = [1] return config
def test_operation_error_with_postgres_with_backoff(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutorWithInfiniteBackoff, #default=TransactionExecutor, doc='a class that will execute transactions' ) required_config.add_option( 'database_class', default=MockConnectionContext, from_string_converter=class_converter ) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{'backoff_delays': [2, 4, 6, 10, 15]}], argv_source=[] ) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class(config, mocked_context) _function_calls = [] # some mutable _sleep_count = [] def mock_function(connection): assert isinstance(connection, MockConnection) _function_calls.append(connection) # the default sleep times are going to be, # 2, 4, 6, 10, 15 # so after 2 + 4 + 6 + 10 + 15 seconds # all will be exhausted if sum(_sleep_count) < sum([2, 4, 6, 10, 15]): raise psycopg2.OperationalError('Arh!') def mock_sleep(n): _sleep_count.append(n) # monkey patch the sleep function from inside transaction_executor _orig_sleep = socorro.database.transaction_executor.time.sleep socorro.database.transaction_executor.time.sleep = mock_sleep try: executor(mock_function) ok_(_function_calls) eq_(commit_count, 1) eq_(rollback_count, 5) ok_(mock_logging.criticals) eq_(len(mock_logging.criticals), 5) ok_(len(_sleep_count) > 10) finally: socorro.database.transaction_executor.time.sleep = _orig_sleep
def test_wrapper(self, metricsmock): fake_crash_store_class = mock.MagicMock() fake_crash_store_class.__name__ = 'Phil' config_manager = ConfigurationManager( [MetricsBenchmarkingWrapper.get_required_config()], values_source_list=[{ 'wrapped_object_class': fake_crash_store_class, 'metrics_prefix': 'phil', 'active_list': 'run', }], argv_source=[] ) with config_manager.context() as config: mbw = MetricsBenchmarkingWrapper(config) with metricsmock as mm: mbw.run() mbw.walk() # Assert that the timing call occurred assert len(mm.get_records()) == 1 assert mm.has_record('timing', stat='phil.Phil.run') # Assert that the wrapped crash storage class .run() and .walk() were # called on the instance fake_crash_store_class.return_value.run.assert_called_with() fake_crash_store_class.return_value.walk.assert_called_with()
def test_success(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) es_storage.save_processed(a_processed_crash) expected_request_args = ('socorro201214', 'crash_reports', a_processed_crash) expected_request_kwargs = { 'replication': 'async', 'id': a_processed_crash['uuid'], } mock_es.index.assert_called_with(*expected_request_args, **expected_request_kwargs)
def test_wrapper(self, metricsmock): fake_crash_store_class = mock.MagicMock() fake_crash_store_class.__name__ = 'Phil' config_manager = ConfigurationManager( [MetricsBenchmarkingWrapper.get_required_config()], values_source_list=[{ 'wrapped_object_class': fake_crash_store_class, 'metrics_prefix': 'phil', 'active_list': 'run', }], argv_source=[]) with config_manager.context() as config: mbw = MetricsBenchmarkingWrapper(config) with metricsmock as mm: mbw.run() mbw.walk() # Assert that the timing call occurred assert len(mm.get_records()) == 1 assert mm.has_record('timing', stat='phil.Phil.run') # Assert that the wrapped crash storage class .run() and .walk() were # called on the instance fake_crash_store_class.return_value.run.assert_called_with() fake_crash_store_class.return_value.walk.assert_called_with()
def test_hbase_crashstorage_basic_error(self): mock_logging = mock.Mock() required_config = HBaseCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'hbase_timeout': 100, 'hbase_host': commonconfig.hbaseHost.default, 'hbase_port': commonconfig.hbasePort.default, }]) with config_manager.context() as config: hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client' with mock.patch(hbaseclient_) as hclient: klass = hclient.HBaseConnectionForCrashReports def retry_raiser(*args, **kwargs): raise SomeThriftError('try again') klass.put_json_dump.side_effect = ValueError('crap!') crashstorage = HBaseCrashStorage(config) raw = ('{"name":"Peter", ' '"submitted_timestamp":"%d"}' % time.time()) # Note, we're not expect it to raise an error self.assertRaises(ValueError, crashstorage.save_raw_crash, json.loads(raw), raw, "abc123")
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = Mock() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, }] ) with config_manager.context() as config: crashstorage = CrashStorageBase( config, quit_check_callback=fake_quit_check ) crashstorage.save_raw_crash({}, 'payload', 'ooid') crashstorage.save_processed({}) self.assertRaises(NotImplementedError, crashstorage.get_raw_crash, 'ooid') self.assertRaises(NotImplementedError, crashstorage.get_raw_dump, 'ooid') self.assertRaises(NotImplementedError, crashstorage.get_processed, 'ooid') self.assertRaises(NotImplementedError, crashstorage.remove, 'ooid') self.assertRaises(StopIteration, crashstorage.new_crashes) crashstorage.close()
def test_no_rollback_exception_with_postgres(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutor, doc='a class that will execute transactions' ) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{'database_class': MockConnectionContext}], ) with config_manager.context() as config: executor = config.transaction_executor_class(config) def mock_function(connection): assert isinstance(connection, MockConnection) raise NameError('crap!') self.assertRaises(NameError, executor, mock_function) self.assertEqual(commit_count, 0) self.assertEqual(rollback_count, 0) self.assertTrue(mock_logging.errors)
def test_basic_setup(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = ProcessorAppRegistrationClient.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[]) m_registration = mock.Mock() class NoRegister(ProcessorAppRegistrationClient): _registration = m_registration with config_manager.context() as config: registrar = NoRegister(config) eq_(registrar.last_checkin_ts, datetime(1999, 1, 1, tzinfo=UTC)) ok_(registrar.processor_id is None) eq_(registrar.processor_name, 'unknown') eq_(m_registration.call_count, 1)
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = Mock() required_config.add_option("logger", default=mock_logging) required_config.update(CrashStorageBase.required_config) config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[{"logger": mock_logging}], argv_source=[], ) with config_manager.context() as config: crashstorage = CrashStorageBase(config, quit_check_callback=fake_quit_check) crashstorage.save_raw_crash({}, "payload", "ooid") crashstorage.save_processed({}) assert_raises(NotImplementedError, crashstorage.get_raw_crash, "ooid") assert_raises(NotImplementedError, crashstorage.get_raw_dump, "ooid") assert_raises(NotImplementedError, crashstorage.get_unredacted_processed, "ooid") assert_raises(NotImplementedError, crashstorage.remove, "ooid") eq_(crashstorage.new_crashes(), []) crashstorage.close()
def test_basic_postgres_save_processed_success_3_truncations(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[] ) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) with mock.patch( 'socorro.external.postgresql.crashstorage.single_value_sql' ) as mocked_sql_execute: fake_connection = mock.Mock(), crashstorage._save_processed_report( fake_connection, a_processed_crash_with_everything_too_long ) mocked_sql_execute.assert_called_with( fake_connection, "\n WITH\n update_report AS (\n UPDATE reports_20120402 SET\n addons_checked = %s, address = %s, app_notes = %s, build = %s, client_crash_date = %s, completed_datetime = %s, cpu_info = %s, cpu_name = %s, date_processed = %s, distributor = %s, distributor_version = %s, email = %s, exploitability = %s, flash_version = %s, hangid = %s, install_age = %s, last_crash = %s, os_name = %s, os_version = %s, processor_notes = %s, process_type = %s, product = %s, productid = %s, reason = %s, release_channel = %s, signature = %s, started_datetime = %s, success = %s, topmost_filenames = %s, truncated = %s, uptime = %s, user_comments = %s, user_id = %s, url = %s, uuid = %s, version = %s\n WHERE uuid = %s\n RETURNING id\n ),\n insert_report AS (\n INSERT INTO reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, productid, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version)\n ( SELECT\n %s as addons_checked, %s as address, %s as app_notes, %s as build, %s as client_crash_date, %s as completed_datetime, %s as cpu_info, %s as cpu_name, %s as date_processed, %s as distributor, %s as distributor_version, %s as email, %s as exploitability, %s as flash_version, %s as hangid, %s as install_age, %s as last_crash, %s as os_name, %s as os_version, %s as processor_notes, %s as process_type, %s as product, %s as productid, %s as reason, %s as release_channel, %s as signature, %s as started_datetime, %s as success, %s as topmost_filenames, %s as truncated, %s as uptime, %s as user_comments, %s as user_id, %s as url, %s as uuid, %s as version\n WHERE NOT EXISTS (\n SELECT uuid from reports_20120402\n WHERE\n uuid = %s\n LIMIT 1\n )\n )\n RETURNING id\n )\n SELECT * from update_report\n UNION ALL\n SELECT * from insert_report\n ", a_processed_report_with_everything_truncated * 2 )
def test_requested_processor_id(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = ProcessorAppRegistrationClient.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }] ) m_registration = mock.Mock() class NoRegister(ProcessorAppRegistrationClient): _registration = m_registration with config_manager.context() as config: registrar = NoRegister(config) i = registrar._requested_processor_id(0) self.assertEqual(i, 0) i = registrar._requested_processor_id(1) self.assertEqual(1, i) i = registrar._requested_processor_id('host') self.assertEqual('host', i) i = registrar._requested_processor_id('auto') self.assertEqual('auto', i) self.assertRaises(ValueError, registrar._requested_processor_id, 'dwight')
def test_wrapper(self, metricsmock): fake_crash_store_class = mock.MagicMock() fake_crash_store_class.__name__ = "Phil" config_manager = ConfigurationManager( [MetricsBenchmarkingWrapper.get_required_config()], values_source_list=[{ "wrapped_object_class": fake_crash_store_class, "metrics_prefix": "phil", "active_list": "run", }], argv_source=[], ) with config_manager.context() as config: mbw = MetricsBenchmarkingWrapper(config) with metricsmock as mm: mbw.run() mbw.walk() mm.assert_timing_once("phil.Phil.run") # Assert that the wrapped crash storage class .run() and .walk() were # called on the instance fake_crash_store_class.return_value.run.assert_called_with() fake_crash_store_class.return_value.walk.assert_called_with()
def test_polycrashstorage_processed_immutability_with_nonmutating(self): """Verifies if a crash storage says it doesn't mutate the class that " we don't do a deepcopy """ n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_namespaces': 'store1', 'store1.crashstorage_class': ('socorro.unittest.external.test_crashstorage_base' '.NonMutatingProcessedCrashCrashStorage'), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {'ooid': '12345'} dump = '12345' processed_crash = {'foo': 'bar'} poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, 'n') # We have a crashstorage that says it's not mutating, but deletes a # key so that we can verify that the code went down the right path # in the processor. assert 'foo' not in processed_crash
def test_poly_crash_storage_processed_crash_immutability(self): n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_namespaces': 'store1', 'store1.crashstorage_class': ('socorro.unittest.external.test_crashstorage_base' '.MutatingProcessedCrashCrashStorage'), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {'ooid': '12345'} dump = '12345' processed_crash = {'foo': 'bar'} poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, 'n') # It's important to be aware that the only thing # MutatingProcessedCrashCrashStorage class does, in its # save_raw_and_processed() is that it deletes a key called # 'foo'. # This test makes sure that the dict processed_crash here # is NOT affected. assert processed_crash['foo'] == 'bar'
def test_operation_error_with_postgres_with_backoff_with_rollback(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutorWithInfiniteBackoff, #default=TransactionExecutor, doc='a class that will execute transactions') required_config.add_option('database_class', default=MockConnectionContext, from_string_converter=class_converter) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'backoff_delays': [2, 4, 6, 10, 15] }], argv_source=[]) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class( config, mocked_context) _function_calls = [] # some mutable _sleep_count = [] def mock_function(connection): assert isinstance(connection, MockConnection) connection.transaction_status = \ psycopg2.extensions.TRANSACTION_STATUS_INTRANS _function_calls.append(connection) # the default sleep times are going to be, # 2, 4, 6, 10, 15 # so after 2 + 4 + 6 + 10 + 15 seconds # all will be exhausted if sum(_sleep_count) < sum([2, 4, 6, 10, 15]): raise psycopg2.OperationalError('Arh!') def mock_sleep(n): _sleep_count.append(n) # monkey patch the sleep function from inside transaction_executor _orig_sleep = socorro.database.transaction_executor.time.sleep socorro.database.transaction_executor.time.sleep = mock_sleep try: executor(mock_function) ok_(_function_calls) eq_(commit_count, 1) eq_(rollback_count, 5) ok_(mock_logging.criticals) eq_(len(mock_logging.criticals), 5) ok_(len(_sleep_count) > 10) finally: socorro.database.transaction_executor.time.sleep = _orig_sleep
def test_basic_usage_with_postgres(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', #default=TransactionExecutorWithBackoff, default=TransactionExecutor, doc='a class that will execute transactions' ) required_config.add_option( 'database_class', default=MockConnectionContext, from_string_converter=class_converter ) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[], ) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class(config, mocked_context) _function_calls = [] # some mutable def mock_function(connection): assert isinstance(connection, MockConnection) _function_calls.append(connection) executor(mock_function) self.assertTrue(_function_calls) self.assertEqual(commit_count, 1) self.assertEqual(rollback_count, 0)
def test_poly_crash_storage_immutability_deeper(self): n = Namespace() n.add_option("storage", default=PolyCrashStorage) n.add_option("logger", default=mock.Mock()) value = { "storage_namespaces": "store1", "store1.crashstorage_class": ("socorro.unittest.external.test_crashstorage_base" ".MutatingProcessedCrashCrashStorage"), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {"ooid": "12345"} dump = "12345" processed_crash = { "foo": DotDict({"other": "thing"}), "bar": DotDict({"something": "else"}), } poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, "n") assert processed_crash["foo"]["other"] == "thing" assert processed_crash["bar"]["something"] == "else"
def test_poly_crash_storage_immutability_deeper(self): n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_classes': ( 'socorro.unittest.external.test_crashstorage_base' '.MutatingProcessedCrashCrashStorage' ), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {'ooid': '12345'} dump = '12345' processed_crash = { 'foo': DotDict({'other': 'thing'}), 'bar': SocorroDotDict({'something': 'else'}), } poly_store = config.storage(config) poly_store.save_raw_and_processed( raw_crash, dump, processed_crash, 'n' ) eq_(processed_crash['foo']['other'], 'thing') eq_(processed_crash['bar']['something'], 'else')
def test_poly_crash_storage_processed_crash_immutability(self): n = Namespace() n.add_option("storage", default=PolyCrashStorage) n.add_option("logger", default=mock.Mock()) value = { "storage_namespaces": "store1", "store1.crashstorage_class": ("socorro.unittest.external.test_crashstorage_base" ".MutatingProcessedCrashCrashStorage"), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {"ooid": "12345"} dump = "12345" processed_crash = {"foo": "bar"} poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, "n") # It's important to be aware that the only thing # MutatingProcessedCrashCrashStorage class does, in its # save_raw_and_processed() is that it deletes a key called # 'foo'. # This test makes sure that the dict processed_crash here # is NOT affected. assert processed_crash["foo"] == "bar"
def test_polycrashstorage_processed_immutability_with_nonmutating(self): """Verifies if a crash storage says it doesn't mutate the class that " we don't do a deepcopy """ n = Namespace() n.add_option("storage", default=PolyCrashStorage) n.add_option("logger", default=mock.Mock()) value = { "storage_namespaces": "store1", "store1.crashstorage_class": ("socorro.unittest.external.test_crashstorage_base" ".NonMutatingProcessedCrashCrashStorage"), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {"ooid": "12345"} dump = "12345" processed_crash = {"foo": "bar"} poly_store = config.storage(config) poly_store.save_raw_and_processed(raw_crash, dump, processed_crash, "n") # We have a crashstorage that says it's not mutating, but deletes a # key so that we can verify that the code went down the right path # in the processor. assert "foo" not in processed_crash
def test_poly_crash_storage_immutability_deeper(self): n = Namespace() n.add_option( 'storage', default=PolyCrashStorage, ) n.add_option( 'logger', default=mock.Mock(), ) value = { 'storage_classes': ( 'socorro.unittest.external.test_crashstorage_base' '.MutatingProcessedCrashCrashStorage' ), } cm = ConfigurationManager(n, values_source_list=[value]) with cm.context() as config: raw_crash = {'ooid': '12345'} dump = '12345' processed_crash = { 'foo': DotDict({'other': 'thing'}), 'bar': SocorroDotDict({'something': 'else'}), } poly_store = config.storage(config) poly_store.save_raw_and_processed( raw_crash, dump, processed_crash, 'n' ) assert processed_crash['foo']['other'] == 'thing' assert processed_crash['bar']['something'] == 'else'
def test_requested_processor_id(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = ProcessorAppRegistrationClient.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[]) m_registration = mock.Mock() class NoRegister(ProcessorAppRegistrationClient): _registration = m_registration with config_manager.context() as config: registrar = NoRegister(config) i = registrar._requested_processor_id(0) eq_(i, 0) i = registrar._requested_processor_id(1) eq_(1, i) i = registrar._requested_processor_id('host') eq_('host', i) i = registrar._requested_processor_id('auto') eq_('auto', i) assert_raises(ValueError, registrar._requested_processor_id, 'dwight')
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = Mock() required_config.add_option('logger', default=mock_logging) required_config.update(CrashStorageBase.required_config) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, }], argv_source=[]) with config_manager.context() as config: crashstorage = CrashStorageBase( config, quit_check_callback=fake_quit_check) crashstorage.save_raw_crash({}, 'payload', 'ooid') crashstorage.save_processed({}) assert_raises(NotImplementedError, crashstorage.get_raw_crash, 'ooid') assert_raises(NotImplementedError, crashstorage.get_raw_dump, 'ooid') assert_raises(NotImplementedError, crashstorage.get_unredacted_processed, 'ooid') assert_raises(NotImplementedError, crashstorage.remove, 'ooid') eq_(crashstorage.new_crashes(), []) crashstorage.close()
def test_basic_setup(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = ProcessorAppRegistrationClient.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }] ) m_registration = mock.Mock() class NoRegister(ProcessorAppRegistrationClient): _registration = m_registration with config_manager.context() as config: registrar = NoRegister(config) self.assertEqual(registrar.last_checkin_ts, datetime(1999, 1, 1, tzinfo=UTC)) self.assertTrue(registrar.processor_id is None) self.assertEqual(registrar.processor_name, 'unknown') self.assertEqual(m_registration.call_count, 1)
def test_basic_crashstorage(self): required_config = Namespace() mock_logging = mock.Mock() required_config.add_option("logger", default=mock_logging) required_config.update(CrashStorageBase.required_config) config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[{ "logger": mock_logging }], argv_source=[], ) with config_manager.context() as config: crashstorage = CrashStorageBase(config) crashstorage.save_raw_crash({}, "payload", "ooid") with pytest.raises(NotImplementedError): crashstorage.get_raw_crash("ooid") with pytest.raises(NotImplementedError): crashstorage.get_raw_dump("ooid") with pytest.raises(NotImplementedError): crashstorage.get_unredacted_processed("ooid") with pytest.raises(NotImplementedError): crashstorage.remove("ooid") crashstorage.close()
def test_benchmarking_crashstore(self): required_config = Namespace() mock_logging = Mock() required_config.add_option("logger", default=mock_logging) required_config.update(BenchmarkingCrashStorage.get_required_config()) fake_crash_store = Mock() config_manager = ConfigurationManager( [required_config], app_name="testapp", app_version="1.0", app_description="app description", values_source_list=[ {"logger": mock_logging, "wrapped_crashstore": fake_crash_store, "benchmark_tag": "test"} ], argv_source=[], ) with config_manager.context() as config: crashstorage = BenchmarkingCrashStorage(config, quit_check_callback=fake_quit_check) crashstorage.start_timer = lambda: 0 crashstorage.end_timer = lambda: 1 fake_crash_store.assert_called_with(config, fake_quit_check) crashstorage.save_raw_crash({}, "payload", "ooid") crashstorage.wrapped_crashstore.save_raw_crash.assert_called_with({}, "payload", "ooid") mock_logging.debug.assert_called_with("%s save_raw_crash %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.save_processed({}) crashstorage.wrapped_crashstore.save_processed.assert_called_with({}) mock_logging.debug.assert_called_with("%s save_processed %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_raw_crash("uuid") crashstorage.wrapped_crashstore.get_raw_crash.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_raw_crash %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_raw_dump("uuid") crashstorage.wrapped_crashstore.get_raw_dump.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_raw_dump %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_raw_dumps("uuid") crashstorage.wrapped_crashstore.get_raw_dumps.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_raw_dumps %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_raw_dumps_as_files("uuid") crashstorage.wrapped_crashstore.get_raw_dumps_as_files.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_raw_dumps_as_files %s", "test", 1) mock_logging.debug.reset_mock() crashstorage.get_unredacted_processed("uuid") crashstorage.wrapped_crashstore.get_unredacted_processed.assert_called_with("uuid") mock_logging.debug.assert_called_with("%s get_unredacted_processed %s", "test", 1) mock_logging.debug.reset_mock()
def test_basic_postgres_save_raw_crash(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[] ) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) database = crashstorage.database.return_value = mock.MagicMock() self.assertTrue(isinstance(database, mock.Mock)) self.assertTrue('submitted_timestamp' in a_raw_crash) m = mock.MagicMock() m.__enter__.return_value = m database = crashstorage.database.return_value = m crashstorage.save_raw_crash( a_raw_crash, '', "936ce666-ff3b-4c7a-9674-367fe2120408" ) self.assertEqual(m.cursor.call_count, 3) self.assertEqual(m.cursor().execute.call_count, 3) expected_execute_args = ( (('savepoint MainThread', None),), (('insert into raw_crashes_20120402 (uuid, raw_crash, date_processed) values (%s, %s, %s)', ( '936ce666-ff3b-4c7a-9674-367fe2120408', '{"submitted_timestamp": "2012-04-08 10:52:42.0", "Version": "6.02E23", "ProductName": "Fennicky"}', "2012-04-08 10:52:42.0" )),), (('release savepoint MainThread', None),), ) actual_execute_args = m.cursor().execute.call_args_list for expected, actual in zip(expected_execute_args, actual_execute_args): expeceted_sql, expected_params = expected[0] expeceted_sql = expeceted_sql.replace('\n', '') expeceted_sql = expeceted_sql.replace(' ', '') actual_sql, actual_params = actual[0] actual_sql = actual_sql.replace('\n', '') actual_sql = actual_sql.replace(' ', '') self.assertEqual(expeceted_sql, actual_sql) self.assertEqual(expected_params, actual_params)
def get_standard_config(self): config_manager = ConfigurationManager([self.required_config], app_name='ElasticSearchTestCase', app_description=__doc__, argv_source=[]) with config_manager.context() as config: return config
def get_standard_config(cls): config_manager = ConfigurationManager([cls.required_config], app_name='PostgreSQLTestCase', app_description=__doc__, argv_source=[]) with config_manager.context() as config: return config
def get_standard_config(self): config_manager = ConfigurationManager( [self.required_config], app_name="PostgreSQLTestCase", app_description=__doc__, argv_source=[] ) with config_manager.context() as config: return config
def test_success_after_limited_retry(self): mock_logging = mock.Mock() required_config = ElasticSearchCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'submission_url': 'http://elasticsearch_host/%s', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) urllib_str = 'socorro.external.elasticsearch.crashstorage.urllib2' m_request = mock.Mock() m_urlopen = mock.Mock() with mock.patch(urllib_str) as mocked_urllib: mocked_urllib.Request = m_request m_request.return_value = 17 mocked_urllib.urlopen = m_urlopen urlopen_results = [ urllib2.socket.timeout, urllib2.socket.timeout ] def urlopen_fn(*args, **kwargs): try: r = urlopen_results.pop(0) raise r except IndexError: return m_urlopen m_urlopen.side_effect = urlopen_fn es_storage.save_processed(a_processed_crash) expected_request_args = ( 'http://elasticsearch_host/9120408936ce666-ff3b-4c7a-9674-' '367fe2120408', {}, ) m_request.assert_called_with(*expected_request_args) self.assertEqual(m_urlopen.call_count, 3) expected_urlopen_args = (17, ) expected_urlopen_kwargs = {'timeout': 0} m_urlopen.assert_called_with(*expected_urlopen_args, **expected_urlopen_kwargs)
def test_indexing(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.exceptions.ElasticHttpNotFoundError = \ pyelasticsearch.exceptions.ElasticHttpNotFoundError pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }], argv_source=[] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) crash_report = a_processed_crash.copy() crash_report['date_processed'] = '2013-01-01 10:56:41.558922' def create_index_fn(index, **kwargs): assert 'socorro20130' in index if index == 'socorro201301': raise IndexAlreadyExistsError() mock_es.create_index.side_effect = create_index_fn # The index does not exist and is created es_storage.save_processed(crash_report) eq_(mock_es.create_index.call_count, 1) call_args = [ args for args, kwargs in mock_logging.info.call_args_list ] ok_( ('created new elasticsearch index: %s', 'socorro201300') in call_args ) # The index exists and is not created crash_report['date_processed'] = '2013-01-10 10:56:41.558922' es_storage.save_processed(crash_report) eq_(mock_es.create_index.call_count, 2) call_args = [ args for args, kwargs in mock_logging.info.call_args_list ] ok_( ('created new elasticsearch index: %s', 'socorro201301') not in call_args )
def test_failure_no_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }], argv_source=[] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) failure_exception = Exception('horrors') mock_es.index.side_effect = failure_exception crash_id = a_processed_crash['uuid'] self.assertRaises( Exception, es_storage.save_raw_and_processed, a_raw_crash, None, a_processed_crash.copy(), crash_id, ) expected_crash = { 'crash_id': crash_id, 'processed_crash': a_processed_crash.copy(), 'raw_crash': a_raw_crash } expected_request_args = ( 'socorro201214', 'crash_reports', expected_crash ) expected_request_kwargs = { 'replication': 'async', 'id': crash_id, } mock_es.index.assert_called_with( *expected_request_args, **expected_request_kwargs )
def test_basic_postgres_save_processed_success(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }] ) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) database = crashstorage.database.return_value = mock.MagicMock() self.assertTrue(isinstance(database, mock.Mock)) crashstorage.save_processed(a_processed_crash) fetch_all_returns = [((666,),), ((23,),), ] def fetch_all_func(*args): result = fetch_all_returns.pop(0) return result m = mock.MagicMock() m.__enter__.return_value = m database = crashstorage.database.return_value = m m.cursor.return_value.fetchall.side_effect=fetch_all_func crashstorage.save_processed(a_processed_crash) self.assertEqual(m.cursor.call_count, 6) self.assertEqual(m.cursor().fetchall.call_count, 2) self.assertEqual(m.cursor().execute.call_count, 6) expected_execute_args = ( (('savepoint MainThread', None),), (('insert into reports_20120402 (addons_checked, address, app_notes, build, client_crash_date, completed_datetime, cpu_info, cpu_name, date_processed, distributor, distributor_version, email, exploitability, flash_version, hangid, install_age, last_crash, os_name, os_version, processor_notes, process_type, product, reason, release_channel, signature, started_datetime, success, topmost_filenames, truncated, uptime, user_comments, user_id, url, uuid, version) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) returning id', [None, '0x1c', '...', '20120309050057', '2012-04-08 10:52:42.0', '2012-04-08 10:56:50.902884', 'None | 0', 'arm', '2012-04-08 10:56:41.558922', None, None, '*****@*****.**', 'high', '[blank]', None, 22385, None, 'Linux', '0.0.0 Linux 2.6.35.7-perf-CL727859 #1 ', 'SignatureTool: signature truncated due to length', 'plugin', 'FennecAndroid', 'SIGSEGV', 'default', 'libxul.so@0x117441c', '2012-04-08 10:56:50.440752', True, [], False, 170, None, None, 'http://embarasing.p**n.com', '936ce666-ff3b-4c7a-9674-367fe2120408', '13.0a1']),), (('release savepoint MainThread', None),), (('select id from plugins where filename = %s and name = %s', ('dwight.txt', 'wilma')),), (('insert into plugins_reports_20120402 (report_id, plugin_id, date_processed, version) values (%s, %s, %s, %s)', (666, 23, '2012-04-08 10:56:41.558922', '69')),), (('insert into extensions_20120402 (report_id, date_processed, extension_key, extension_id, extension_version)values (%s, %s, %s, %s, %s)', (666, '2012-04-08 10:56:41.558922', 0, '{1a5dabbd-0e74-41da-b532-a364bb552cab}', '1.0.4.1')),), (('release savepoint MainThread', None),), ) actual_execute_args = m.cursor().execute.call_args_list for expected, actual in zip(expected_execute_args, actual_execute_args): self.assertEqual(expected, actual)
def get_standard_config(self): config_manager = ConfigurationManager( [self.required_config], app_name='ElasticSearchTestCase', app_description=__doc__ ) with config_manager.context() as config: return config
def get_standard_config(self): config_manager = ConfigurationManager([ self.required_config, ], app_name='PostgreSQLTestCase', app_description=__doc__) with config_manager.context() as config: return config
def test_operation_error_with_postgres_with_backoff_with_rollback(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutorWithBackoff, #default=TransactionExecutor, doc='a class that will execute transactions' ) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{'database_class': MockConnectionContext, 'backoff_delays': [2, 4, 6, 10, 15]}], ) with config_manager.context() as config: executor = config.transaction_executor_class(config) _function_calls = [] # some mutable _sleep_count = [] def mock_function(connection): assert isinstance(connection, MockConnection) connection.transaction_status = \ psycopg2.extensions.TRANSACTION_STATUS_INTRANS _function_calls.append(connection) # the default sleep times are going to be, # 2, 4, 6, 10, 15 # so after 2 + 4 + 6 + 10 + 15 seconds # all will be exhausted if sum(_sleep_count) < sum([2, 4, 6, 10, 15]): raise psycopg2.OperationalError('Arh!') def mock_sleep(n): _sleep_count.append(n) # monkey patch the sleep function from inside transaction_executor _orig_sleep = socorro.database.transaction_executor.time.sleep socorro.database.transaction_executor.time.sleep = mock_sleep try: executor(mock_function) self.assertTrue(_function_calls) self.assertEqual(commit_count, 1) self.assertEqual(rollback_count, 5) self.assertTrue(mock_logging.warnings) self.assertEqual(len(mock_logging.warnings), 5) self.assertTrue(len(_sleep_count) > 10) finally: socorro.database.transaction_executor.time.sleep = _orig_sleep
def test_success_after_limited_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) esindex_results = [pyelasticsearch.exceptions.Timeout, pyelasticsearch.exceptions.Timeout] def esindex_fn(*args, **kwargs): try: r = esindex_results.pop(0) raise r except IndexError: return mock_es.index mock_es.index.side_effect = esindex_fn es_storage.save_processed(a_processed_crash) expected_request_args = ( 'socorro201214', 'crash_reports', a_processed_crash ) expected_request_kwargs = { 'replication': 'async', 'id': a_processed_crash['uuid'], } mock_es.index.assert_called_with( *expected_request_args, **expected_request_kwargs )
def test_operation_error_with_postgres_with_backoff(self): required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutorWithInfiniteBackoff, doc='a class that will execute transactions') required_config.add_option('database_class', default=MockConnectionContext, from_string_converter=class_converter) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'backoff_delays': [2, 4, 6, 10, 15] }], argv_source=[]) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class( config, mocked_context) _function_calls = [] # some mutable _sleep_count = [] def mock_function(connection): assert isinstance(connection, MockConnection) _function_calls.append(connection) # the default sleep times are going to be, # 2, 4, 6, 10, 15 # so after 2 + 4 + 6 + 10 + 15 seconds # all will be exhausted if sum(_sleep_count) < sum([2, 4, 6, 10, 15]): raise psycopg2.OperationalError('Arh!') def sleep_counter(n): _sleep_count.append(n) with mock.patch( 'socorro.lib.transaction.time.sleep') as mock_sleep: mock_sleep.side_effect = sleep_counter executor(mock_function) assert _function_calls assert commit_count == 1 assert rollback_count == 5 assert mock_logging.criticals assert len(mock_logging.criticals) == 5 assert len(_sleep_count) > 10
def test_success_after_limited_retry(self): mock_logging = mock.Mock() required_config = ElasticSearchCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'submission_url': 'http://elasticsearch_host/%s', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) urllib_str = 'socorro.external.elasticsearch.crashstorage.urllib2' m_request = mock.Mock() m_urlopen = mock.Mock() with mock.patch(urllib_str) as mocked_urllib: mocked_urllib.Request = m_request m_request.return_value = 17 mocked_urllib.urlopen = m_urlopen urlopen_results = [urllib2.socket.timeout, urllib2.socket.timeout] def urlopen_fn(*args, **kwargs): try: r = urlopen_results.pop(0) raise r except IndexError: return m_urlopen m_urlopen.side_effect = urlopen_fn es_storage.save_processed(a_processed_crash) expected_request_args = ( 'http://elasticsearch_host/9120408936ce666-ff3b-4c7a-9674-' '367fe2120408', {}, ) m_request.assert_called_with(*expected_request_args) self.assertEqual(m_urlopen.call_count, 3) expected_urlopen_args = (17,) expected_urlopen_kwargs = {'timeout': 0} m_urlopen.assert_called_with(*expected_urlopen_args, **expected_urlopen_kwargs)
def test_failure_no_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }], argv_source=[]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) failure_exception = Exception('horrors') mock_es.index.side_effect = failure_exception crash_id = a_processed_crash['uuid'] self.assertRaises( Exception, es_storage.save_raw_and_processed, a_raw_crash, None, a_processed_crash.copy(), crash_id, ) expected_crash = { 'crash_id': crash_id, 'processed_crash': a_processed_crash.copy(), 'raw_crash': a_raw_crash } expected_request_args = ('socorro201214', 'crash_reports', expected_crash) expected_request_kwargs = { 'replication': 'async', 'id': crash_id, } mock_es.index.assert_called_with(*expected_request_args, **expected_request_kwargs)
def get_standard_config(cls): config_manager = ConfigurationManager( [cls.required_config, ], app_name='PostgreSQLTestCase', app_description=__doc__, argv_source=[] ) with config_manager.context() as config: return config
def test_success_after_limited_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) esindex_results = [ pyelasticsearch.exceptions.Timeout, pyelasticsearch.exceptions.Timeout ] def esindex_fn(*args, **kwargs): try: r = esindex_results.pop(0) raise r except IndexError: return mock_es.index mock_es.index.side_effect = esindex_fn es_storage.save_processed(a_processed_crash) expected_request_args = ('socorro201214', 'crash_reports', a_processed_crash) expected_request_kwargs = { 'replication': 'async', 'id': a_processed_crash['uuid'], } mock_es.index.assert_called_with(*expected_request_args, **expected_request_kwargs)
def test_basic_postgres_save_raw_crash(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }]) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) database = crashstorage.database.return_value = mock.MagicMock() self.assertTrue(isinstance(database, mock.Mock)) self.assertTrue('submitted_timestamp' in a_raw_crash) m = mock.MagicMock() m.__enter__.return_value = m database = crashstorage.database.return_value = m crashstorage.save_raw_crash( a_raw_crash, '', "936ce666-ff3b-4c7a-9674-367fe2120408") self.assertEqual(m.cursor.call_count, 3) self.assertEqual(m.cursor().execute.call_count, 3) expected_execute_args = ( (('savepoint MainThread', None), ), (('insert into raw_crashes_20120402 (uuid, raw_crash, date_processed) values (%s, %s, %s)', ('936ce666-ff3b-4c7a-9674-367fe2120408', '{"submitted_timestamp": "2012-04-08 10:52:42.0", "Version": "6.02E23", "ProductName": "Fennicky"}', "2012-04-08 10:52:42.0")), ), (('release savepoint MainThread', None), ), ) actual_execute_args = m.cursor().execute.call_args_list for expected, actual in zip(expected_execute_args, actual_execute_args): expeceted_sql, expected_params = expected[0] expeceted_sql = expeceted_sql.replace('\n', '') expeceted_sql = expeceted_sql.replace(' ', '') actual_sql, actual_params = actual[0] actual_sql = actual_sql.replace('\n', '') actual_sql = actual_sql.replace(' ', '') self.assertEqual(expeceted_sql, actual_sql) self.assertEqual(expected_params, actual_params)
def test_abandon_backoff_transaction(self): """this is when a transaction is intentionally aborted, not because of an error, but because the client of the TransactionExcutor has determined that the transaction is of no further use. This test uses the TransactionExecutorWithInfiniteBackoff class instead of the base TransactionExcutor""" required_config = Namespace() required_config.add_option( 'transaction_executor_class', default=TransactionExecutorWithInfiniteBackoff, doc='a class that will execute transactions' ) required_config.add_option( 'database_class', default=MockConnectionContext, from_string_converter=class_converter ) mock_logging = MockLogging() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[], argv_source=[] ) with config_manager.context() as config: mocked_context = config.database_class(config) executor = config.transaction_executor_class( config, mocked_context ) class AbandonTransaction(Exception): abandon_transaction = True def mock_function(connection): assert isinstance(connection, MockConnection) connection.transaction_status = \ psycopg2.extensions.TRANSACTION_STATUS_INTRANS raise AbandonTransaction('crap!') # the method to test executor(mock_function) eq_(commit_count, 0) eq_(rollback_count, 1) ok_(not mock_logging.errors)
def test_checkin_done(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = ProcessorAppRegistrationClient.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[] ) m_registration = mock.Mock() class NoRegister(ProcessorAppRegistrationClient): _registration = m_registration with config_manager.context() as config: registrar = NoRegister(config) utc_now_mock_str = 'socorro.processor.registration_client.utc_now' with mock.patch(utc_now_mock_str) as m_utc_now: a_date = datetime(year=2012, month=5, day=4, hour=15, minute=10, tzinfo=UTC) m_utc_now.return_value = a_date m_database = mock.MagicMock() m_database.__enter__.return_value = m_database m_connection = m_database registrar.database.return_value = m_database registrar.processor_id = 17 m_cursor = mock.Mock() m_connection.cursor.return_value = m_cursor m_execute = mock.Mock() m_cursor.execute = m_execute registrar.checkin() m_execute.assert_called_once_with( "update processors set lastseendatetime = %s where id = %s", (a_date, 17) )
def test_hbase_crashstorage_success_after_retries(self): mock_logging = mock.Mock() required_config = HBaseCrashStorage.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'hbase_timeout': 100, 'hbase_host': commonconfig.hbaseHost.default, 'hbase_port': commonconfig.hbasePort.default, 'transaction_executor_class': TransactionExecutorWithLimitedBackoff, 'backoff_delays': [0, 0, 0] }] ) with config_manager.context() as config: hbaseclient_ = 'socorro.external.hbase.crashstorage.hbase_client' with mock.patch(hbaseclient_) as hclient: class SomeThriftError(Exception): pass klass = hclient.HBaseConnectionForCrashReports klass.return_value = klass klass.operational_exceptions = (SomeThriftError,) _attempts = [SomeThriftError, SomeThriftError] def retry_raiser_iterator(*args, **kwargs): try: raise _attempts.pop(0) except IndexError: return klass klass.put_json_dump.side_effect = retry_raiser_iterator crashstorage = HBaseCrashStorage(config) raw = ('{"name":"Peter", ' '"submitted_timestamp":"%d"}' % time.time()) crashstorage.save_raw_crash(json.loads(raw), raw, "abc123") self.assertEqual(klass.put_json_dump.call_count, 3)
def test_checkin_not_necessary(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = ProcessorAppRegistrationClient.required_config required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres }], argv_source=[] ) m_registration = mock.Mock() class NoRegister(ProcessorAppRegistrationClient): _registration = m_registration with config_manager.context() as config: registrar = NoRegister(config) utc_now_mock_str = 'socorro.processor.registration_client.utc_now' with mock.patch(utc_now_mock_str) as m_utc_now: a_date = datetime(year=1999, month=1, day=1, hour=0, minute=2, tzinfo=UTC) m_utc_now.return_value = a_date m_database = mock.MagicMock() m_database.__enter__.return_value = m_database m_connection = m_database registrar.database.return_value = m_database registrar.processor_id = 17 m_cursor = mock.Mock() m_connection.cursor.return_value = m_cursor m_execute = mock.Mock() m_cursor.execute = m_execute registrar.checkin() eq_(m_execute.call_count, 0)
def test_count(self, metricsmock): config_manager = ConfigurationManager( [MetricsCounter.get_required_config()], values_source_list=[{ 'metrics_prefix': 'phil', 'active_list': 'run', }], argv_source=[] ) with config_manager.context() as config: counter = MetricsCounter(config) with metricsmock as mm: counter.run() counter.walk() assert len(mm.get_records()) == 1 assert mm.has_record('incr', stat='phil.run', value=1)
def test_basic_postgres_save_processed_operational_error(self): mock_logging = mock.Mock() mock_postgres = mock.Mock() required_config = PostgreSQLCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'database_class': mock_postgres, 'transaction_executor_class': TransactionExecutorWithLimitedBackoff, 'backoff_delays': [0, 0, 0], }], argv_source=[] ) with config_manager.context() as config: crashstorage = PostgreSQLCrashStorage(config) crashstorage.database.operational_exceptions = (OperationalError,) database = crashstorage.database.return_value = mock.MagicMock() self.assertTrue(isinstance(database, mock.Mock)) fetch_all_returns = [((666,),), None, ((23,),), ] def fetch_all_func(*args): result = fetch_all_returns.pop(0) return result m = mock.MagicMock() m.__enter__.return_value = m database = crashstorage.database.return_value = m m.cursor.side_effect = OperationalError('bad') self.assertRaises(OperationalError, crashstorage.save_processed, a_processed_crash) self.assertEqual(m.cursor.call_count, 3)