def test_failure_limited_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }], argv_source=[] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) failure_exception = pyelasticsearch.exceptions.Timeout mock_es.index.side_effect = failure_exception crash_id = a_processed_crash['uuid'] assert_raises( pyelasticsearch.exceptions.Timeout, es_storage.save_raw_and_processed, a_raw_crash, None, a_processed_crash.copy(), crash_id, ) expected_crash = { 'crash_id': crash_id, 'processed_crash': a_processed_crash.copy(), 'raw_crash': a_raw_crash } expected_request_args = ( 'socorro201214', 'crash_reports', expected_crash ) expected_request_kwargs = { 'id': crash_id, } mock_es.index.assert_called_with( *expected_request_args, **expected_request_kwargs )
def test_success(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) es_storage.save_processed(a_processed_crash) expected_request_args = ('socorro201214', 'crash_reports', a_processed_crash) expected_request_kwargs = { 'replication': 'async', 'id': a_processed_crash['uuid'], } mock_es.index.assert_called_with(*expected_request_args, **expected_request_kwargs)
def test_indexing(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.exceptions.ElasticHttpNotFoundError = \ pyelasticsearch.exceptions.ElasticHttpNotFoundError pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }], argv_source=[] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) crash_report = a_processed_crash.copy() crash_report['date_processed'] = '2013-01-01 10:56:41.558922' def create_index_fn(index, **kwargs): assert 'socorro20130' in index if index == 'socorro201301': raise IndexAlreadyExistsError() mock_es.create_index.side_effect = create_index_fn # The index does not exist and is created es_storage.save_processed(crash_report) eq_(mock_es.create_index.call_count, 1) call_args = [ args for args, kwargs in mock_logging.info.call_args_list ] ok_( ('created new elasticsearch index: %s', 'socorro201300') in call_args ) # The index exists and is not created crash_report['date_processed'] = '2013-01-10 10:56:41.558922' es_storage.save_processed(crash_report) eq_(mock_es.create_index.call_count, 2) call_args = [ args for args, kwargs in mock_logging.info.call_args_list ] ok_( ('created new elasticsearch index: %s', 'socorro201301') not in call_args )
def _setup_storage_config(self): mock_logging = mock.Mock() storage_conf = ElasticSearchCrashStorage.get_required_config() storage_conf.add_option('logger', default=mock_logging) return ConfigurationManager([storage_conf], values_source_list=[os.environ], argv_source=[])
def test_failure_no_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }], argv_source=[] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) failure_exception = Exception('horrors') mock_es.index.side_effect = failure_exception crash_id = a_processed_crash['uuid'] self.assertRaises( Exception, es_storage.save_raw_and_processed, a_raw_crash, None, a_processed_crash.copy(), crash_id, ) expected_crash = { 'crash_id': crash_id, 'processed_crash': a_processed_crash.copy(), 'raw_crash': a_raw_crash } expected_request_args = ( 'socorro201214', 'crash_reports', expected_crash ) expected_request_kwargs = { 'replication': 'async', 'id': crash_id, } mock_es.index.assert_called_with( *expected_request_args, **expected_request_kwargs )
def test_success_after_limited_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) esindex_results = [pyelasticsearch.exceptions.Timeout, pyelasticsearch.exceptions.Timeout] def esindex_fn(*args, **kwargs): try: r = esindex_results.pop(0) raise r except IndexError: return mock_es.index mock_es.index.side_effect = esindex_fn es_storage.save_processed(a_processed_crash) expected_request_args = ( 'socorro201214', 'crash_reports', a_processed_crash ) expected_request_kwargs = { 'replication': 'async', 'id': a_processed_crash['uuid'], } mock_es.index.assert_called_with( *expected_request_args, **expected_request_kwargs )
def _setup_storage_config(self): mock_logging = mock.Mock() storage_conf = ElasticSearchCrashStorage.get_required_config() storage_conf.add_option('logger', default=mock_logging) return ConfigurationManager( [storage_conf], values_source_list=[os.environ], argv_source=[] )
def test_success_after_limited_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', 'timeout': 0, 'backoff_delays': [0, 0, 0], 'transaction_executor_class': TransactionExecutorWithLimitedBackoff }]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) esindex_results = [ pyelasticsearch.exceptions.Timeout, pyelasticsearch.exceptions.Timeout ] def esindex_fn(*args, **kwargs): try: r = esindex_results.pop(0) raise r except IndexError: return mock_es.index mock_es.index.side_effect = esindex_fn es_storage.save_processed(a_processed_crash) expected_request_args = ('socorro201214', 'crash_reports', a_processed_crash) expected_request_kwargs = { 'replication': 'async', 'id': a_processed_crash['uuid'], } mock_es.index.assert_called_with(*expected_request_args, **expected_request_kwargs)
def _setup_storage_config(self): required_config = ElasticSearchCrashStorage.get_required_config() overrides = { 'elasticsearch_index': 'socorro_integration_test', 'elasticsearch_emails_index': 'socorro_integration_test_emails', 'elasticsearch_timeout': 5, 'backoff_delays': [1], } return get_config_manager_for_crontabber( more_definitions=required_config, overrides=overrides)
def _setup_storage_config(self): storage_conf = ElasticSearchCrashStorage.get_required_config() storage_conf.add_option('logger', default=mock.Mock()) return ConfigurationManager( [storage_conf], values_source_list=[{ 'elasticsearch_index': 'socorro_integration_test', 'elasticsearch_emails_index': 'socorro_integration_test_emails' }], argv_source=[] )
def test_failure_no_retry(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }], argv_source=[]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) failure_exception = Exception('horrors') mock_es.index.side_effect = failure_exception crash_id = a_processed_crash['uuid'] self.assertRaises( Exception, es_storage.save_raw_and_processed, a_raw_crash, None, a_processed_crash.copy(), crash_id, ) expected_crash = { 'crash_id': crash_id, 'processed_crash': a_processed_crash.copy(), 'raw_crash': a_raw_crash } expected_request_args = ('socorro201214', 'crash_reports', expected_crash) expected_request_kwargs = { 'replication': 'async', 'id': crash_id, } mock_es.index.assert_called_with(*expected_request_args, **expected_request_kwargs)
def _setup_storage_config(self): required_config = ElasticSearchCrashStorage.get_required_config() overrides = { 'elasticsearch_index': 'socorro_integration_test', 'elasticsearch_emails_index': 'socorro_integration_test_emails', 'elasticsearch_timeout': 5, 'backoff_delays': [1], } return get_config_manager_for_crontabber( more_definitions=required_config, overrides=overrides )
def _setup_storage_config(self): storage_conf = ElasticSearchCrashStorage.get_required_config() storage_conf.add_option('logger', default=mock.Mock()) values_source_list = { 'elasticsearch_index': 'socorro_integration_test', 'elasticsearch_emails_index': 'socorro_integration_test_emails', 'elasticsearch_timeout': 5, 'backoff_delays': [1], } return ConfigurationManager( [storage_conf], values_source_list=[os.environ, values_source_list], argv_source=[])
def _setup_storage_config(self): storage_conf = ElasticSearchCrashStorage.get_required_config() storage_conf.add_option('logger', default=mock.Mock()) values_source_list = { 'elasticsearch_index': 'socorro_integration_test', 'elasticsearch_emails_index': 'socorro_integration_test_emails', 'elasticsearch_timeout': 5, 'backoff_delays': [1], } return ConfigurationManager( [storage_conf], values_source_list=[os.environ, values_source_list], argv_source=[] )
def test_indexing(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.exceptions.ElasticHttpNotFoundError = \ pyelasticsearch.exceptions.ElasticHttpNotFoundError pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }]) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) crash_report = a_processed_crash.copy() crash_report['date_processed'] = '2013-01-01 10:56:41.558922' def status_fn(index): assert 'socorro20130' in index if index == 'socorro201300': raise pyelasticsearch.exceptions.ElasticHttpNotFoundError() mock_es.status = status_fn # The index does not exist and is created es_storage.save_processed(crash_report) self.assertEqual(mock_es.create_index.call_count, 1) # The index exists and is not created crash_report['date_processed'] = '2013-01-10 10:56:41.558922' es_storage.save_processed(crash_report) self.assertEqual(mock_es.create_index.call_count, 1)
def test_indexing(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.exceptions.ElasticHttpNotFoundError = \ pyelasticsearch.exceptions.ElasticHttpNotFoundError pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) crash_report = a_processed_crash.copy() crash_report['date_processed'] = '2013-01-01 10:56:41.558922' def status_fn(index): assert 'socorro20130' in index if index == 'socorro201300': raise pyelasticsearch.exceptions.ElasticHttpNotFoundError() mock_es.status = status_fn # The index does not exist and is created es_storage.save_processed(crash_report) self.assertEqual(mock_es.create_index.call_count, 1) # The index exists and is not created crash_report['date_processed'] = '2013-01-10 10:56:41.558922' es_storage.save_processed(crash_report) self.assertEqual(mock_es.create_index.call_count, 1)
def get_config_context(self): storage_config = ElasticSearchCrashStorage.get_required_config() storage_config.add_option('logger', default=self.config.logger) values_source = { 'resource.elasticsearch.elasticsearch_default_index': 'socorro_integration_test', 'resource.elasticsearch.elasticsearch_index': 'socorro_integration_test_reports', 'resource.elasticsearch.backoff_delays': [1], 'resource.elasticsearch.elasticsearch_timeout': 5, 'resource.elasticsearch.use_mapping_file': False, } config_manager = ConfigurationManager( [storage_config], app_name='test_elasticsearch_indexing', app_version='1.0', app_description=__doc__, values_source_list=[os.environ, values_source], argv_source=[], ) return config_manager.get_config()
def test_success(self, pyes_mock): mock_logging = mock.Mock() mock_es = mock.Mock() pyes_mock.ElasticSearch.return_value = mock_es required_config = ElasticSearchCrashStorage.get_required_config() required_config.add_option('logger', default=mock_logging) config_manager = ConfigurationManager( [required_config], app_name='testapp', app_version='1.0', app_description='app description', values_source_list=[{ 'logger': mock_logging, 'elasticsearch_urls': 'http://elasticsearch_host:9200', }] ) with config_manager.context() as config: es_storage = ElasticSearchCrashStorage(config) es_storage.save_processed(a_processed_crash) expected_request_args = ( 'socorro201214', 'crash_reports', a_processed_crash ) expected_request_kwargs = { 'replication': 'async', 'id': a_processed_crash['uuid'], } mock_es.index.assert_called_with( *expected_request_args, **expected_request_kwargs )