def test_failure_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
'timeout': 0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
failure_exception = pyelasticsearch.exceptions.Timeout
mock_es.index.side_effect = failure_exception
crash_id = a_processed_crash['uuid']
assert_raises(
pyelasticsearch.exceptions.Timeout,
es_storage.save_raw_and_processed,
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = (
'socorro201214',
'crash_reports',
expected_crash
)
expected_request_kwargs = {
'id': crash_id,
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def test_success(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
es_storage.save_processed(a_processed_crash)
expected_request_args = ('socorro201214', 'crash_reports',
a_processed_crash)
expected_request_kwargs = {
'replication': 'async',
'id': a_processed_crash['uuid'],
}
mock_es.index.assert_called_with(*expected_request_args,
**expected_request_kwargs)
def test_failure_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
'timeout': 0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
failure_exception = pyelasticsearch.exceptions.Timeout
mock_es.index.side_effect = failure_exception
crash_id = a_processed_crash['uuid']
assert_raises(
pyelasticsearch.exceptions.Timeout,
es_storage.save_raw_and_processed,
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = (
'socorro201214',
'crash_reports',
expected_crash
)
expected_request_kwargs = {
'id': crash_id,
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def test_indexing(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.exceptions.ElasticHttpNotFoundError = \
pyelasticsearch.exceptions.ElasticHttpNotFoundError
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
crash_report = a_processed_crash.copy()
crash_report['date_processed'] = '2013-01-01 10:56:41.558922'
def create_index_fn(index, **kwargs):
assert 'socorro20130' in index
if index == 'socorro201301':
raise IndexAlreadyExistsError()
mock_es.create_index.side_effect = create_index_fn
# The index does not exist and is created
es_storage.save_processed(crash_report)
eq_(mock_es.create_index.call_count, 1)
call_args = [
args for args, kwargs in mock_logging.info.call_args_list
]
ok_(
('created new elasticsearch index: %s', 'socorro201300')
in call_args
)
# The index exists and is not created
crash_report['date_processed'] = '2013-01-10 10:56:41.558922'
es_storage.save_processed(crash_report)
eq_(mock_es.create_index.call_count, 2)
call_args = [
args for args, kwargs in mock_logging.info.call_args_list
]
ok_(
('created new elasticsearch index: %s', 'socorro201301')
not in call_args
)
def _setup_storage_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
return ConfigurationManager([storage_conf],
values_source_list=[os.environ],
argv_source=[])
def test_indexing(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.exceptions.ElasticHttpNotFoundError = \
pyelasticsearch.exceptions.ElasticHttpNotFoundError
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
crash_report = a_processed_crash.copy()
crash_report['date_processed'] = '2013-01-01 10:56:41.558922'
def create_index_fn(index, **kwargs):
assert 'socorro20130' in index
if index == 'socorro201301':
raise IndexAlreadyExistsError()
mock_es.create_index.side_effect = create_index_fn
# The index does not exist and is created
es_storage.save_processed(crash_report)
eq_(mock_es.create_index.call_count, 1)
call_args = [
args for args, kwargs in mock_logging.info.call_args_list
]
ok_(
('created new elasticsearch index: %s', 'socorro201300')
in call_args
)
# The index exists and is not created
crash_report['date_processed'] = '2013-01-10 10:56:41.558922'
es_storage.save_processed(crash_report)
eq_(mock_es.create_index.call_count, 2)
call_args = [
args for args, kwargs in mock_logging.info.call_args_list
]
ok_(
('created new elasticsearch index: %s', 'socorro201301')
not in call_args
)
def test_failure_no_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
failure_exception = Exception('horrors')
mock_es.index.side_effect = failure_exception
crash_id = a_processed_crash['uuid']
self.assertRaises(
Exception,
es_storage.save_raw_and_processed,
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = (
'socorro201214',
'crash_reports',
expected_crash
)
expected_request_kwargs = {
'replication': 'async',
'id': crash_id,
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def test_success_after_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
'timeout': 0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
esindex_results = [pyelasticsearch.exceptions.Timeout,
pyelasticsearch.exceptions.Timeout]
def esindex_fn(*args, **kwargs):
try:
r = esindex_results.pop(0)
raise r
except IndexError:
return mock_es.index
mock_es.index.side_effect = esindex_fn
es_storage.save_processed(a_processed_crash)
expected_request_args = (
'socorro201214',
'crash_reports',
a_processed_crash
)
expected_request_kwargs = {
'replication': 'async',
'id': a_processed_crash['uuid'],
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def _setup_storage_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
return ConfigurationManager(
[storage_conf],
values_source_list=[os.environ],
argv_source=[]
)
def test_success_after_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
'timeout':
0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
esindex_results = [
pyelasticsearch.exceptions.Timeout,
pyelasticsearch.exceptions.Timeout
]
def esindex_fn(*args, **kwargs):
try:
r = esindex_results.pop(0)
raise r
except IndexError:
return mock_es.index
mock_es.index.side_effect = esindex_fn
es_storage.save_processed(a_processed_crash)
expected_request_args = ('socorro201214', 'crash_reports',
a_processed_crash)
expected_request_kwargs = {
'replication': 'async',
'id': a_processed_crash['uuid'],
}
mock_es.index.assert_called_with(*expected_request_args,
**expected_request_kwargs)
def _setup_storage_config(self):
required_config = ElasticSearchCrashStorage.get_required_config()
overrides = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
more_definitions=required_config, overrides=overrides)
def _setup_storage_config(self):
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock.Mock())
return ConfigurationManager(
[storage_conf],
values_source_list=[{
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails'
}],
argv_source=[]
)
def test_failure_no_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
}],
argv_source=[])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
failure_exception = Exception('horrors')
mock_es.index.side_effect = failure_exception
crash_id = a_processed_crash['uuid']
self.assertRaises(
Exception,
es_storage.save_raw_and_processed,
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = ('socorro201214', 'crash_reports',
expected_crash)
expected_request_kwargs = {
'replication': 'async',
'id': crash_id,
}
mock_es.index.assert_called_with(*expected_request_args,
**expected_request_kwargs)
def _setup_storage_config(self):
required_config = ElasticSearchCrashStorage.get_required_config()
overrides = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
more_definitions=required_config,
overrides=overrides
)
def _setup_storage_config(self):
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock.Mock())
values_source_list = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return ConfigurationManager(
[storage_conf],
values_source_list=[os.environ, values_source_list],
argv_source=[])
def _setup_storage_config(self):
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock.Mock())
values_source_list = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return ConfigurationManager(
[storage_conf],
values_source_list=[os.environ, values_source_list],
argv_source=[]
)
def test_indexing(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.exceptions.ElasticHttpNotFoundError = \
pyelasticsearch.exceptions.ElasticHttpNotFoundError
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
crash_report = a_processed_crash.copy()
crash_report['date_processed'] = '2013-01-01 10:56:41.558922'
def status_fn(index):
assert 'socorro20130' in index
if index == 'socorro201300':
raise pyelasticsearch.exceptions.ElasticHttpNotFoundError()
mock_es.status = status_fn
# The index does not exist and is created
es_storage.save_processed(crash_report)
self.assertEqual(mock_es.create_index.call_count, 1)
# The index exists and is not created
crash_report['date_processed'] = '2013-01-10 10:56:41.558922'
es_storage.save_processed(crash_report)
self.assertEqual(mock_es.create_index.call_count, 1)
def test_indexing(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.exceptions.ElasticHttpNotFoundError = \
pyelasticsearch.exceptions.ElasticHttpNotFoundError
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
crash_report = a_processed_crash.copy()
crash_report['date_processed'] = '2013-01-01 10:56:41.558922'
def status_fn(index):
assert 'socorro20130' in index
if index == 'socorro201300':
raise pyelasticsearch.exceptions.ElasticHttpNotFoundError()
mock_es.status = status_fn
# The index does not exist and is created
es_storage.save_processed(crash_report)
self.assertEqual(mock_es.create_index.call_count, 1)
# The index exists and is not created
crash_report['date_processed'] = '2013-01-10 10:56:41.558922'
es_storage.save_processed(crash_report)
self.assertEqual(mock_es.create_index.call_count, 1)
def get_config_context(self):
storage_config = ElasticSearchCrashStorage.get_required_config()
storage_config.add_option('logger', default=self.config.logger)
values_source = {
'resource.elasticsearch.elasticsearch_default_index': 'socorro_integration_test',
'resource.elasticsearch.elasticsearch_index': 'socorro_integration_test_reports',
'resource.elasticsearch.backoff_delays': [1],
'resource.elasticsearch.elasticsearch_timeout': 5,
'resource.elasticsearch.use_mapping_file': False,
}
config_manager = ConfigurationManager(
[storage_config],
app_name='test_elasticsearch_indexing',
app_version='1.0',
app_description=__doc__,
values_source_list=[os.environ, values_source],
argv_source=[],
)
return config_manager.get_config()
def test_success(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
es_storage.save_processed(a_processed_crash)
expected_request_args = (
'socorro201214',
'crash_reports',
a_processed_crash
)
expected_request_kwargs = {
'replication': 'async',
'id': a_processed_crash['uuid'],
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
