def __init__(self, *args, **kwargs):
super(IntegrationTestElasticsearchCleanup,
self).__init__(*args, **kwargs)
storage_config = self._setup_storage_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def test_sending_many_emails(self, exacttarget_mock):
"""Test that we can send emails to a lot of users in the same run. """
# First add a lot of emails.
now = utc_now() - datetime.timedelta(minutes=30)
config_manager = self._setup_storage_config()
with config_manager.context() as config:
storage = ElasticSearchCrashStorage(config)
for i in range(21):
storage.save_processed({
'uuid': 'fake-%s' % i,
'email': '*****@*****.**' % i,
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
})
storage.es.refresh()
config_manager = self._setup_simple_config()
with config_manager.context() as config:
job = automatic_emails.AutomaticEmailsCronApp(config, '')
job.run(utc_now())
et_mock = exacttarget_mock.return_value
# Verify that we have the default 4 results + the 21 we added.
self.assertEqual(et_mock.trigger_send.call_count, 25)
def test_failure_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
'timeout': 0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
failure_exception = pyelasticsearch.exceptions.Timeout
mock_es.index.side_effect = failure_exception
crash_id = a_processed_crash['uuid']
assert_raises(
pyelasticsearch.exceptions.Timeout,
es_storage.save_raw_and_processed,
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = (
'socorro201214',
'crash_reports',
expected_crash
)
expected_request_kwargs = {
'id': crash_id,
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def test_sending_many_emails(self, exacttarget_mock):
"""Test that we can send emails to a lot of users in the same run. """
# First add a lot of emails.
now = utc_now() - datetime.timedelta(minutes=30)
config_manager = self._setup_storage_config()
with config_manager.context() as config:
storage = ElasticSearchCrashStorage(config)
for i in range(21):
storage.save_processed({
'uuid': 'fake-%s' % i,
'email': '*****@*****.**' % i,
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
})
storage.es.refresh()
config_manager = self._setup_simple_config()
with config_manager.context() as config:
job = automatic_emails.AutomaticEmailsCronApp(config, '')
job.run(utc_now())
et_mock = exacttarget_mock.return_value
# Verify that we have the default 4 results + the 21 we added.
eq_(et_mock.trigger_send.call_count, 25)
def test_success(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
es_storage.save_processed(a_processed_crash)
expected_request_args = ('socorro201214', 'crash_reports',
a_processed_crash)
expected_request_kwargs = {
'replication': 'async',
'id': a_processed_crash['uuid'],
}
mock_es.index.assert_called_with(*expected_request_args,
**expected_request_kwargs)
def test_success_after_limited_retry(self):
mock_logging = mock.Mock()
required_config = ElasticSearchCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'submission_url':
'http://elasticsearch_host/%s',
'timeout':
0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
urllib_str = 'socorro.external.elasticsearch.crashstorage.urllib2'
m_request = mock.Mock()
m_urlopen = mock.Mock()
with mock.patch(urllib_str) as mocked_urllib:
mocked_urllib.Request = m_request
m_request.return_value = 17
mocked_urllib.urlopen = m_urlopen
urlopen_results = [
urllib2.socket.timeout, urllib2.socket.timeout
]
def urlopen_fn(*args, **kwargs):
try:
r = urlopen_results.pop(0)
raise r
except IndexError:
return m_urlopen
m_urlopen.side_effect = urlopen_fn
es_storage.save_processed(a_processed_crash)
expected_request_args = (
'http://elasticsearch_host/9120408936ce666-ff3b-4c7a-9674-'
'367fe2120408',
{},
)
m_request.assert_called_with(*expected_request_args)
self.assertEqual(m_urlopen.call_count, 3)
expected_urlopen_args = (17, )
expected_urlopen_kwargs = {'timeout': 0}
m_urlopen.assert_called_with(*expected_urlopen_args,
**expected_urlopen_kwargs)
def test_success_after_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
'timeout': 0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
esindex_results = [pyelasticsearch.exceptions.Timeout,
pyelasticsearch.exceptions.Timeout]
def esindex_fn(*args, **kwargs):
try:
r = esindex_results.pop(0)
raise r
except IndexError:
return mock_es.index
mock_es.index.side_effect = esindex_fn
es_storage.save_processed(a_processed_crash)
expected_request_args = (
'socorro201214',
'crash_reports',
a_processed_crash
)
expected_request_kwargs = {
'replication': 'async',
'id': a_processed_crash['uuid'],
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def test_success_after_limited_retry(self):
mock_logging = mock.Mock()
required_config = ElasticSearchCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'submission_url': 'http://elasticsearch_host/%s',
'timeout': 0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class': TransactionExecutorWithLimitedBackoff
}]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
urllib_str = 'socorro.external.elasticsearch.crashstorage.urllib2'
m_request = mock.Mock()
m_urlopen = mock.Mock()
with mock.patch(urllib_str) as mocked_urllib:
mocked_urllib.Request = m_request
m_request.return_value = 17
mocked_urllib.urlopen = m_urlopen
urlopen_results = [urllib2.socket.timeout,
urllib2.socket.timeout]
def urlopen_fn(*args, **kwargs):
try:
r = urlopen_results.pop(0)
raise r
except IndexError:
return m_urlopen
m_urlopen.side_effect = urlopen_fn
es_storage.save_processed(a_processed_crash)
expected_request_args = (
'http://elasticsearch_host/9120408936ce666-ff3b-4c7a-9674-'
'367fe2120408',
{},
)
m_request.assert_called_with(*expected_request_args)
self.assertEqual(m_urlopen.call_count, 3)
expected_urlopen_args = (17,)
expected_urlopen_kwargs = {'timeout': 0}
m_urlopen.assert_called_with(*expected_urlopen_args,
**expected_urlopen_kwargs)
def test_failure_no_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
}],
argv_source=[])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
failure_exception = Exception('horrors')
mock_es.index.side_effect = failure_exception
crash_id = a_processed_crash['uuid']
self.assertRaises(
Exception,
es_storage.save_raw_and_processed,
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = ('socorro201214', 'crash_reports',
expected_crash)
expected_request_kwargs = {
'replication': 'async',
'id': crash_id,
}
mock_es.index.assert_called_with(*expected_request_args,
**expected_request_kwargs)
def test_success_after_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
'timeout':
0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
esindex_results = [
pyelasticsearch.exceptions.Timeout,
pyelasticsearch.exceptions.Timeout
]
def esindex_fn(*args, **kwargs):
try:
r = esindex_results.pop(0)
raise r
except IndexError:
return mock_es.index
mock_es.index.side_effect = esindex_fn
es_storage.save_processed(a_processed_crash)
expected_request_args = ('socorro201214', 'crash_reports',
a_processed_crash)
expected_request_kwargs = {
'replication': 'async',
'id': a_processed_crash['uuid'],
}
mock_es.index.assert_called_with(*expected_request_args,
**expected_request_kwargs)
def test_success(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}],
argv_source=[]
)
with config_manager.context() as config:
crash_id = a_processed_crash['uuid']
es_storage = ElasticSearchCrashStorage(config)
es_storage.save_raw_and_processed(
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = (
'socorro201214',
'crash_reports',
expected_crash
)
expected_request_kwargs = {
'id': crash_id,
'replication': 'async',
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def test_failure_limited_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
'timeout': 0,
'backoff_delays': [0, 0, 0],
'transaction_executor_class':
TransactionExecutorWithLimitedBackoff
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
failure_exception = pyelasticsearch.exceptions.Timeout
mock_es.index.side_effect = failure_exception
crash_id = a_processed_crash['uuid']
assert_raises(
pyelasticsearch.exceptions.Timeout,
es_storage.save_raw_and_processed,
a_raw_crash,
None,
a_processed_crash.copy(),
crash_id,
)
expected_crash = {
'crash_id': crash_id,
'processed_crash': a_processed_crash.copy(),
'raw_crash': a_raw_crash
}
expected_request_args = (
'socorro201214',
'crash_reports',
expected_crash
)
expected_request_kwargs = {
'id': crash_id,
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def __init__(self, *args, **kwargs):
super(
IntegrationTestIndexCleaner,
self
).__init__(*args, **kwargs)
storage_config = self._setup_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def tearDown(self):
config_manager = self._setup_storage_config()
with config_manager.context() as config:
storage = ElasticSearchCrashStorage(config)
storage.es.delete_index(config.elasticsearch_index)
storage.es.delete_index(config.elasticsearch_emails_index)
storage.es.flush()
super(IntegrationTestAutomaticEmails, self).tearDown()
def _setup_storage_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
return ConfigurationManager([storage_conf],
values_source_list=[os.environ],
argv_source=[])
def test_indexing(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.exceptions.ElasticHttpNotFoundError = \
pyelasticsearch.exceptions.ElasticHttpNotFoundError
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
crash_report = a_processed_crash.copy()
crash_report['date_processed'] = '2013-01-01 10:56:41.558922'
def create_index_fn(index, **kwargs):
assert 'socorro20130' in index
if index == 'socorro201301':
raise IndexAlreadyExistsError()
mock_es.create_index.side_effect = create_index_fn
# The index does not exist and is created
es_storage.save_processed(crash_report)
eq_(mock_es.create_index.call_count, 1)
call_args = [
args for args, kwargs in mock_logging.info.call_args_list
]
ok_(
('created new elasticsearch index: %s', 'socorro201300')
in call_args
)
# The index exists and is not created
crash_report['date_processed'] = '2013-01-10 10:56:41.558922'
es_storage.save_processed(crash_report)
eq_(mock_es.create_index.call_count, 2)
call_args = [
args for args, kwargs in mock_logging.info.call_args_list
]
ok_(
('created new elasticsearch index: %s', 'socorro201301')
not in call_args
)
def test_indexing(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.exceptions.ElasticHttpNotFoundError = \
pyelasticsearch.exceptions.ElasticHttpNotFoundError
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
crash_report = a_processed_crash.copy()
crash_report['date_processed'] = '2013-01-01 10:56:41.558922'
def status_fn(index):
assert 'socorro20130' in index
if index == 'socorro201300':
raise pyelasticsearch.exceptions.ElasticHttpNotFoundError()
mock_es.status = status_fn
# The index does not exist and is created
es_storage.save_processed(crash_report)
self.assertEqual(mock_es.create_index.call_count, 1)
# The index exists and is not created
crash_report['date_processed'] = '2013-01-10 10:56:41.558922'
es_storage.save_processed(crash_report)
self.assertEqual(mock_es.create_index.call_count, 1)
def _setup_storage_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
return ConfigurationManager(
[storage_conf],
values_source_list=[os.environ],
argv_source=[]
)
def _setup_storage_config(self):
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock.Mock())
return ConfigurationManager(
[storage_conf],
values_source_list=[{
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails'
}],
argv_source=[]
)
def _setup_storage_config(self):
required_config = ElasticSearchCrashStorage.get_required_config()
overrides = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
more_definitions=required_config, overrides=overrides)
def test_success(self):
mock_logging = mock.Mock()
required_config = ElasticSearchCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'submission_url':
'http://elasticsearch_host/%s'
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
urllib_str = 'socorro.external.elasticsearch.crashstorage.urllib2'
m_request = mock.Mock()
m_urlopen = mock.Mock()
with mock.patch(urllib_str) as mocked_urllib:
mocked_urllib.Request = m_request
m_request.return_value = 17
mocked_urllib.urlopen = m_urlopen
es_storage.save_processed(a_processed_crash)
expected_request_args = (
'http://elasticsearch_host/9120408936ce666-ff3b-4c7a-9674-'
'367fe2120408',
{},
)
m_request.assert_called_with(*expected_request_args)
expected_urlopen_args = (17, )
expected_urlopen_kwargs = {'timeout': 2}
m_urlopen.assert_called_with(*expected_urlopen_args,
**expected_urlopen_kwargs)
def test_success(self):
mock_logging = mock.Mock()
required_config = ElasticSearchCrashStorage.required_config
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'submission_url': 'http://elasticsearch_host/%s'
}]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
urllib_str = 'socorro.external.elasticsearch.crashstorage.urllib2'
m_request = mock.Mock()
m_urlopen = mock.Mock()
with mock.patch(urllib_str) as mocked_urllib:
mocked_urllib.Request = m_request
m_request.return_value = 17
mocked_urllib.urlopen = m_urlopen
es_storage.save_processed(a_processed_crash)
expected_request_args = (
'http://elasticsearch_host/9120408936ce666-ff3b-4c7a-9674-'
'367fe2120408',
{},
)
m_request.assert_called_with(*expected_request_args)
expected_urlopen_args = (17,)
expected_urlopen_kwargs = {'timeout': 2}
m_urlopen.assert_called_with(*expected_urlopen_args,
**expected_urlopen_kwargs)
def _setup_storage_config(self):
required_config = ElasticSearchCrashStorage.get_required_config()
overrides = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
more_definitions=required_config,
overrides=overrides
)
def test_indexing(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.exceptions.ElasticHttpNotFoundError = \
pyelasticsearch.exceptions.ElasticHttpNotFoundError
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}],
argv_source=[]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
crash_report = a_processed_crash.copy()
crash_report['date_processed'] = '2013-01-01 10:56:41.558922'
def create_index_fn(index, **kwargs):
assert 'socorro20130' in index
if index == 'socorro201301':
raise IndexAlreadyExistsError()
mock_es.create_index.side_effect = create_index_fn
# The index does not exist and is created
es_storage.save_processed(crash_report)
eq_(mock_es.create_index.call_count, 1)
call_args = [
args for args, kwargs in mock_logging.info.call_args_list
]
ok_(
('created new elasticsearch index: %s', 'socorro201300')
in call_args
)
# The index exists and is not created
crash_report['date_processed'] = '2013-01-10 10:56:41.558922'
es_storage.save_processed(crash_report)
eq_(mock_es.create_index.call_count, 2)
call_args = [
args for args, kwargs in mock_logging.info.call_args_list
]
ok_(
('created new elasticsearch index: %s', 'socorro201301')
not in call_args
)
def _setup_storage_config(self):
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock.Mock())
values_source_list = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return ConfigurationManager(
[storage_conf],
values_source_list=[os.environ, values_source_list],
argv_source=[])
def _setup_storage_config(self):
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock.Mock())
values_source_list = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return ConfigurationManager(
[storage_conf],
values_source_list=[os.environ, values_source_list],
argv_source=[]
)
def test_failure_no_retry(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger': mock_logging,
'elasticsearch_urls': 'http://elasticsearch_host:9200',
}]
)
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
failure_exception = Exception('horrors')
mock_es.index.side_effect = failure_exception
self.assertRaises(Exception,
es_storage.save_processed,
a_processed_crash)
expected_request_args = (
'socorro201214',
'crash_reports',
a_processed_crash
)
expected_request_kwargs = {
'replication': 'async',
'id': a_processed_crash['uuid'],
}
mock_es.index.assert_called_with(
*expected_request_args,
**expected_request_kwargs
)
def test_indexing(self, pyes_mock):
mock_logging = mock.Mock()
mock_es = mock.Mock()
pyes_mock.exceptions.ElasticHttpNotFoundError = \
pyelasticsearch.exceptions.ElasticHttpNotFoundError
pyes_mock.ElasticSearch.return_value = mock_es
required_config = ElasticSearchCrashStorage.get_required_config()
required_config.add_option('logger', default=mock_logging)
config_manager = ConfigurationManager(
[required_config],
app_name='testapp',
app_version='1.0',
app_description='app description',
values_source_list=[{
'logger':
mock_logging,
'elasticsearch_urls':
'http://elasticsearch_host:9200',
}])
with config_manager.context() as config:
es_storage = ElasticSearchCrashStorage(config)
crash_report = a_processed_crash.copy()
crash_report['date_processed'] = '2013-01-01 10:56:41.558922'
def status_fn(index):
assert 'socorro20130' in index
if index == 'socorro201300':
raise pyelasticsearch.exceptions.ElasticHttpNotFoundError()
mock_es.status = status_fn
# The index does not exist and is created
es_storage.save_processed(crash_report)
self.assertEqual(mock_es.create_index.call_count, 1)
# The index exists and is not created
crash_report['date_processed'] = '2013-01-10 10:56:41.558922'
es_storage.save_processed(crash_report)
self.assertEqual(mock_es.create_index.call_count, 1)
def get_config_context(self):
storage_config = ElasticSearchCrashStorage.get_required_config()
storage_config.add_option('logger', default=self.config.logger)
values_source = {
'resource.elasticsearch.elasticsearch_default_index': 'socorro_integration_test',
'resource.elasticsearch.elasticsearch_index': 'socorro_integration_test_reports',
'resource.elasticsearch.backoff_delays': [1],
'resource.elasticsearch.elasticsearch_timeout': 5,
'resource.elasticsearch.use_mapping_file': False,
}
config_manager = ConfigurationManager(
[storage_config],
app_name='test_elasticsearch_indexing',
app_version='1.0',
app_description=__doc__,
values_source_list=[os.environ, values_source],
argv_source=[],
)
return config_manager.get_config()
def setUp(self):
super(IntegrationTestAutomaticEmails, self).setUp()
# prep a fake table
now = utc_now() - datetime.timedelta(minutes=30)
last_month = now - datetime.timedelta(days=31)
config_manager = self._setup_storage_config()
with config_manager.context() as config:
storage = ElasticSearchCrashStorage(config)
# clear the indices cache so the index is created on every test
storage.indices_cache = set()
storage.save_processed({
'uuid': '1',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': 'unknown'
}
}
})
storage.save_processed({
'uuid': '2',
'email': '"Quidam" <*****@*****.**>',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': None
}
}
})
storage.save_processed({
'uuid': '3',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': 'bitguard'
}
}
})
storage.save_processed({
'uuid': '4',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '5',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '6',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '7',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '8',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '9',
'email': '*****@*****.**',
'product': 'EarthRaccoon',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '18',
'email': 'z\xc3\[email protected]',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Let's insert a duplicate
storage.save_processed({
'uuid': '10',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# And let's insert some invalid crashes
storage.save_processed({
'uuid': '11',
'email': None,
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '12',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': last_month
})
storage.save_processed({
'uuid': '13',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '14',
'email': '*****@*****.**',
'product': 'WindBear',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Finally some invalid email addresses
storage.save_processed({
'uuid': '15',
'email': ' ',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '16',
'email': 'invalid@email',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '17',
'email': 'i.do.not.work',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Create some email addresses.
storage.create_emails_index()
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '*****@*****.**',
'last_sending': last_month
},
id='*****@*****.**',
)
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '"Quidam" <*****@*****.**>',
'last_sending': last_month
},
id='"Quidam" <*****@*****.**>',
)
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '*****@*****.**',
'last_sending': now
},
id='*****@*****.**',
)
# As indexing is asynchronous, we need to force elasticsearch to
# make the newly created content searchable before we run the
# tests.
storage.es.refresh()
def test_email_after_delay(self, exacttarget_mock):
"""Test that a user will receive an email if he or she sends us a new
crash report after the delay is passed (but not before). """
config_manager = self._setup_config_manager(
delay_between_emails=1, restrict_products=['EarthRaccoon'])
email = '*****@*****.**'
list_service_mock = exacttarget_mock.return_value.list.return_value
list_service_mock.get_subscriber.return_value = {'token': email}
trigger_send_mock = exacttarget_mock.return_value.trigger_send
tomorrow = utc_now() + datetime.timedelta(days=1, hours=2)
twohourslater = utc_now() + datetime.timedelta(hours=2)
storage_config_manager = self._setup_storage_config()
with storage_config_manager.context() as storage_config:
storage = ElasticSearchCrashStorage(storage_config)
with config_manager.context() as config:
# 1. Send an email to the user and update emailing data
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
exacttarget_mock.return_value.trigger_send.assert_called_with(
'socorro_dev_test', {
'EMAIL_ADDRESS_': email,
'EMAIL_FORMAT_': 'H',
'TOKEN': email
})
eq_(trigger_send_mock.call_count, 1)
# 2. Test that before 'delay' is passed user doesn't receive
# another email
# Insert a new crash report with the same email address
storage.save_processed({
'uuid':
'50',
'email':
email,
'product':
'EarthRaccoon',
'version':
'20.0',
'release_channel':
'Release',
'date_processed':
utc_now() + datetime.timedelta(hours=1)
})
storage.es.refresh()
# Run crontabber with time pushed by two hours
with mock.patch('crontabber.app.utc_now') as cronutc_mock:
with mock.patch('crontabber.base.utc_now') as baseutc_mock:
cronutc_mock.return_value = twohourslater
baseutc_mock.return_value = twohourslater
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# No new email was sent
eq_(trigger_send_mock.call_count, 1)
# 3. Verify that, after 'delay' is passed, a new email is sent
# to our user
# Insert a new crash report with the same email address
storage.save_processed({
'uuid':
'51',
'email':
email,
'product':
'EarthRaccoon',
'version':
'20.0',
'release_channel':
'Release',
'date_processed':
utc_now() + datetime.timedelta(days=1)
})
storage.es.refresh()
# Run crontabber with time pushed by a day
with mock.patch('crontabber.app.utc_now') as cronutc_mock:
with mock.patch('crontabber.base.utc_now') as baseutc_mock:
cronutc_mock.return_value = tomorrow
baseutc_mock.return_value = tomorrow
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# A new email was sent
eq_(trigger_send_mock.call_count, 2)
class IntegrationTestIndexCleaner(ElasticSearchTestCase):
def __init__(self, *args, **kwargs):
super(
IntegrationTestIndexCleaner,
self
).__init__(*args, **kwargs)
storage_config = self._setup_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def setUp(self):
self.indices = []
def tearDown(self):
# Clean up created indices.
for index in self.indices:
try:
self.storage.es.delete_index(index)
# "Missing" indices have already been deleted, no need to worry.
except pyelasticsearch.exceptions.ElasticHttpNotFoundError:
pass
super(IntegrationTestIndexCleaner, self).tearDown()
def _setup_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
cleaner_conf = IndexCleaner.get_required_config()
cleaner_conf.add_option('logger', default=mock_logging)
return ConfigurationManager(
[storage_conf, cleaner_conf],
values_source_list=[environment],
argv_source=[]
)
@maximum_es_version('0.90')
def test_correct_indices_are_deleted(self):
config_manager = self._setup_config()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create old indices to be deleted.
self.storage.create_index('socorro200142', {})
self.indices.append('socorro200142')
self.storage.create_index('socorro200000', {})
self.indices.append('socorro200000')
# Create an old aliased index.
self.storage.create_index('socorro200201_20030101', {})
self.indices.append('socorro200201_20030101')
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro200201_20030101',
'alias': 'socorro200201'
}
}]
})
# Create a recent aliased index.
last_week_index = self.storage.get_index_for_crash(
utc_now() - datetime.timedelta(weeks=1)
)
self.storage.create_index('socorro_some_aliased_index', {})
self.indices.append('socorro_some_aliased_index')
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro_some_aliased_index',
'alias': last_week_index
}
}]
})
# Create a recent index that should not be deleted.
now_index = self.storage.get_index_for_crash(utc_now())
self.storage.create_index(now_index, {})
self.indices.append(now_index)
# These will raise an error if an index was not correctly created.
es.status('socorro200142')
es.status('socorro200000')
es.status('socorro200201')
es.status(now_index)
es.status(last_week_index)
api = IndexCleaner(config)
api.delete_old_indices()
# Verify the recent index is still there.
es.status(now_index)
es.status(last_week_index)
# Verify the old indices are gone.
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200142'
)
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200000'
)
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200201'
)
@maximum_es_version('0.90')
def test_other_indices_are_not_deleted(self):
"""Verify that non-week-based indices are not removed. For example,
the socorro_email index should not be deleted by the cron job.
"""
config_manager = self._setup_config()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create the socorro emails index.
self.storage.create_emails_index()
self.indices.append('socorro_emails')
# This will raise an error if the index was not correctly created.
es.status('socorro_emails')
api = IndexCleaner(config)
api.delete_old_indices()
# Verify the email index is still there. This will raise an error
# if the index does not exist.
es.status('socorro_emails')
class IntegrationTestElasticsearchCleanup(IntegrationTestBase):
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.elasticsearch_cleanup.'
'ElasticsearchCleanupCronApp|30d', )
def __init__(self, *args, **kwargs):
super(IntegrationTestElasticsearchCleanup,
self).__init__(*args, **kwargs)
storage_config = self._setup_storage_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def tearDown(self):
# Clean up created indices.
self.storage.es.delete_index('socorro*')
super(IntegrationTestElasticsearchCleanup, self).tearDown()
def _setup_storage_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
return ConfigurationManager([storage_conf],
values_source_list=[os.environ],
argv_source=[])
def test_right_indices_are_deleted(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create old indices to be deleted.
self.storage.create_index('socorro200142', {})
self.storage.create_index('socorro200000', {})
# Create an old aliased index.
self.storage.create_index('socorro200201_20030101', {})
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro200201_20030101',
'alias': 'socorro200201'
}
}]
})
# Create a recent aliased index.
last_week_index = self.storage.get_index_for_crash(
utc_now() - datetime.timedelta(weeks=1))
self.storage.create_index('socorro_some_aliased_index', {})
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro_some_aliased_index',
'alias': last_week_index
}
}]
})
# Create a recent index that should not be deleted.
now_index = self.storage.get_index_for_crash(utc_now())
self.storage.create_index(now_index, {})
# These will raise an error if an index was not correctly created.
es.status('socorro200142')
es.status('socorro200000')
es.status('socorro200201')
es.status(now_index)
es.status(last_week_index)
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
# Verify the recent index is still there.
es.status(now_index)
es.status(last_week_index)
# Verify the old indices are gone.
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200142')
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200000')
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200201')
def test_other_indices_are_not_deleted(self):
"""Verify that non-week-based indices are not removed. For example,
the socorro_email index should not be deleted by the cron job.
"""
config_manager = self._setup_config_manager()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create the socorro emails index.
self.storage.create_emails_index()
# This will raise an error if the index was not correctly created.
es.status('socorro_emails')
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
# Verify the email index is still there. This will raise an error
# if the index does not exist.
es.status('socorro_emails')
def test_email_after_delay(self, exacttarget_mock):
"""Test that a user will receive an email if he or she sends us a new
crash report after the delay is passed (but not before). """
config_manager = self._setup_config_manager(
delay_between_emails=1,
restrict_products=['EarthRaccoon']
)
email = '*****@*****.**'
list_service_mock = exacttarget_mock.return_value.list.return_value
list_service_mock.get_subscriber.return_value = {
'token': email
}
trigger_send_mock = exacttarget_mock.return_value.trigger_send
tomorrow = utc_now() + datetime.timedelta(days=1, hours=2)
twohourslater = utc_now() + datetime.timedelta(hours=2)
storage_config_manager = self._setup_storage_config()
with storage_config_manager.context() as storage_config:
storage = ElasticSearchCrashStorage(storage_config)
with config_manager.context() as config:
# 1. Send an email to the user and update emailing data
tab = crontabber.CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
exacttarget_mock.return_value.trigger_send.assert_called_with(
'socorro_dev_test',
{
'EMAIL_ADDRESS_': email,
'EMAIL_FORMAT_': 'H',
'TOKEN': email
}
)
self.assertEqual(trigger_send_mock.call_count, 1)
# 2. Test that before 'delay' is passed user doesn't receive
# another email
# Insert a new crash report with the same email address
storage.save_processed({
'uuid': '50',
'email': email,
'product': 'EarthRaccoon',
'version': '20.0',
'release_channel': 'Release',
'date_processed': utc_now() + datetime.timedelta(hours=1)
})
storage.es.refresh()
# Run crontabber with time pushed by two hours
with mock.patch('socorro.cron.crontabber.utc_now') as cronutc_mock:
with mock.patch('socorro.cron.base.utc_now') as baseutc_mock:
cronutc_mock.return_value = twohourslater
baseutc_mock.return_value = twohourslater
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# No new email was sent
self.assertEqual(trigger_send_mock.call_count, 1)
# 3. Verify that, after 'delay' is passed, a new email is sent
# to our user
# Insert a new crash report with the same email address
storage.save_processed({
'uuid': '51',
'email': email,
'product': 'EarthRaccoon',
'version': '20.0',
'release_channel': 'Release',
'date_processed': utc_now() + datetime.timedelta(days=1)
})
storage.es.refresh()
# Run crontabber with time pushed by a day
with mock.patch('socorro.cron.crontabber.utc_now') as cronutc_mock:
with mock.patch('socorro.cron.base.utc_now') as baseutc_mock:
cronutc_mock.return_value = tomorrow
baseutc_mock.return_value = tomorrow
tab.run_all()
information = self._load_structure()
assert information['automatic-emails']
assert not information['automatic-emails']['last_error']
assert information['automatic-emails']['last_success']
# A new email was sent
self.assertEqual(trigger_send_mock.call_count, 2)
class IntegrationTestIndexCleaner(ElasticSearchTestCase):
def __init__(self, *args, **kwargs):
super(IntegrationTestIndexCleaner, self).__init__(*args, **kwargs)
storage_config = self._setup_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def setUp(self):
self.indices = []
def tearDown(self):
# Clean up created indices.
for index in self.indices:
try:
self.storage.es.delete_index(index)
# "Missing" indices have already been deleted, no need to worry.
except pyelasticsearch.exceptions.ElasticHttpNotFoundError:
pass
super(IntegrationTestIndexCleaner, self).tearDown()
def _setup_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
cleaner_conf = IndexCleaner.get_required_config()
cleaner_conf.add_option('logger', default=mock_logging)
return ConfigurationManager([storage_conf, cleaner_conf],
values_source_list=[environment],
argv_source=[])
@maximum_es_version('0.90')
def test_correct_indices_are_deleted(self):
config_manager = self._setup_config()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create old indices to be deleted.
self.storage.create_index('socorro200142', {})
self.indices.append('socorro200142')
self.storage.create_index('socorro200000', {})
self.indices.append('socorro200000')
# Create an old aliased index.
self.storage.create_index('socorro200201_20030101', {})
self.indices.append('socorro200201_20030101')
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro200201_20030101',
'alias': 'socorro200201'
}
}]
})
# Create a recent aliased index.
last_week_index = self.storage.get_index_for_crash(
utc_now() - datetime.timedelta(weeks=1))
self.storage.create_index('socorro_some_aliased_index', {})
self.indices.append('socorro_some_aliased_index')
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro_some_aliased_index',
'alias': last_week_index
}
}]
})
# Create a recent index that should not be deleted.
now_index = self.storage.get_index_for_crash(utc_now())
self.storage.create_index(now_index, {})
self.indices.append(now_index)
# These will raise an error if an index was not correctly created.
es.status('socorro200142')
es.status('socorro200000')
es.status('socorro200201')
es.status(now_index)
es.status(last_week_index)
api = IndexCleaner(config)
api.delete_old_indices()
# Verify the recent index is still there.
es.status(now_index)
es.status(last_week_index)
# Verify the old indices are gone.
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200142')
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200000')
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200201')
@maximum_es_version('0.90')
def test_other_indices_are_not_deleted(self):
"""Verify that non-week-based indices are not removed. For example,
the socorro_email index should not be deleted by the cron job.
"""
config_manager = self._setup_config()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create the socorro emails index.
self.storage.create_emails_index()
self.indices.append('socorro_emails')
# This will raise an error if the index was not correctly created.
es.status('socorro_emails')
api = IndexCleaner(config)
api.delete_old_indices()
# Verify the email index is still there. This will raise an error
# if the index does not exist.
es.status('socorro_emails')
def setUp(self):
super(IntegrationTestAutomaticEmails, self).setUp()
# prep a fake table
now = utc_now() - datetime.timedelta(minutes=30)
last_month = now - datetime.timedelta(days=31)
config_manager = self._setup_storage_config()
with config_manager.context() as config:
storage = ElasticSearchCrashStorage(config)
# clear the indices cache so the index is created on every test
storage.indices_cache = set()
storage.save_processed({
'uuid': '1',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': 'unknown'
}
}
})
storage.save_processed({
'uuid': '2',
'email': '"Quidam" <*****@*****.**>',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': None
}
}
})
storage.save_processed({
'uuid': '3',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': 'bitguard'
}
}
})
storage.save_processed({
'uuid': '4',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '5',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '6',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '7',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '8',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '9',
'email': '*****@*****.**',
'product': 'EarthRaccoon',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '18',
'email': 'z\xc3\[email protected]',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Let's insert a duplicate
storage.save_processed({
'uuid': '10',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# And let's insert some invalid crashes
storage.save_processed({
'uuid': '11',
'email': None,
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '12',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': last_month
})
storage.save_processed({
'uuid': '13',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '14',
'email': '*****@*****.**',
'product': 'WindBear',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Finally some invalid email addresses
storage.save_processed({
'uuid': '15',
'email': ' ',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '16',
'email': 'invalid@email',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '17',
'email': 'i.do.not.work',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Create some email addresses.
storage.create_emails_index()
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '*****@*****.**',
'last_sending': last_month
},
id='*****@*****.**',
)
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '"Quidam" <*****@*****.**>',
'last_sending': last_month
},
id='"Quidam" <*****@*****.**>',
)
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '*****@*****.**',
'last_sending': now
},
id='*****@*****.**',
)
# As indexing is asynchronous, we need to force elasticsearch to
# make the newly created content searchable before we run the
# tests.
storage.es.refresh()
class IntegrationTestElasticsearchCleanup(IntegrationTestBase):
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.elasticsearch_cleanup.'
'ElasticsearchCleanupCronApp|30d',
)
def __init__(self, *args, **kwargs):
super(
IntegrationTestElasticsearchCleanup,
self
).__init__(*args, **kwargs)
storage_config = self._setup_storage_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def tearDown(self):
# Clean up created indices.
self.storage.es.delete_index('socorro*')
super(IntegrationTestElasticsearchCleanup, self).tearDown()
def _setup_storage_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
return ConfigurationManager(
[storage_conf],
values_source_list=[os.environ],
argv_source=[]
)
def test_right_indices_are_deleted(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create old indices to be deleted.
self.storage.create_index('socorro200142', {})
self.storage.create_index('socorro200000', {})
# Create an old aliased index.
self.storage.create_index('socorro200201_20030101', {})
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro200201_20030101',
'alias': 'socorro200201'
}
}]
})
# Create a recent aliased index.
last_week_index = self.storage.get_index_for_crash(
utc_now() - datetime.timedelta(weeks=1)
)
self.storage.create_index('socorro_some_aliased_index', {})
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro_some_aliased_index',
'alias': last_week_index
}
}]
})
# Create a recent index that should not be deleted.
now_index = self.storage.get_index_for_crash(utc_now())
self.storage.create_index(now_index, {})
# These will raise an error if an index was not correctly created.
es.status('socorro200142')
es.status('socorro200000')
es.status('socorro200201')
es.status(now_index)
es.status(last_week_index)
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
# Verify the recent index is still there.
es.status(now_index)
es.status(last_week_index)
# Verify the old indices are gone.
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200142'
)
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200000'
)
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200201'
)
def test_other_indices_are_not_deleted(self):
"""Verify that non-week-based indices are not removed. For example,
the socorro_email index should not be deleted by the cron job.
"""
config_manager = self._setup_config_manager()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create the socorro emails index.
self.storage.create_emails_index()
# This will raise an error if the index was not correctly created.
es.status('socorro_emails')
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
# Verify the email index is still there. This will raise an error
# if the index does not exist.
es.status('socorro_emails')
