def setUp(self):
super(IntegrationTestAutomaticEmails, self).setUp()
# prep a fake table
now = utc_now() - datetime.timedelta(minutes=30)
last_month = now - datetime.timedelta(days=31)
config_manager = self._setup_storage_config()
with config_manager.context() as config:
storage = ElasticSearchCrashStorage(config)
# clear the indices cache so the index is created on every test
storage.indices_cache = set()
storage.save_processed({
'uuid': '1',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': 'unknown'
}
}
})
storage.save_processed({
'uuid': '2',
'email': '"Quidam" <*****@*****.**>',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': None
}
}
})
storage.save_processed({
'uuid': '3',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': 'bitguard'
}
}
})
storage.save_processed({
'uuid': '4',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '5',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '6',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '7',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '8',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '9',
'email': '*****@*****.**',
'product': 'EarthRaccoon',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '18',
'email': 'z\xc3\[email protected]',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Let's insert a duplicate
storage.save_processed({
'uuid': '10',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# And let's insert some invalid crashes
storage.save_processed({
'uuid': '11',
'email': None,
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '12',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': last_month
})
storage.save_processed({
'uuid': '13',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '14',
'email': '*****@*****.**',
'product': 'WindBear',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Finally some invalid email addresses
storage.save_processed({
'uuid': '15',
'email': ' ',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '16',
'email': 'invalid@email',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '17',
'email': 'i.do.not.work',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Create some email addresses.
storage.create_emails_index()
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '*****@*****.**',
'last_sending': last_month
},
id='*****@*****.**',
)
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '"Quidam" <*****@*****.**>',
'last_sending': last_month
},
id='"Quidam" <*****@*****.**>',
)
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '*****@*****.**',
'last_sending': now
},
id='*****@*****.**',
)
# As indexing is asynchronous, we need to force elasticsearch to
# make the newly created content searchable before we run the
# tests.
storage.es.refresh()
def setUp(self):
super(IntegrationTestAutomaticEmails, self).setUp()
# prep a fake table
now = utc_now() - datetime.timedelta(minutes=30)
last_month = now - datetime.timedelta(days=31)
config_manager = self._setup_storage_config()
with config_manager.context() as config:
storage = ElasticSearchCrashStorage(config)
# clear the indices cache so the index is created on every test
storage.indices_cache = set()
storage.save_processed({
'uuid': '1',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': 'unknown'
}
}
})
storage.save_processed({
'uuid': '2',
'email': '"Quidam" <*****@*****.**>',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': None
}
}
})
storage.save_processed({
'uuid': '3',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now,
'classifications': {
'support': {
'classification': 'bitguard'
}
}
})
storage.save_processed({
'uuid': '4',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '5',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '6',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '7',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '8',
'email': '*****@*****.**',
'product': 'NightlyTrain',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '9',
'email': '*****@*****.**',
'product': 'EarthRaccoon',
'version': '1.0',
'release_channel': 'Nightly',
'date_processed': now
})
storage.save_processed({
'uuid': '18',
'email': 'z\xc3\[email protected]',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Let's insert a duplicate
storage.save_processed({
'uuid': '10',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# And let's insert some invalid crashes
storage.save_processed({
'uuid': '11',
'email': None,
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '12',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': last_month
})
storage.save_processed({
'uuid': '13',
'email': '*****@*****.**',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '14',
'email': '*****@*****.**',
'product': 'WindBear',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Finally some invalid email addresses
storage.save_processed({
'uuid': '15',
'email': ' ',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '16',
'email': 'invalid@email',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
storage.save_processed({
'uuid': '17',
'email': 'i.do.not.work',
'product': 'WaterWolf',
'version': '20.0',
'release_channel': 'Release',
'date_processed': now
})
# Create some email addresses.
storage.create_emails_index()
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '*****@*****.**',
'last_sending': last_month
},
id='*****@*****.**',
)
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '"Quidam" <*****@*****.**>',
'last_sending': last_month
},
id='"Quidam" <*****@*****.**>',
)
storage.es.index(
index=config.elasticsearch_emails_index,
doc_type='emails',
doc={
'email': '*****@*****.**',
'last_sending': now
},
id='*****@*****.**',
)
# As indexing is asynchronous, we need to force elasticsearch to
# make the newly created content searchable before we run the
# tests.
storage.es.refresh()
class IntegrationTestIndexCleaner(ElasticSearchTestCase):
def __init__(self, *args, **kwargs):
super(IntegrationTestIndexCleaner, self).__init__(*args, **kwargs)
storage_config = self._setup_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def setUp(self):
self.indices = []
def tearDown(self):
# Clean up created indices.
for index in self.indices:
try:
self.storage.es.delete_index(index)
# "Missing" indices have already been deleted, no need to worry.
except pyelasticsearch.exceptions.ElasticHttpNotFoundError:
pass
super(IntegrationTestIndexCleaner, self).tearDown()
def _setup_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
cleaner_conf = IndexCleaner.get_required_config()
cleaner_conf.add_option('logger', default=mock_logging)
return ConfigurationManager([storage_conf, cleaner_conf],
values_source_list=[environment],
argv_source=[])
@maximum_es_version('0.90')
def test_correct_indices_are_deleted(self):
config_manager = self._setup_config()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create old indices to be deleted.
self.storage.create_index('socorro200142', {})
self.indices.append('socorro200142')
self.storage.create_index('socorro200000', {})
self.indices.append('socorro200000')
# Create an old aliased index.
self.storage.create_index('socorro200201_20030101', {})
self.indices.append('socorro200201_20030101')
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro200201_20030101',
'alias': 'socorro200201'
}
}]
})
# Create a recent aliased index.
last_week_index = self.storage.get_index_for_crash(
utc_now() - datetime.timedelta(weeks=1))
self.storage.create_index('socorro_some_aliased_index', {})
self.indices.append('socorro_some_aliased_index')
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro_some_aliased_index',
'alias': last_week_index
}
}]
})
# Create a recent index that should not be deleted.
now_index = self.storage.get_index_for_crash(utc_now())
self.storage.create_index(now_index, {})
self.indices.append(now_index)
# These will raise an error if an index was not correctly created.
es.status('socorro200142')
es.status('socorro200000')
es.status('socorro200201')
es.status(now_index)
es.status(last_week_index)
api = IndexCleaner(config)
api.delete_old_indices()
# Verify the recent index is still there.
es.status(now_index)
es.status(last_week_index)
# Verify the old indices are gone.
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200142')
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200000')
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200201')
@maximum_es_version('0.90')
def test_other_indices_are_not_deleted(self):
"""Verify that non-week-based indices are not removed. For example,
the socorro_email index should not be deleted by the cron job.
"""
config_manager = self._setup_config()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create the socorro emails index.
self.storage.create_emails_index()
self.indices.append('socorro_emails')
# This will raise an error if the index was not correctly created.
es.status('socorro_emails')
api = IndexCleaner(config)
api.delete_old_indices()
# Verify the email index is still there. This will raise an error
# if the index does not exist.
es.status('socorro_emails')
class IntegrationTestIndexCleaner(ElasticSearchTestCase):
def __init__(self, *args, **kwargs):
super(
IntegrationTestIndexCleaner,
self
).__init__(*args, **kwargs)
storage_config = self._setup_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def setUp(self):
self.indices = []
def tearDown(self):
# Clean up created indices.
for index in self.indices:
try:
self.storage.es.delete_index(index)
# "Missing" indices have already been deleted, no need to worry.
except pyelasticsearch.exceptions.ElasticHttpNotFoundError:
pass
super(IntegrationTestIndexCleaner, self).tearDown()
def _setup_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
cleaner_conf = IndexCleaner.get_required_config()
cleaner_conf.add_option('logger', default=mock_logging)
return ConfigurationManager(
[storage_conf, cleaner_conf],
values_source_list=[environment],
argv_source=[]
)
@maximum_es_version('0.90')
def test_correct_indices_are_deleted(self):
config_manager = self._setup_config()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create old indices to be deleted.
self.storage.create_index('socorro200142', {})
self.indices.append('socorro200142')
self.storage.create_index('socorro200000', {})
self.indices.append('socorro200000')
# Create an old aliased index.
self.storage.create_index('socorro200201_20030101', {})
self.indices.append('socorro200201_20030101')
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro200201_20030101',
'alias': 'socorro200201'
}
}]
})
# Create a recent aliased index.
last_week_index = self.storage.get_index_for_crash(
utc_now() - datetime.timedelta(weeks=1)
)
self.storage.create_index('socorro_some_aliased_index', {})
self.indices.append('socorro_some_aliased_index')
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro_some_aliased_index',
'alias': last_week_index
}
}]
})
# Create a recent index that should not be deleted.
now_index = self.storage.get_index_for_crash(utc_now())
self.storage.create_index(now_index, {})
self.indices.append(now_index)
# These will raise an error if an index was not correctly created.
es.status('socorro200142')
es.status('socorro200000')
es.status('socorro200201')
es.status(now_index)
es.status(last_week_index)
api = IndexCleaner(config)
api.delete_old_indices()
# Verify the recent index is still there.
es.status(now_index)
es.status(last_week_index)
# Verify the old indices are gone.
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200142'
)
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200000'
)
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200201'
)
@maximum_es_version('0.90')
def test_other_indices_are_not_deleted(self):
"""Verify that non-week-based indices are not removed. For example,
the socorro_email index should not be deleted by the cron job.
"""
config_manager = self._setup_config()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create the socorro emails index.
self.storage.create_emails_index()
self.indices.append('socorro_emails')
# This will raise an error if the index was not correctly created.
es.status('socorro_emails')
api = IndexCleaner(config)
api.delete_old_indices()
# Verify the email index is still there. This will raise an error
# if the index does not exist.
es.status('socorro_emails')
class IntegrationTestElasticsearchCleanup(IntegrationTestBase):
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.elasticsearch_cleanup.'
'ElasticsearchCleanupCronApp|30d',
)
def __init__(self, *args, **kwargs):
super(
IntegrationTestElasticsearchCleanup,
self
).__init__(*args, **kwargs)
storage_config = self._setup_storage_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def tearDown(self):
# Clean up created indices.
self.storage.es.delete_index('socorro*')
super(IntegrationTestElasticsearchCleanup, self).tearDown()
def _setup_storage_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
return ConfigurationManager(
[storage_conf],
values_source_list=[os.environ],
argv_source=[]
)
def test_right_indices_are_deleted(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create old indices to be deleted.
self.storage.create_index('socorro200142', {})
self.storage.create_index('socorro200000', {})
# Create an old aliased index.
self.storage.create_index('socorro200201_20030101', {})
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro200201_20030101',
'alias': 'socorro200201'
}
}]
})
# Create a recent aliased index.
last_week_index = self.storage.get_index_for_crash(
utc_now() - datetime.timedelta(weeks=1)
)
self.storage.create_index('socorro_some_aliased_index', {})
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro_some_aliased_index',
'alias': last_week_index
}
}]
})
# Create a recent index that should not be deleted.
now_index = self.storage.get_index_for_crash(utc_now())
self.storage.create_index(now_index, {})
# These will raise an error if an index was not correctly created.
es.status('socorro200142')
es.status('socorro200000')
es.status('socorro200201')
es.status(now_index)
es.status(last_week_index)
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
# Verify the recent index is still there.
es.status(now_index)
es.status(last_week_index)
# Verify the old indices are gone.
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200142'
)
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200000'
)
assert_raises(
pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status,
'socorro200201'
)
def test_other_indices_are_not_deleted(self):
"""Verify that non-week-based indices are not removed. For example,
the socorro_email index should not be deleted by the cron job.
"""
config_manager = self._setup_config_manager()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create the socorro emails index.
self.storage.create_emails_index()
# This will raise an error if the index was not correctly created.
es.status('socorro_emails')
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
# Verify the email index is still there. This will raise an error
# if the index does not exist.
es.status('socorro_emails')
class IntegrationTestElasticsearchCleanup(IntegrationTestBase):
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.elasticsearch_cleanup.'
'ElasticsearchCleanupCronApp|30d', )
def __init__(self, *args, **kwargs):
super(IntegrationTestElasticsearchCleanup,
self).__init__(*args, **kwargs)
storage_config = self._setup_storage_config()
with storage_config.context() as config:
self.storage = ElasticSearchCrashStorage(config)
def tearDown(self):
# Clean up created indices.
self.storage.es.delete_index('socorro*')
super(IntegrationTestElasticsearchCleanup, self).tearDown()
def _setup_storage_config(self):
mock_logging = mock.Mock()
storage_conf = ElasticSearchCrashStorage.get_required_config()
storage_conf.add_option('logger', default=mock_logging)
return ConfigurationManager([storage_conf],
values_source_list=[os.environ],
argv_source=[])
def test_right_indices_are_deleted(self):
config_manager = self._setup_config_manager()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create old indices to be deleted.
self.storage.create_index('socorro200142', {})
self.storage.create_index('socorro200000', {})
# Create an old aliased index.
self.storage.create_index('socorro200201_20030101', {})
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro200201_20030101',
'alias': 'socorro200201'
}
}]
})
# Create a recent aliased index.
last_week_index = self.storage.get_index_for_crash(
utc_now() - datetime.timedelta(weeks=1))
self.storage.create_index('socorro_some_aliased_index', {})
es.update_aliases({
'actions': [{
'add': {
'index': 'socorro_some_aliased_index',
'alias': last_week_index
}
}]
})
# Create a recent index that should not be deleted.
now_index = self.storage.get_index_for_crash(utc_now())
self.storage.create_index(now_index, {})
# These will raise an error if an index was not correctly created.
es.status('socorro200142')
es.status('socorro200000')
es.status('socorro200201')
es.status(now_index)
es.status(last_week_index)
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
# Verify the recent index is still there.
es.status(now_index)
es.status(last_week_index)
# Verify the old indices are gone.
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200142')
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200000')
assert_raises(pyelasticsearch.exceptions.ElasticHttpNotFoundError,
es.status, 'socorro200201')
def test_other_indices_are_not_deleted(self):
"""Verify that non-week-based indices are not removed. For example,
the socorro_email index should not be deleted by the cron job.
"""
config_manager = self._setup_config_manager()
with config_manager.context() as config:
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
es = self.storage.es
# Create the socorro emails index.
self.storage.create_emails_index()
# This will raise an error if the index was not correctly created.
es.status('socorro_emails')
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['elasticsearch-cleanup']
assert not information['elasticsearch-cleanup']['last_error']
assert information['elasticsearch-cleanup']['last_success']
# Verify the email index is still there. This will raise an error
# if the index does not exist.
es.status('socorro_emails')
