def test_all_daily_indices_found(self):
client = Mock()
client.indices.get_settings.return_value = {
'prefix-2014.01.03': True,
'prefix-2014.01.02': True,
'prefix-2014.01.01': True,
'prefix-2013.12.31': True,
'prefix-2013.12.30': True,
'prefix-2013.12.29': True,
'prefix-2013.01.03': True,
'prefix-2013.01.03.10': True,
'prefix-2013.01': True,
'prefix-2013.12': True,
'prefix-2013.51': True,
}
index_list = curator.get_object_list(client,
prefix='prefix-',
suffix='')
expired = curator.filter_by_timestamp(object_list=index_list,
time_unit='days',
older_than=4,
prefix='prefix-',
suffix='',
timestring='%Y.%m.%d',
utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
'prefix-2013.01.03',
'prefix-2013.12.29',
'prefix-2013.12.30',
], expired)
def test_all_daily_indices_found_with_star_prefix_and_suffix(self):
client = Mock()
client.indices.get_settings.return_value = {
'log-2014.01.03-bar': True,
'log-2014.01.02-baz': True,
'log-2014.01.01-b': True,
'log0-2013.12.31-c': True,
'logstash-2013.12.30-bigdata': True,
'l-2013.12.29-closet': True,
'prod-2013.01.03-basketball': True,
'cert-2013.01.03.10-a': True,
'test-2013.01-d': True,
'fail-2013.12-f': True,
'index-2013.51-e': True,
}
index_list = curator.get_object_list(client, prefix='*', suffix='*')
expired = curator.filter_by_timestamp(object_list=index_list,
time_unit='days',
older_than=4,
prefix='*',
suffix='*',
timestring='%Y.%m.%d',
utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
'cert-2013.01.03.10-a',
'l-2013.12.29-closet',
'logstash-2013.12.30-bigdata',
'prod-2013.01.03-basketball',
], expired)
def test_all_daily_indices_found_with_wildcard_prefix(self):
client = Mock()
client.indices.get_settings.return_value = {
'log-2014.01.03': True,
'log-2014.01.02': True,
'log-2014.01.01': True,
'log0-2013.12.31': True,
'logstash-2013.12.30': True,
'l-2013.12.29': True,
'prod-2013.01.03': True,
'cert-2013.01.03.10': True,
'test-2013.01': True,
'fail-2013.12': True,
'index-2013.51': True,
}
index_list = curator.get_object_list(client, prefix='l.*', suffix='')
expired = curator.find_expired_data(object_list=index_list, time_unit='days', older_than=4, prefix='l.*', suffix='', timestring='%Y.%m.%d', utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
'l-2013.12.29',
'logstash-2013.12.30',
],
expired
)
def test_all_daily_indices_found_with_suffix_and_no_prefix(self):
client = Mock()
client.indices.get_settings.return_value = {
'2014.01.03-suffix': True,
'prefix-2014.01.02': True,
'2014.01.01-suffix': True,
'prefix-2013.12.31': True,
'2013.12.30-suffix': True,
'prefix-2013.12.29': True,
'2013.01.03-suffix': True,
'2013.01.03.10': True,
'prefix-2013.01': True,
'prefix-2013.12-suffix': True,
'prefix-2013.51': True,
}
index_list = curator.get_object_list(client, prefix='', suffix='-suffix')
expired = curator.find_expired_data(object_list=index_list, time_unit='days', older_than=4, prefix='', suffix='-suffix', timestring='%Y.%m.%d', utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
'2013.01.03-suffix',
'2013.12.30-suffix',
],
expired
)
def test_all_daily_indices_found_with_star_prefix_and_suffix(self):
client = Mock()
client.indices.get_settings.return_value = {
'log-2014.01.03-bar': True,
'log-2014.01.02-baz': True,
'log-2014.01.01-b': True,
'log0-2013.12.31-c': True,
'logstash-2013.12.30-bigdata': True,
'l-2013.12.29-closet': True,
'prod-2013.01.03-basketball': True,
'cert-2013.01.03.10-a': True,
'test-2013.01-d': True,
'fail-2013.12-f': True,
'index-2013.51-e': True,
}
index_list = curator.get_object_list(client, prefix='*', suffix='*')
expired = curator.filter_by_timestamp(object_list=index_list, time_unit='days', older_than=4, prefix='*', suffix='*', timestring='%Y.%m.%d', utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
'cert-2013.01.03.10-a',
'l-2013.12.29-closet',
'logstash-2013.12.30-bigdata',
'prod-2013.01.03-basketball',
],
expired
)
def test_all_daily_indices_found(self):
client = Mock()
client.indices.get_settings.return_value = {
'prefix-2014.01.03': True,
'prefix-2014.01.02': True,
'prefix-2014.01.01': True,
'prefix-2013.12.31': True,
'prefix-2013.12.30': True,
'prefix-2013.12.29': True,
'prefix-2013.01.03': True,
'prefix-2013.01.03.10': True,
}
index_list = curator.get_object_list(client, prefix='prefix-')
expired = curator.find_expired_data(client, object_list=index_list, time_unit='days', older_than=4, prefix='prefix-', utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
('prefix-2013.01.03', timedelta(days=362)),
('prefix-2013.12.29', timedelta(days=2)),
('prefix-2013.12.30', timedelta(days=1)),
],
expired
)
def test_all_daily_indices_found_with_wildcard_prefix(self):
client = Mock()
client.indices.get_settings.return_value = {
'log-2014.01.03': True,
'log-2014.01.02': True,
'log-2014.01.01': True,
'log0-2013.12.31': True,
'logstash-2013.12.30': True,
'l-2013.12.29': True,
'prod-2013.01.03': True,
'cert-2013.01.03.10': True,
'test-2013.01': True,
'fail-2013.12': True,
'index-2013.51': True,
}
index_list = curator.get_object_list(client, prefix='l.*', suffix='')
expired = curator.find_expired_data(object_list=index_list,
time_unit='days',
older_than=4,
prefix='l.*',
suffix='',
timestring='%Y.%m.%d',
utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
'l-2013.12.29',
'logstash-2013.12.30',
], expired)
def test_all_daily_indices_found_with_suffix_and_no_prefix(self):
client = Mock()
client.indices.get_settings.return_value = {
'2014.01.03-suffix': True,
'prefix-2014.01.02': True,
'2014.01.01-suffix': True,
'prefix-2013.12.31': True,
'2013.12.30-suffix': True,
'prefix-2013.12.29': True,
'2013.01.03-suffix': True,
'2013.01.03.10': True,
'prefix-2013.01': True,
'prefix-2013.12-suffix': True,
'prefix-2013.51': True,
}
index_list = curator.get_object_list(client,
prefix='',
suffix='-suffix')
expired = curator.find_expired_data(object_list=index_list,
time_unit='days',
older_than=4,
prefix='',
suffix='-suffix',
timestring='%Y.%m.%d',
utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
'2013.01.03-suffix',
'2013.12.30-suffix',
], expired)
def test_all_daily_indices_found(self):
client = Mock()
client.indices.get_settings.return_value = {
'prefix-2014.01.03': True,
'prefix-2014.01.02': True,
'prefix-2014.01.01': True,
'prefix-2013.12.31': True,
'prefix-2013.12.30': True,
'prefix-2013.12.29': True,
'prefix-2013.01.03': True,
'prefix-2013.01.03.10': True,
'prefix-2013.01': True,
'prefix-2013.12': True,
'prefix-2013.51': True,
}
index_list = curator.get_object_list(client, prefix='prefix-', suffix='')
expired = curator.filter_by_timestamp(object_list=index_list, time_unit='days', older_than=4, prefix='prefix-', suffix='', timestring='%Y.%m.%d', utc_now=datetime(2014, 1, 3))
expired = list(expired)
self.assertEquals([
'prefix-2013.01.03',
'prefix-2013.12.29',
'prefix-2013.12.30',
],
expired
)
def test_find_indices_ignores_indices_with_different_prefix_or_time_unit(self):
self.create_index('logstash-2012.01.01') # wrong precision
self.create_index('not-logstash-2012.01.01.00') # wrong prefix
self.create_index('logstash-2012.01.01.00')
index_list = curator.get_object_list(self.client, prefix=self.args['prefix'])
expired = list(curator.find_expired_data(self.client, time_unit='hours', older_than=1, object_list=index_list))
self.assertEquals(1, len(expired))
self.assertEquals('logstash-2012.01.01.00', expired[0][0])
def test_find_indices_ignores_indices_with_different_prefix_or_time_unit(
self):
self.create_index('logstash-2012.01.01') # wrong precision
self.create_index('not-logstash-2012.01.01.00') # wrong prefix
self.create_index('logstash-2012.01.01.00')
index_list = curator.get_object_list(self.client,
prefix=self.args['prefix'])
expired = list(
curator.find_expired_data(self.client,
time_unit='hours',
older_than=1,
object_list=index_list))
self.assertEquals(1, len(expired))
self.assertEquals('logstash-2012.01.01.00', expired[0][0])
def test_find_reports_correct_month_interval_from_cutoff(self):
self.create_index('l-2014.01')
self.create_index('l-2014.02')
# the last one is always saved since we reset to the first of the month and do <, not <=
self.create_index('l-2014.03')
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(curator.find_expired_data(time_unit='months', older_than=1, timestring='%Y.%m', object_list=index_list,
utc_now=datetime(2014, 5, 1), prefix='l-'))
self.assertEquals(
[ 'l-2014.01',
'l-2014.02',
'l-2014.03',
],
expired
)
def test_find_closed_monthly_indices(self):
self.create_index('l-2014.03')
self.client.indices.close(index='l-2014.03')
self.create_index('l-2014.04')
# all indices should be expired
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(curator.find_expired_data(time_unit='months', older_than=1, timestring='%Y.%m', object_list=index_list,
utc_now=datetime(2014, 6, 1, 0, 0, 0), prefix='l-'))
self.assertEquals(
[ 'l-2014.03',
'l-2014.04',
],
expired
)
def test_find_reports_correct_time_interval_from_cutoff(self):
self.create_index('l-2014.01.01')
self.create_index('l-2014.01.02')
# yesterday is always save since we reset to mignight and do <, not <=
self.create_index('l-2014.01.03')
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(curator.find_expired_data(time_unit='days', older_than=1, timestring='%Y.%m.%d', object_list=index_list,
utc_now=datetime(2014, 1, 4, 3, 45, 50), prefix='l-'))
self.assertEquals(
[ 'l-2014.01.01',
'l-2014.01.02',
'l-2014.01.03',
],
expired
)
def test_find_closed_indices(self):
self.create_index('l-2014.01.03')
self.client.indices.close(index='l-2014.01.03')
self.create_index('l-2014.01.01')
# all indices should be expired
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(curator.filter_by_timestamp(time_unit='days', older_than=1, timestring='%Y.%m.%d', object_list=index_list,
utc_now=datetime(2014, 1, 8, 3, 45, 50), prefix='l-'))
self.assertEquals(
[
'l-2014.01.01',
'l-2014.01.03',
],
expired
)
def test_find_closed_indices(self):
self.create_index('l-2014.01.03')
self.client.indices.close(index='l-2014.01.03')
self.create_index('l-2014.01.01')
# all indices should be expired
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(curator.find_expired_data(self.client, time_unit='days', older_than=1, object_list=index_list,
utc_now=datetime(2014, 1, 8, 3, 45, 50), prefix='l-'))
self.assertEquals(
[
('l-2014.01.01', timedelta(7)),
('l-2014.01.03', timedelta(5)),
],
expired
)
def test_find_reports_correct_time_interval_from_cutoff(self):
self.create_index('l-2014.01.01')
self.create_index('l-2014.01.02')
# yesterday is always save since we reset to mignight and do <, not <=
self.create_index('l-2014.01.03')
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(
curator.find_expired_data(self.client,
time_unit='days',
older_than=1,
object_list=index_list,
utc_now=datetime(2014, 1, 4, 3, 45, 50),
prefix='l-'))
self.assertEquals([
(u'l-2014.01.01', timedelta(3)),
(u'l-2014.01.02', timedelta(2)),
(u'l-2014.01.03', timedelta(1)),
], expired)
def test_find_closed_indices(self):
self.create_index('l-2014.01.03')
self.client.indices.close(index='l-2014.01.03')
self.create_index('l-2014.01.01')
# all indices should be expired
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(
curator.find_expired_data(self.client,
time_unit='days',
older_than=1,
object_list=index_list,
utc_now=datetime(2014, 1, 8, 3, 45, 50),
prefix='l-'))
self.assertEquals([
('l-2014.01.01', timedelta(7)),
('l-2014.01.03', timedelta(5)),
], expired)
def test_find_reports_correct_month_interval_from_cutoff(self):
self.create_index('l-2014.01')
self.create_index('l-2014.02')
# the last one is always saved since we reset to the first of the month and do <, not <=
self.create_index('l-2014.03')
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(
curator.filter_by_timestamp(time_unit='months',
older_than=1,
timestring='%Y.%m',
object_list=index_list,
utc_now=datetime(2014, 5, 1),
prefix='l-'))
self.assertEquals([
'l-2014.01',
'l-2014.02',
'l-2014.03',
], expired)
def test_find_closed_monthly_indices(self):
self.create_index('l-2014.03')
self.client.indices.close(index='l-2014.03')
self.create_index('l-2014.04')
# all indices should be expired
index_list = curator.get_object_list(self.client, prefix='l-')
expired = list(
curator.filter_by_timestamp(time_unit='months',
older_than=1,
timestring='%Y.%m',
object_list=index_list,
utc_now=datetime(2014, 6, 1, 0, 0, 0),
prefix='l-'))
self.assertEquals([
'l-2014.03',
'l-2014.04',
], expired)
