def test_restore_with_rename(self):
indices = []
for i in range(1, 4):
self.add_docs('my_index{0}'.format(i))
indices.append('my_index{0}'.format(i))
snap_name = 'snapshot1'
self.create_snapshot(snap_name, ','.join(indices))
snapshot = curator.get_snapshot(self.client, self.args['repository'],
'_all')
self.assertEqual(1, len(snapshot['snapshots']))
self.client.indices.delete(','.join(indices))
self.assertEqual([], curator.get_indices(self.client))
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.restore_snapshot_proto.format(self.args['repository'],
snap_name, indices, False,
False, True, False,
'my_index(.+)',
'new_index$1', ' ', True,
False, 301))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
restored_indices = sorted(curator.get_indices(self.client))
self.assertEqual(['new_index1', 'new_index2', 'new_index3'],
restored_indices)
def test_restore(self):
indices = []
for i in range(1,4):
self.add_docs('my_index{0}'.format(i))
indices.append('my_index{0}'.format(i))
snap_name = 'snapshot1'
self.create_snapshot(snap_name, ','.join(indices))
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
self.assertEqual(1, len(snapshot['snapshots']))
self.client.indices.delete(','.join(indices))
self.assertEqual([], curator.get_indices(self.client))
args = self.get_runner_args()
args += [
'--config', self.args['configfile'],
'restore',
'--repository', self.args['repository'],
'--name', snap_name,
'--index', indices[0],
'--index', indices[1],
'--index', indices[2],
'--wait_interval', '1',
'--max_wait', '3',
'--filter_list', '{"filtertype":"none"}',
]
self.assertEqual(0, self.run_subprocess(args, logname='TestCLIRestore.test_restore'))
restored_indices = sorted(curator.get_indices(self.client))
self.assertEqual(indices, restored_indices)
# The test runs so fast that it tries to execute the cleanup step
# and delete the repository before Elasticsearch is actually ready
time.sleep(0.5)
def test_restore(self):
indices = []
for i in range(1, 4):
self.add_docs('my_index{0}'.format(i))
indices.append('my_index{0}'.format(i))
snap_name = 'snapshot1'
self.create_snapshot(snap_name, ','.join(indices))
snapshot = curator.get_snapshot(self.client, self.args['repository'],
'_all')
self.assertEqual(1, len(snapshot['snapshots']))
self.client.indices.delete(','.join(indices))
self.assertEqual([], curator.get_indices(self.client))
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.restore_snapshot_proto.format(self.args['repository'],
snap_name, indices, False,
False, True, False, ' ',
' ', ' ', True, False, 301,
1, 3))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
restored_indices = sorted(curator.get_indices(self.client))
self.assertEqual(indices, restored_indices)
# The test runs so fast that it tries to execute the cleanup step
# and delete the repository before Elasticsearch is actually ready
time.sleep(0.5)
def test_plain(self):
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.create_index.format('testing'))
self.assertEqual([], curator.get_indices(self.client))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEqual(['testing'], curator.get_indices(self.client))
def test_restore_wildcard(self):
indices = []
my_indices = []
wildcard = ['my_*']
for i in range(1,4):
for prefix in ['my_', 'not_my_']:
self.add_docs('{0}index{1}'.format(prefix, i))
indices.append('{0}index{1}'.format(prefix, i))
if prefix == 'my_':
my_indices.append('{0}index{1}'.format(prefix, i))
snap_name = 'snapshot1'
self.create_snapshot(snap_name, ','.join(indices))
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
self.assertEqual(1, len(snapshot['snapshots']))
self.client.indices.delete(','.join(indices))
self.assertEqual([], curator.get_indices(self.client))
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.restore_snapshot_proto.format(
self.args['repository'],
snap_name,
wildcard,
False,
False,
True,
False,
' ',
' ',
' ',
True,
False,
301,
1,
3
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
restored_indices = sorted(curator.get_indices(self.client))
self.assertEqual(my_indices, restored_indices)
# The test runs so fast that it tries to execute the cleanup step
# and delete the repository before Elasticsearch is actually ready
time.sleep(0.5)
def test_restore_with_rename(self):
indices = []
for i in range(1,4):
self.add_docs('my_index{0}'.format(i))
indices.append('my_index{0}'.format(i))
snap_name = 'snapshot1'
self.create_snapshot(snap_name, ','.join(indices))
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
time.sleep(1)
self.assertEqual(1, len(snapshot['snapshots']))
self.client.indices.delete(','.join(indices))
self.assertEqual([], curator.get_indices(self.client))
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.restore_snapshot_proto.format(
self.args['repository'],
snap_name,
indices,
False,
False,
True,
False,
'my_index(.+)',
'new_index$1',
' ',
True,
False,
301,
1,
3,
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
time.sleep(1)
restored_indices = sorted(curator.get_indices(self.client))
self.assertEqual(
['new_index1', 'new_index2', 'new_index3'],
restored_indices
)
# The test runs so fast that it tries to execute the cleanup step
# and delete the repository before Elasticsearch is actually ready
time.sleep(1)
def test_with_date_math(self):
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.create_index.format('<testing-{now/d}>'))
self.assertEqual([], curator.get_indices(self.client))
name = curator.parse_date_pattern('testing-%Y.%m.%d')
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEqual([name], curator.get_indices(self.client))
def test_plain(self):
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.create_index.format('testing'))
self.assertEqual([], curator.get_indices(self.client))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEqual(['testing'], curator.get_indices(self.client))
def test_shrink_with_extras(self):
suffix = '-shrunken'
allocation_type = 'exclude'
key = '_name'
value = 'not_this_node'
self.builder(
with_extra_settings.format(
'DETERMINISTIC',
'permit_masters',
'True',
1,
0,
'',
suffix,
'False',
'index.codec',
'best_compression',
dict(),
allocation_type,
key,
value
)
)
indices = curator.get_indices(self.client)
self.assertEqual(2, len(indices)) # Should only have `my_index-shrunken`
settings = self.client.indices.get_settings()
self.assertTrue(settings[self.target]['settings']['index']['routing']['allocation'][allocation_type][key] == value)
self.assertTrue(settings[self.idx]['settings']['index']['routing']['allocation']['require']['_name'] == '')
self.assertEqual(settings[self.target]['settings']['index']['codec'], 'best_compression')
self.assertTrue(self.client.indices.exists_alias(index=self.target, name='my_alias'))
def test_shrink_explicit_shard_filter(self):
self.create_index('my_invalid_shrink_index', shards=3)
self.create_index('my_valid_shrink_index', shards=5)
suffix = '-shrunken'
self.builder(
shrink_filter_by_shards.format(
'DETERMINISTIC',
'permit_masters',
'True',
1,
0,
'',
suffix,
'True',
'False',
5,
'greater_than_or_equal'
)
)
indices = curator.get_indices(self.client)
self.assertEqual(3, len(indices))
self.assertTrue('my_invalid_shrink_index-shrunken' not in indices)
self.assertTrue('my_valid_shrink_index-shrunken' in indices)
self.assertTrue('my_index-shrunken' not in indices)
def test_shrink_with_extras(self):
suffix = '-shrunken'
allocation_type = 'exclude'
key = '_name'
value = 'not_this_node'
self.builder(
with_extra_settings.format(
'DETERMINISTIC',
'permit_masters',
'True',
1,
0,
'',
suffix,
'False',
'index.codec',
'best_compression',
dict(),
allocation_type,
key,
value
)
)
indices = curator.get_indices(self.client)
self.assertEqual(2, len(indices)) # Should only have `my_index-shrunken`
settings = self.client.indices.get_settings()
self.assertTrue(settings[self.target]['settings']['index']['routing']['allocation'][allocation_type][key] == value)
self.assertTrue(settings[self.idx]['settings']['index']['routing']['allocation']['require']['_name'] == '')
self.assertEqual(settings[self.target]['settings']['index']['codec'], 'best_compression')
self.assertTrue(self.client.indices.exists_alias(index=self.target, name='my_alias'))
def test_issue_826(self):
client = Mock()
client.info.return_value = {'version': {'number': '2.4.2'}}
client.indices.get_settings.return_value = testvars.settings_two
client.indices.exists.return_value = True
self.assertEqual(['.security', 'index-2016.03.03', 'index-2016.03.04'],
sorted(curator.get_indices(client)))
def test_with_date(self):
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.create_index.format('testing-%Y.%m.%d'))
self.assertEqual([], curator.get_indices(self.client))
name = curator.parse_date_pattern('testing-%Y.%m.%d')
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEqual([name], curator.get_indices(self.client))
def test_retention_from_name_days_ignore_failed_match(self):
# Test extraction of unit_count from index name
# Create indices for 10 days with retention time of 5 days in index name
# Create indices for 10 days with no retention time in index name
# Expected: 5 oldest indices are deleted, 5 remain - 10 indices without retention time are ignored and remain
self.args['prefix'] = 'logstash_5_'
self.create_indices(10)
self.args['prefix'] = 'logstash_'
self.create_indices(10)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.delete_pattern_proto.format(
'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 30, '_([0-9]+)_', ' ', ' ', ' '
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEquals(15, len(curator.get_indices(self.client)))
def test_delete_in_period(self):
# filtertype: {0}
# source: {1}
# range_from: {2}
# range_to: {3}
# timestring: {4}
# unit: {5}
# field: {6}
# stats_result: {7}
# intersect: {8}
# epoch: {9}
# week_starts_on: {10}
self.create_indices(10)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.delete_period_proto.format(
'period', 'name', '-5', '-1', "'%Y.%m.%d'", 'days',
' ', ' ', ' ', ' ', 'monday'
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEqual(0, result.exit_code)
self.assertEquals(5, len(curator.get_indices(self.client)))
def test_retention_from_name_days_keep_exclude_false_after_failed_match(
self):
# Test extraction of unit_count from index name and confirm correct
# behavior after a failed regex match with no fallback time - see gh issue 1206
# Create indices for 30 days with retention time of 5 days in index name
#
# Create indices for 10 days with no retention time in index name
# that alphabetically sort before the 10 with retention time
#
# Create indices for 10 days with no retention time in index name
# that sort after the 10 with retention time
# Expected: 45 oldest matching indices are deleted, 5 matching indices remain
# 20 indices without retention time are ignored and remain
# overall 25 indices should remain
self.args['prefix'] = 'logstash-aanomatch-'
self.create_indices(10)
self.args['prefix'] = 'logstash-match-5-'
self.create_indices(30)
self.args['prefix'] = 'logstash-zznomatch-'
self.create_indices(10)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.delete_pattern_proto.format(
'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', -1,
r'logstash-\w+-([0-9]+)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}', ' ',
' ', ' '))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEquals(25, len(curator.get_indices(self.client)))
def main(argv):
parser = OptionParser()
parser.add_option("-p", "--prefix", dest="prefix", action="store", help="prefix to indices filter ")
parser.add_option("-r", "--repository", dest="repository", action="store", help="repository name")
(opts, args) = parser.parse_args()
if not opts.repository: # if repository is not given
print 'repository not given'
usage()
sys.exit()
if not opts.prefix: # if repository is not given
print 'prefix not given'
usage()
sys.exit()
client = elasticsearch.Elasticsearch([
{'host': 'localhost', 'port': 9200}
])
indices_list = curator.get_indices(client)
_filter = curator.build_filter(kindOf='prefix', value=opts.prefix)
working_list = curator.apply_filter(indices_list, **_filter)
res = curator.create_snapshot(client, indices=working_list, name=None, prefix='curator-',
repository=opts.repository, ignore_unavailable=True, include_global_state=False,
partial=False, wait_for_completion=True, request_timeout=21600,
skip_repo_validation=False)
print res
def test_cli_snapshot_indices(self):
self.create_indices(5)
self.create_repository()
snap_name = 'snapshot1'
indices = curator.get_indices(self.client)
expected = sorted(indices, reverse=True)[:4]
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--logfile', os.devnull,
'--host', host,
'--port', str(port),
'snapshot',
'--repository', self.args['repository'],
'--name', snap_name,
'indices',
'--all-indices',
],
obj={"filters":[]})
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
self.assertEqual(1, len(snapshot['snapshots']))
self.assertEqual(snap_name, snapshot['snapshots'][0]['snapshot'])
def test_retention_from_name_days_ignore_failed_match(self):
# Test extraction of unit_count from index name
# Create indices for 10 days with retention time of 5 days in index name
# Create indices for 10 days with no retention time in index name
# Expected: 5 oldest indices are deleted, 5 remain - 10 indices without retention time are ignored and remain
self.args['prefix'] = 'logstash_5_'
self.create_indices(10)
self.args['prefix'] = 'logstash_'
self.create_indices(10)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.delete_pattern_proto.format(
'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 30, '_([0-9]+)_', ' ', ' ', ' '
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEquals(15, len(curator.get_indices(self.client)))
def test_delete_indices_by_space_dry_run(self):
for i in range(1, 10):
self.client.create(
index="index" + str(i),
doc_type='log',
body={'message': 'TEST DOCUMENT'},
)
test = clicktest.CliRunner()
result = test.invoke(curator.cli, [
'--dry-run',
'--host',
host,
'--port',
str(port),
'delete',
'--disk-space',
'0.0000001',
'indices',
'--all-indices',
],
obj={"filters": []})
l = curator.get_indices(self.client)
self.assertEquals(9, len(l))
output = sorted(result.output.splitlines(), reverse=True)
# I tried doing a nested, double list comprehension here.
# It works in the interpreter, but not here for some reason.
output = [x.split(' ')[-1:] for x in output]
output = [x[0] for x in output if x[0].startswith('index')]
self.assertEqual(sorted(l, reverse=True), output)
def test_positive(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_two
self.assertEqual(
['index-2016.03.03', 'index-2016.03.04'],
sorted(curator.get_indices(client))
)
def test_cli_snapshot_indices(self):
self.create_indices(5)
self.create_repository()
snap_name = 'snapshot1'
indices = curator.get_indices(self.client)
expected = sorted(indices, reverse=True)[:4]
test = clicktest.CliRunner()
result = test.invoke(curator.cli, [
'--logfile',
os.devnull,
'--host',
host,
'--port',
str(port),
'snapshot',
'--repository',
self.args['repository'],
'--name',
snap_name,
'indices',
'--all-indices',
],
obj={"filters": []})
snapshot = curator.get_snapshot(self.client, self.args['repository'],
'_all')
self.assertEqual(1, len(snapshot['snapshots']))
self.assertEqual(snap_name, snapshot['snapshots'][0]['snapshot'])
def test_delete_indices_by_space_dry_run_huge_list(self):
for i in range(100, 150):
self.client.create(
index=
"superlongindexnamebyanystandardyouchoosethisissillyhowbigcanthisgetbeforeitbreaks"
+ str(i),
doc_type='log',
body={'message': 'TEST DOCUMENT'},
)
l = curator.get_indices(self.client)
self.assertEquals(50, len(l))
test = clicktest.CliRunner()
result = test.invoke(curator.cli, [
'--dry-run',
'--host',
host,
'--port',
str(port),
'delete',
'--disk-space',
'0.0000001',
'indices',
'--all-indices',
],
obj={"filters": []})
output = sorted(result.output.splitlines(), reverse=True)
# # I tried doing a nested, double list comprehension here.
# # It works in the interpreter, but not here for some reason.
output = [x.split(' ')[-1:] for x in output]
output = [
x[0] for x in output if x[0].startswith('superlongindexname')
]
self.assertEqual(sorted(l, reverse=True), output)
def test_delete_indices_by_space_dry_run_huge_list(self):
for i in range(100,150):
self.client.create(
index="superlongindexnamebyanystandardyouchoosethisissillyhowbigcanthisgetbeforeitbreaks" + str(i), doc_type='log',
body={'message':'TEST DOCUMENT'},
)
l = curator.get_indices(self.client)
self.assertEquals(50, len(l))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--dry-run',
'--host', host,
'--port', str(port),
'delete',
'--disk-space', '0.0000001',
'indices',
'--all-indices',
],
obj={"filters":[]})
output = sorted(result.output.splitlines(), reverse=True)
# # I tried doing a nested, double list comprehension here.
# # It works in the interpreter, but not here for some reason.
output = [ x.split(' ')[-1:] for x in output ]
output = [ x[0] for x in output if x[0].startswith('superlongindexname') ]
self.assertEqual(sorted(l, reverse=True), output)
def test_allow_ilm_indices_false(self):
# ILM will not be added until 6.4
if curator.get_version(self.client) < (6,4,0):
self.assertTrue(True)
else:
self.create_indices(10)
settings = {
'index': {
'lifecycle': {
'name': 'mypolicy'
}
}
}
self.client.indices.put_settings(index='_all', body=settings)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.ilm_delete_proto.format(
'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'false'
)
)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
[ '--config', self.args['configfile'], self.args['actionfile'] ],
)
self.assertEquals(10, len(curator.get_indices(self.client)))
def show(client, **kwargs):
"""
Show indices or snapshots matching supplied parameters and exit.
:arg client: The Elasticsearch client connection
:arg data_type: Either ``index`` or ``snapshot``
:arg prefix: A string that comes before the datestamp in an index name.
Can be empty. Wildcards acceptable. Default is ``logstash-``.
:arg suffix: A string that comes after the datestamp of an index name.
Can be empty. Wildcards acceptable. Default is empty, ``''``.
:arg repository: The Elasticsearch snapshot repository to use (only with
snapshots)
:arg snapshot_prefix: Override the default with this value. Defaults to
``curator-``
"""
if kwargs['show_indices']:
for index_name in curator.get_indices(client,
prefix=kwargs['prefix'],
suffix=kwargs['suffix']):
print('{0}'.format(index_name))
sys.exit(0)
elif kwargs['show_snapshots']:
for snapshot in curator.get_snaplist(
client,
kwargs['repository'],
snapshot_prefix=kwargs['snapshot_prefix']):
print('{0}'.format(snapshot))
sys.exit(0)
def test_positive(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_two
self.assertEqual(
['index-2016.03.03', 'index-2016.03.04'],
sorted(curator.get_indices(client))
)
def test_do_something_with_int_value(self):
self.create_indices(10)
evar = random_envvar(8)
os.environ[evar] = "1234"
dollar = '${' + evar + '}'
self.write_config(
self.args['configfile'],
testvars.client_config_envvars.format(host, port, dollar)
)
cfg = curator.get_yaml(self.args['configfile'])
self.assertEqual(
cfg['client']['timeout'],
os.environ.get(evar)
)
self.write_config(self.args['actionfile'],
testvars.delete_proto.format(
'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' '
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEquals(5, len(curator.get_indices(self.client)))
del os.environ[evar]
def test_delete_in_period(self):
# filtertype: {0}
# source: {1}
# range_from: {2}
# range_to: {3}
# timestring: {4}
# unit: {5}
# field: {6}
# stats_result: {7}
# intersect: {8}
# epoch: {9}
# week_starts_on: {10}
self.create_indices(10)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.delete_period_proto.format('period', 'name', '-5', '-1',
"'%Y.%m.%d'", 'days', ' ', ' ',
' ', ' ', 'monday'))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEqual(0, result.exit_code)
self.assertEquals(5, len(curator.get_indices(self.client)))
def test_delete_indices_by_space_dry_run(self):
for i in range(1,10):
self.client.create(
index="index" + str(i), doc_type='log',
body={'message':'TEST DOCUMENT'},
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--dry-run',
'--host', host,
'--port', str(port),
'delete',
'--disk-space', '0.0000001',
'indices',
'--all-indices',
],
obj={"filters":[]})
l = curator.get_indices(self.client)
self.assertEquals(9, len(l))
output = sorted(result.output.splitlines(), reverse=True)
# I tried doing a nested, double list comprehension here.
# It works in the interpreter, but not here for some reason.
output = [ x.split(' ')[-1:] for x in output ]
output = [ x[0] for x in output if x[0].startswith('index') ]
self.assertEqual(sorted(l, reverse=True), output)
def test_positive(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_two
client.info.return_value = {'version': {'number': '5.0.0'} }
self.assertEqual(
['index-2016.03.03', 'index-2016.03.04'],
sorted(curator.get_indices(client))
)
def test_positive(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_two
client.info.return_value = {'version': {'number': '2.4.1'} }
self.assertEqual(
['index-2016.03.03', 'index-2016.03.04'],
sorted(curator.get_indices(client))
)
def test_restore_with_rename(self):
indices = []
for i in range(1,4):
self.add_docs('my_index{0}'.format(i))
indices.append('my_index{0}'.format(i))
snap_name = 'snapshot1'
self.create_snapshot(snap_name, ','.join(indices))
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
self.assertEqual(1, len(snapshot['snapshots']))
self.client.indices.delete(','.join(indices))
self.assertEqual([], curator.get_indices(self.client))
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.restore_snapshot_proto.format(
self.args['repository'],
snap_name,
indices,
False,
False,
True,
False,
'my_index(.+)',
'new_index$1',
' ',
True,
False,
301
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
restored_indices = sorted(curator.get_indices(self.client))
self.assertEqual(
['new_index1', 'new_index2', 'new_index3'],
restored_indices
)
def test_shrink(self):
suffix = '-shrunken'
self.builder(
shrink.format('DETERMINISTIC', 'permit_masters', 'True', 1, 0, '',
suffix, 'True'))
indices = curator.get_indices(self.client)
self.assertEqual(1,
len(indices)) # Should only have `my_index-shrunken`
self.assertEqual(indices[0], self.target)
def test_delete_indices_huge_list(self):
self.create_indices(365)
pre = curator.get_indices(self.client)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--logfile', os.devnull,
'--host', host,
'--port', str(port),
'delete',
'indices',
'--all-indices',
'--exclude', pre[0],
],
obj={"filters":[]})
post = curator.get_indices(self.client)
self.assertEquals(1, len(post))
def test_delete_indices_huge_list(self):
self.create_indices(365)
pre = curator.get_indices(self.client)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--logfile', os.devnull,
'--host', host,
'--port', str(port),
'delete',
'indices',
'--all-indices',
'--exclude', pre[0],
],
obj={"filters":[]})
post = curator.get_indices(self.client)
self.assertEquals(1, len(post))
def test_issue_826(self):
client = Mock()
client.info.return_value = {'version': {'number': '2.4.2'} }
client.indices.get_settings.return_value = testvars.settings_two
client.indices.exists.return_value = True
self.assertEqual(
['.security', 'index-2016.03.03', 'index-2016.03.04'],
sorted(curator.get_indices(client))
)
def test_name_older_than_now_cli(self):
self.create_indices(10)
args = self.get_runner_args()
args += [
'--config', self.args['configfile'],
'delete_indices',
'--filter_list', '{"filtertype":"age","source":"name","direction":"older","timestring":"%Y.%m.%d","unit":"days","unit_count":5}',
]
self.assertEqual(0, self.run_subprocess(args, logname='TestCLIDeleteIndices.test_name_older_than_now_cli'))
self.assertEquals(5, len(curator.get_indices(self.client)))
def test_permit_masters_false(self):
suffix = '-shrunken'
self.builder(
no_permit_masters.format('DETERMINISTIC', 1, 0, '', suffix,
'True'))
indices = curator.get_indices(self.client)
self.assertEqual(1,
len(indices)) # Should only have `my_index-shrunken`
self.assertEqual(indices[0], self.idx)
self.assertEqual(self.result.exit_code, 1)
def test_name_older_than_now_cli(self):
self.create_indices(10)
args = self.get_runner_args()
args += [
'--config', self.args['configfile'],
'delete_indices',
'--filter_list', '{"filtertype":"age","source":"name","direction":"older","timestring":"%Y.%m.%d","unit":"days","unit_count":5}',
]
self.assertEqual(0, self.run_subprocess(args, logname='TestCLIDeleteIndices.test_name_older_than_now_cli'))
self.assertEquals(5, len(curator.get_indices(self.client)))
def test_filter_by_array_of_aliases(self):
alias = 'testalias'
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.filter_by_alias.format(' [ testalias, foo ]', False))
self.create_index('my_index')
self.create_index('dummy')
self.client.indices.put_alias(index='dummy', name=alias)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
ver = curator.get_version(self.client)
if ver >= (5, 5, 0):
self.assertEquals(2, len(curator.get_indices(self.client)))
else:
self.assertEquals(1, len(curator.get_indices(self.client)))
def test_shrink_with_copy_alias(self):
suffix = '-shrunken'
self.builder(
copy_aliases.format('DETERMINISTIC', 'permit_masters', 'True', 1,
0, '', suffix, 'True', 'True'))
indices = curator.get_indices(self.client)
self.assertEqual(1, len(indices)) # Should have `my_index-shrunken`
self.assertEqual(indices[0], self.target)
self.assertTrue(
self.client.indices.exists_alias(index=self.target,
name=self.alias))
def test_extra_option(self):
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.bad_option_proto_test.format('rollover'))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEqual([], curator.get_indices(self.client))
self.assertEqual(-1, _.exit_code)
def test_filter_by_array_of_aliases(self):
alias = 'testalias'
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.filter_by_alias.format(' [ testalias, foo ]', False))
self.create_index('my_index')
self.create_index('dummy')
self.client.indices.put_alias(index='dummy', name=alias)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
ver = curator.get_version(self.client)
if ver >= (5,5,0):
self.assertEquals(2, len(curator.get_indices(self.client)))
else:
self.assertEquals(1, len(curator.get_indices(self.client)))
def test_logging_with_debug_flag(self):
self.create_indices(10)
indices = curator.get_indices(self.client)
expected = sorted(indices, reverse=True)[:4]
test = clicktest.CliRunner()
result = test.invoke(curator.cli, [
'--debug', '--logfile', os.devnull, '--host', host, '--port',
str(port), 'show', 'indices', '--newer-than', '5', '--timestring',
'%Y.%m.%d', '--time-unit', 'days'
],
obj={"filters": []})
output = sorted(result.output.splitlines(), reverse=True)[:4]
self.assertEqual(expected, output)
def filter_indices(module, client):
prefix = module.params.get('prefix')
older_than = module.params.get('older_than')
time_unit = module.params.get('time_unit')
timestring = module.params.get('timestring')
indices = curator.get_indices(client)
_filter = curator.build_filter(kindOf='prefix', value=prefix)
indices = curator.apply_filter(indices, **_filter)
_filter = curator.build_filter(kindOf='older_than', value=int(older_than), time_unit=time_unit, timestring=timestring)
return curator.apply_filter(indices, **_filter)
def test_cli_show_indices_older_than_zero(self):
self.create_indices(10)
indices = curator.get_indices(self.client)
expected = sorted(indices, reverse=True)
test = clicktest.CliRunner()
result = test.invoke(curator.cli, [
'--logfile', os.devnull, '--host', host, '--port',
str(port), 'show', 'indices', '--older-than', '0', '--timestring',
'%Y.%m.%d', '--time-unit', 'days'
],
obj={"filters": []})
output = sorted(result.output.splitlines(), reverse=True)
self.assertEqual(expected, output)
def main():
global logger
parser = argparse.ArgumentParser()
parser.add_argument("-c", default="dopey.yaml", help="yaml config")
parser.add_argument("-l", default="-", help="log file")
parser.add_argument("--level", default="info")
args = parser.parse_args()
config = yaml.load(open(args.c))
initlog(level=args.level, log=config["l"] if "log" in config else args.l)
logger = logging.getLogger("dopey")
eshosts = config.get("esclient")
logger.debug(eshosts)
if eshosts is not None:
esclient = elasticsearch.Elasticsearch(eshosts, timeout=300)
else:
esclient = elasticsearch.Elasticsearch(timeout=300)
all_indices = curator.get_indices(esclient)
logger.debug("all_indices: {}".format(all_indices))
if all_indices is False:
raise Exception("could not get indices")
process_threads = []
for index_prefix, index_config in config.get("indices").items():
t = Thread(target=process,
args=(esclient, all_indices, index_prefix, index_config))
t.start()
process_threads.append(t)
for t in process_threads:
t.join()
not_dealt = list(set(all_indices).difference(_dealt))
dopey_summary.add(
json.dumps(
{
"not_dealt": not_dealt,
"delete": _delete,
"close": _close,
"optimize": _optimize
},
indent=2))
sumary_config = config.get("sumary")
for action, kargs in sumary_config.items():
if kargs:
getattr(dopey_summary, action)(**kargs)
else:
getattr(dopey_summary, action)()
def test_action_disabled(self):
self.create_indices(10)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.disabled_proto.format('close', 'delete_indices'))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEquals(0, len(curator.get_indices(self.client)))
self.assertEqual(0, result.exit_code)
def test_show_indices_with_suffix_and_no_prefix(self):
client = Mock()
client.indices.get_settings.return_value = {
'2014.01.03-suffix': True,
'2014.01.02-suffix': True,
'2014.01.01-suffix': True
}
indices = curator.get_indices(client, prefix='', suffix='-suffix')
self.assertEquals([
'2014.01.01-suffix',
'2014.01.02-suffix',
'2014.01.03-suffix',
], indices)
def test_with_extra_settings(self):
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.create_index_with_extra_settings.format('testing'))
self.assertEqual([], curator.get_indices(self.client))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
ilo = curator.IndexList(self.client)
self.assertEqual(['testing'], ilo.indices)
self.assertEqual(ilo.index_info['testing']['number_of_shards'], '1')
self.assertEqual(ilo.index_info['testing']['number_of_replicas'], '0')
def test_extra_option(self):
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.bad_option_proto_test.format('rollover'))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEqual([], curator.get_indices(self.client))
self.assertEqual(-1, result.exit_code)
def test_action_disabled(self):
self.create_indices(10)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.disabled_proto.format('close', 'delete_indices'))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEquals(0, len(curator.get_indices(self.client)))
self.assertEqual(0, result.exit_code)
def test_permit_masters_false(self):
suffix = '-shrunken'
self.builder(
no_permit_masters.format(
'DETERMINISTIC',
1,
0,
'',
suffix,
'True'
)
)
indices = curator.get_indices(self.client)
self.assertEqual(1, len(indices)) # Should only have `my_index-shrunken`
self.assertEqual(indices[0], self.idx)
self.assertEqual(self.result.exit_code, 1)
def test_show_indices(self):
client = Mock()
client.indices.get_settings.return_value = {
'prefix-2014.01.03': True,
'prefix-2014.01.02': True,
'prefix-2014.01.01': True
}
indices = curator.get_indices(client, prefix='prefix-')
self.assertEquals([
'prefix-2014.01.01',
'prefix-2014.01.02',
'prefix-2014.01.03',
],
indices
)
def test_show_indices_with_suffix_and_no_prefix(self):
client = Mock()
client.indices.get_settings.return_value = {
'2014.01.03-suffix': True,
'2014.01.02-suffix': True,
'2014.01.01-suffix': True
}
indices = curator.get_indices(client, prefix='', suffix='-suffix')
self.assertEquals([
'2014.01.01-suffix',
'2014.01.02-suffix',
'2014.01.03-suffix',
],
indices
)
def test_reindex_empty_list(self):
wait_interval = 1
max_wait = 3
source = 'my_source'
dest = 'my_dest'
expected = '.tasks'
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.reindex.format(wait_interval, max_wait, source, dest))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEqual(expected, curator.get_indices(self.client)[0])
def test_index_selection_only_timestamp_filter(self):
self.create_indices(10)
indices = curator.get_indices(self.client)
# expected = sorted(indices, reverse=True)[:4]
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--logfile', os.devnull,
'--host', host,
'--port', str(port),
'close',
'indices',
'--timestring', '%Y.%m.%d',
],
obj={"filters":[]})
self.assertEqual(0, result.exit_code)