def test_exclude(self):
key = 'tag'
value = 'value'
at = 'exclude'
ver = curator.get_version(self.client)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.allocation_test.format(key, value, at, False))
self.create_index('my_index')
self.create_index('not_my_index')
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEquals(
value,
self.client.indices.get_settings(index='my_index')['my_index']
['settings']['index']['routing']['allocation'][at][key])
if ver >= (7, 10, 0):
self.assertEquals(
EMPTY710ROUTING,
self.client.indices.get_settings(index='not_my_index')
['not_my_index']['settings']['index']['routing'])
else:
self.assertNotIn(
'routing',
self.client.indices.get_settings(
index='not_my_index')['not_my_index']['settings']['index'])
def test_allow_ilm_indices_false(self):
# ILM will not be added until 6.4
if curator.get_version(self.client) < (6,4,0):
self.assertTrue(True)
else:
self.create_indices(10)
settings = {
'index': {
'lifecycle': {
'name': 'mypolicy'
}
}
}
self.client.indices.put_settings(index='_all', body=settings)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.ilm_delete_proto.format(
'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'false'
)
)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
[ '--config', self.args['configfile'], self.args['actionfile'] ],
)
self.assertEquals(10, len(curator.get_indices(self.client)))
def test_is_write_alias(self):
oldindex = 'rolltome-000001'
newindex = 'rolltome-000002'
alias = 'delamitri'
condition = 'max_age'
value = '1s'
ver = curator.get_version(self.client)
if ver >= (6, 5, 0):
request_body = {'aliases': {alias: {'is_write_index': True}}}
expected = 2
else:
request_body = {'aliases': {alias: {}}}
expected = 1
self.client.indices.create(index=oldindex, body=request_body)
time.sleep(1)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.rollover_one.format(alias, condition, value))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEqual(expected,
len(self.client.indices.get_alias(name=alias)))
def test_reindex_bad_mapping(self):
# This test addresses GitHub issue #1260
wait_interval = 1
max_wait = 3
source = 'my_source'
dest = 'my_dest'
expected = 1
ver = curator.get_version(self.client)
if ver < (7, 0, 0):
request_body = {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"doc": {
"properties": {
"doc1": {
"type": "keyword"
}
}
}
}
}
else:
request_body = {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"doc1": {
"type": "keyword"
}
}
}
}
self.client.indices.create(index=source, body=request_body)
self.add_docs(source)
# Create the dest index with a different mapping.
if ver < (7, 0, 0):
request_body['mappings']['doc']['properties']['doc1'][
'type'] = 'integer'
else:
request_body['mappings']['properties']['doc1']['type'] = 'integer'
self.client.indices.create(index=dest, body=request_body)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.reindex.format(wait_interval, max_wait, source, dest))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEqual(expected, _.exit_code)
def check_version(client):
"""
Verify version is within acceptable range. Exit with error if it is not.
:arg client: The Elasticsearch client connection
"""
version_number = curator.get_version(client)
logger.debug('Detected Elasticsearch version {0}'.format(".".join(map(str,version_number))))
if version_number >= version_max or version_number < version_min:
print('Expected Elasticsearch version range > {0} < {1}'.format(".".join(map(str,version_min)),".".join(map(str,version_max))))
print('ERROR: Incompatible with version {0} of Elasticsearch. Exiting.'.format(".".join(map(str,version_number))))
sys.exit(1)
def test_count_indices_by_age_same_age(self):
key = 'tag'
value = 'value'
at = 'include'
ver = curator.get_version(self.client)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.allocation_count_test.format(key, value, at, False))
self.create_index('c-2017.10.01')
self.create_index('c-2017.10.02')
self.create_index('c-2017.10.03')
self.create_index('a-2017.10.01')
self.create_index('a-2017.10.02')
self.create_index('a-2017.10.03')
self.create_index('b-2017.10.01')
self.create_index('b-2017.10.02')
self.create_index('b-2017.10.03')
self.create_index('d-2017.10.01')
self.create_index('d-2017.10.02')
self.create_index('d-2017.10.03')
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEquals(
value,
self.client.indices.get_settings(
index='c-2017.10.03')['c-2017.10.03']['settings']['index']
['routing']['allocation'][at][key])
self.assertEquals(
value,
self.client.indices.get_settings(
index='d-2017.10.03')['d-2017.10.03']['settings']['index']
['routing']['allocation'][at][key])
idxlist = [
'a-2017.10.01', 'a-2017.10.02', 'a-2017.10.03', 'b-2017.10.01',
'b-2017.10.02', 'b-2017.10.03', 'c-2017.10.01', 'c-2017.10.02',
'd-2017.10.01', 'd-2017.10.02'
]
for idx in idxlist:
if ver >= (7, 10, 0):
self.assertEquals(
EMPTY710ROUTING,
self.client.indices.get_settings(
index=idx)[idx]['settings']['index']['routing'])
else:
self.assertNotIn(
'routing',
self.client.indices.get_settings(
index=idx)[idx]['settings']['index'])
def check_version(client):
"""
Verify version is within acceptable range. Exit with error if it is not.
:arg client: The Elasticsearch client connection
"""
version_number = curator.get_version(client)
logger.debug('Detected Elasticsearch version {0}'.format(".".join(map(str,version_number))))
if version_number >= version_max or version_number < version_min:
print('Expected Elasticsearch version range > {0} < {1}'.format(".".join(map(str,version_min)),".".join(map(str,version_max))))
print('ERROR: Incompatible with version {0} of Elasticsearch. Exiting.'.format(".".join(map(str,version_number))))
sys.exit(1)
def add_docs(self, idx):
for i in ["1", "2", "3"]:
ver = get_version(self.client)
if ver >= (7, 0, 0):
self.client.create(
index=idx, doc_type='_doc', id=i, body={"doc" + i :'TEST DOCUMENT'})
else:
self.client.create(
index=idx, doc_type='doc', id=i, body={"doc" + i :'TEST DOCUMENT'})
# This should force each doc to be in its own segment.
# pylint: disable=E1123
self.client.indices.flush(index=idx, force=True)
self.client.indices.refresh(index=idx)
def add_docs(self, idx):
for i in ["1", "2", "3"]:
ver = get_version(self.client)
if ver >= (7, 0, 0):
self.client.create(index=idx,
doc_type='_doc',
id=i,
body={"doc" + i: 'TEST DOCUMENT'})
else:
self.client.create(index=idx,
doc_type='doc',
id=i,
body={"doc" + i: 'TEST DOCUMENT'})
# This should force each doc to be in its own segment.
# pylint: disable=E1123
self.client.indices.flush(index=idx, force=True)
self.client.indices.refresh(index=idx)
def test_remove_exclude_with_none_value(self):
key = 'tag'
value = ''
at = 'exclude'
ver = curator.get_version(self.client)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.allocation_test.format(key, value, at, False))
self.create_index('my_index')
self.create_index('not_my_index')
# Put a setting in place before we start the test.
self.client.indices.put_settings(
index='my_index',
body={'index.routing.allocation.{0}.{1}'.format(at, key): 'bar'})
# Ensure we _have_ it here first.
self.assertEquals(
'bar',
self.client.indices.get_settings(index='my_index')['my_index']
['settings']['index']['routing']['allocation'][at][key])
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
if ver >= (7, 10, 0):
self.assertEquals(
EMPTY710ROUTING,
self.client.indices.get_settings(index='my_index')['my_index']
['settings']['index']['routing'])
self.assertEquals(
EMPTY710ROUTING,
self.client.indices.get_settings(index='not_my_index')
['not_my_index']['settings']['index']['routing'])
else:
self.assertNotIn(
'routing',
self.client.indices.get_settings(
index='my_index')['my_index']['settings']['index'])
self.assertNotIn(
'routing',
self.client.indices.get_settings(
index='not_my_index')['not_my_index']['settings']['index'])
def test_filter_by_array_of_aliases(self):
alias = 'testalias'
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.filter_by_alias.format(' [ testalias, foo ]', False))
self.create_index('my_index')
self.create_index('dummy')
self.client.indices.put_alias(index='dummy', name=alias)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
ver = curator.get_version(self.client)
if ver >= (5, 5, 0):
self.assertEquals(2, len(curator.get_indices(self.client)))
else:
self.assertEquals(1, len(curator.get_indices(self.client)))
def test_allow_ilm_indices_false(self):
# ILM will not be added until 6.6
if curator.get_version(self.client) < (6, 6, 0):
self.assertTrue(True)
else:
import requests
name = 'test'
policy = {
'policy': {
'phases': {
'hot': {
'min_age': '0ms',
'actions': {
'rollover': {
'max_age': '2h',
'max_docs': 4
}
}
}
}
}
}
url = 'http://{0}:{1}/_ilm/policy/{2}'.format(host, port, name)
r = requests.put(url, json=policy)
# print(r.text) # logging reminder
self.create_indices(10, ilm_policy=name)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.ilm_delete_proto.format('age', 'name', 'older',
'\'%Y.%m.%d\'', 'days', 5,
' ', ' ', ' ', 'false'))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEquals(
10, len(exclude_ilm_history(curator.get_indices(self.client))))
def test_reindex_selected_many_to_one(self):
wait_interval = 1
max_wait = 3
source1 = 'my_source1'
source2 = 'my_source2'
dest = 'my_dest'
expected = 6
self.create_index(source1)
self.add_docs(source1)
self.create_index(source2)
for i in ["4", "5", "6"]:
ver = curator.get_version(self.client)
if ver >= (7, 0, 0):
self.client.create(index=source2,
doc_type='doc',
id=i,
body={"doc" + i: 'TEST DOCUMENT'})
else:
self.client.create(index=source2,
doc_type='doc',
id=i,
body={"doc" + i: 'TEST DOCUMENT'})
# Decorators make this pylint exception necessary
# pylint: disable=E1123
self.client.indices.flush(index=source2, force=True)
self.client.indices.refresh(index=source2)
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.reindex.format(wait_interval, max_wait,
'REINDEX_SELECTION', dest))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.client.indices.refresh(index=dest)
self.assertEqual(expected, self.client.count(index=dest)['count'])
def test_include(self):
key = 'tag'
value = 'value'
at = 'include'
ver = curator.get_version(self.client)
self.create_index('my_index')
self.create_index('not_my_index')
args = self.get_runner_args()
args += [
'--config',
self.args['configfile'],
'allocation',
'--key',
key,
'--value',
value,
'--allocation_type',
at,
'--wait_for_completion',
'--filter_list',
'{"filtertype":"pattern","kind":"prefix","value":"my"}',
]
self.assertEqual(
0,
self.run_subprocess(args,
logname='TestCLIAllocation.test_include'))
self.assertEquals(
value,
self.client.indices.get_settings(index='my_index')['my_index']
['settings']['index']['routing']['allocation'][at][key])
if ver >= (7, 10, 0):
self.assertEquals(
EMPTY710ROUTING,
self.client.indices.get_settings(index='not_my_index')
['not_my_index']['settings']['index']['routing'])
else:
self.assertNotIn(
'routing',
self.client.indices.get_settings(
index='not_my_index')['not_my_index']['settings']['index'])
def test_allow_ilm_indices_false(self):
# ILM will not be added until 6.6
if curator.get_version(self.client) < (6,6,0):
self.assertTrue(True)
else:
import requests
name = 'test'
policy = {
'policy': {
'phases': {
'hot': {
'min_age': '0ms',
'actions': {
'rollover': {
'max_age': '2h',
'max_docs': 4
}
}
}
}
}
}
url = 'http://{0}:{1}/_ilm/policy/{2}'.format(host, port, name)
r = requests.put(url, json=policy)
# print(r.text) # logging reminder
self.create_indices(10, ilm_policy=name)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.ilm_delete_proto.format(
'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'false'
)
)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
[ '--config', self.args['configfile'], self.args['actionfile'] ],
)
self.assertEquals(10, len(curator.get_indices(self.client)))
def test_add_only_skip_closed(self):
alias = 'testalias'
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.alias_add_only.format(alias))
self.create_index('my_index')
self.client.indices.close(index='my_index')
self.create_index('dummy')
self.client.indices.put_alias(index='dummy', name=alias)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
version = curator.get_version(self.client)
if version > (3, 0, 0):
self.assertEquals(2,
len(self.client.indices.get_alias(name=alias)))
else:
self.assertEquals(1,
len(self.client.indices.get_alias(name=alias)))
def test_filter_by_array_of_aliases(self):
alias = 'testalias'
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.filter_by_alias.format(' [ testalias, foo ]', False))
self.create_index('my_index')
self.create_index('dummy')
self.client.indices.put_alias(index='dummy', name=alias)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
ver = curator.get_version(self.client)
if ver >= (5,5,0):
self.assertEquals(2, len(curator.get_indices(self.client)))
else:
self.assertEquals(1, len(curator.get_indices(self.client)))
def test_add_only_skip_closed(self):
alias = 'testalias'
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.alias_add_only.format(alias))
self.create_index('my_index')
self.client.indices.close(index='my_index')
self.create_index('dummy')
self.client.indices.put_alias(index='dummy', name=alias)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
version = curator.get_version(self.client)
if version > (3,0,0):
self.assertEquals(2, len(self.client.indices.get_alias(name=alias)))
else:
self.assertEquals(1, len(self.client.indices.get_alias(name=alias)))
def test_reindex_bad_mapping(self):
# This test addresses GitHub issue #1260
wait_interval = 1
max_wait = 3
source = 'my_source'
dest = 'my_dest'
expected = 1
ver = curator.get_version(self.client)
if ver < (7, 0, 0):
request_body = {
"settings": { "number_of_shards": 1, "number_of_replicas": 0},
"mappings": { "doc": { "properties": { "doc1": { "type": "keyword" }}}}
}
else:
request_body = {
"settings": { "number_of_shards": 1, "number_of_replicas": 0},
"mappings": { "properties": { "doc1": { "type": "keyword" }}}
}
self.client.indices.create(index=source, body=request_body)
self.add_docs(source)
# Create the dest index with a different mapping.
if ver < (7, 0, 0):
request_body['mappings']['doc']['properties']['doc1']['type'] = 'integer'
else:
request_body['mappings']['properties']['doc1']['type'] = 'integer'
self.client.indices.create(index=dest, body=request_body)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.reindex.format(wait_interval, max_wait, source, dest))
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.assertEqual(expected, _.exit_code)
def test_reindex_selected_many_to_one(self):
wait_interval = 1
max_wait = 3
source1 = 'my_source1'
source2 = 'my_source2'
dest = 'my_dest'
expected = 6
self.create_index(source1)
self.add_docs(source1)
self.create_index(source2)
for i in ["4", "5", "6"]:
ver = curator.get_version(self.client)
if ver >= (7, 0, 0):
self.client.create(
index=source2, doc_type='doc', id=i, body={"doc" + i :'TEST DOCUMENT'})
else:
self.client.create(
index=source2, doc_type='doc', id=i, body={"doc" + i :'TEST DOCUMENT'})
# Decorators make this pylint exception necessary
# pylint: disable=E1123
self.client.indices.flush(index=source2, force=True)
self.client.indices.refresh(index=source2)
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.reindex.format(wait_interval, max_wait, 'REINDEX_SELECTION', dest)
)
test = clicktest.CliRunner()
_ = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
self.client.indices.refresh(index=dest)
self.assertEqual(expected, self.client.count(index=dest)['count'])
def test_dev_version_with_dash(self):
client = Mock()
client.info.return_value = {'version': {'number': '9.9.9-dev'}}
version = curator.get_version(client)
self.assertEqual(version, (9, 9, 9))
def test_negative(self):
client = Mock()
client.info.return_value = {'version': {'number': '9.9.9'}}
version = curator.get_version(client)
self.assertNotEqual(version, (8, 8, 8))
def test_dev_version_with_dash(self):
client = Mock()
client.info.return_value = {'version': {'number': '9.9.9-dev'} }
version = curator.get_version(client)
self.assertEqual(version, (9,9,9))
def test_negative(self):
client = Mock()
client.info.return_value = {'version': {'number': '9.9.9'} }
version = curator.get_version(client)
self.assertNotEqual(version, (8,8,8))
class TestCLIFixFor687(CuratorTestCase):
@unittest.skipIf(
curator.get_version(global_client) >= (3, 0, 0),
'not supported for this version of ES')
def test_fix_for_687(self):
self.create_repository()
snap_name = 'test687'
# 7 june (week 23)
self.client.indices.create(index='logstash-2016.23',
body={
'settings': {
'creation_date': 1465293737000,
'number_of_shards': 1,
'number_of_replicas': 0
}
})
# 14 june (week 24)
self.client.indices.create(index='logstash-2016.24',
body={
'settings': {
'creation_date': 1465898537000,
'number_of_shards': 1,
'number_of_replicas': 0
}
})
# 21 june (week 25)
self.client.indices.create(index='logstash-2016.25',
body={
'settings': {
'creation_date': 1466503337000,
'number_of_shards': 1,
'number_of_replicas': 0
}
})
# 28 july (week 26)
self.client.indices.create(index='logstash-2016.26',
body={
'settings': {
'creation_date': 1467108137000,
'number_of_shards': 1,
'number_of_replicas': 0
}
})
# 5 july (week 27)
self.client.indices.create(index='logstash-2016.27',
body={
'settings': {
'creation_date': 1467712937000,
'number_of_shards': 1,
'number_of_replicas': 0
}
})
# 12 july (week 28)
self.client.indices.create(index='logstash-2016.28',
body={
'settings': {
'creation_date': 1468317737000,
'number_of_shards': 1,
'number_of_replicas': 0
}
})
self.client.cluster.health(wait_for_status='yellow')
self.write_config(self.args['configfile'],
testvars.client_config.format(host, port))
self.write_config(
self.args['actionfile'],
testvars.test_687.format(self.args['repository'], snap_name))
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
['--config', self.args['configfile'], self.args['actionfile']],
)
snapshot = curator.get_snapshot(self.client, self.args['repository'],
'_all')
self.assertEquals(6, len(curator.get_indices(self.client)))
