class TestCommandSetupILMPolicy(BaseTest):
"""
Test beat command `setup` related to ILM policy
"""
def setUp(self):
super(TestCommandSetupILMPolicy, self).setUp()
self.setupCmd = "--index-management"
self.alias_name = self.index_name = self.beat_name + "-9.9.9"
self.policy_name = self.beat_name
self.custom_alias = self.beat_name + "_foo"
self.custom_policy = self.beat_name + "_bar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.index_name)
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
policies=[self.policy_name, self.custom_policy])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
policies=[self.policy_name, self.custom_policy])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
es_template_name=self.index_name,
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_ilm_policy_and_template(self):
"""
Test combination of ilm policy and template setup
"""
self.render_config()
# NOTE: --template is deprecated for 8.0.0./
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd, "--template"])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_ilm_default(self):
"""
Test ilm policy setup with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"])
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_ilm_disabled(self):
"""
Test ilm policy setup when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_policy_name(self):
"""
Test ilm policy setup when policy_name is configured
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.policy_name=" + self.custom_policy])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name)
self.idxmgmt.assert_policy_created(self.custom_policy)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_rollover_alias(self):
"""
Test ilm policy setup when rollover_alias is configured
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.rollover_alias=" + self.custom_alias])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
self.idxmgmt.assert_alias_created(self.custom_alias)
class TestCommandSetupTemplate(BaseTest):
"""
Test beat command `setup` with focus on template
"""
def setUp(self):
super(TestCommandSetupTemplate, self).setUp()
# auto-derived default settings, if nothing else is set
self.setupCmd = "--template"
self.index_name = self.beat_name + "-9.9.9"
self.custom_alias = self.beat_name + "_foo"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.index_name)
self.idxmgmt.delete(index=self.custom_alias)
self.idxmgmt.delete(index=self.index_name)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(index=self.custom_alias)
self.idxmgmt.delete(index=self.index_name)
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup(self):
"""
Test setup cmd with template and ilm-policy subcommands
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd, "--ilm-policy"])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name)
self.idxmgmt.assert_alias_created(self.index_name)
self.idxmgmt.assert_policy_created(self.index_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_default(self):
"""
Test template setup with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name)
self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"])
# when running `setup --template`
# write_alias and rollover_policy related to ILM are also created
self.idxmgmt.assert_alias_created(self.index_name)
self.idxmgmt.assert_policy_created(self.index_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_disabled(self):
"""
Test template setup when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.template.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_not_loaded(self.index_name)
# when running `setup --template` and `setup.template.enabled=false`
# write_alias and rollover_policy related to ILM are still created
self.idxmgmt.assert_alias_created(self.index_name)
self.idxmgmt.assert_policy_created(self.index_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_with_opts(self):
"""
Test template setup with config options
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.settings.index.number_of_shards=2"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
# check that settings are overwritten
resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name)
assert self.index_name in resp
index = resp[self.index_name]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_with_ilm_changed_pattern(self):
"""
Test template setup with changed ilm.rollover_alias config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.rollover_alias=" + self.custom_alias])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias)
self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"])
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_template_created_on_ilm_policy_created(self):
"""
Test template setup overwrites template when new ilm policy is created
"""
# ensure template with ilm rollover_alias name is created, but ilm policy not yet
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.name=" + self.custom_alias,
"-E", "setup.template.pattern=" + self.custom_alias + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_alias)
self.idxmgmt.assert_policy_not_created(self.index_name)
# ensure ilm policy is created, triggering overwriting existing template
exit_code = self.run_beat(extra_args=["setup", self.setupCmd,
"-E", "setup.template.overwrite=false",
"-E", "setup.template.settings.index.number_of_shards=2",
"-E", "setup.ilm.rollover_alias=" + self.custom_alias])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias)
self.idxmgmt.assert_policy_created(self.index_name)
# check that template was overwritten
resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias)
assert self.custom_alias in resp
index = resp[self.custom_alias]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
class TestRunILM(BaseTest):
def setUp(self):
super(TestRunILM, self).setUp()
self.alias_name = self.index_name = self.beat_name + "-9.9.9"
self.policy_name = self.beat_name
self.custom_alias = self.beat_name + "_foo"
self.custom_policy = self.beat_name + "_bar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.index_name)
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
policies=[self.policy_name, self.custom_policy])
def tearDown(self):
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy],
policies=[self.policy_name, self.custom_policy])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
es_template_name=self.index_name,
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_ilm_default(self):
"""
Test ilm default settings to load ilm policy, write alias and ilm template
"""
self.render_config()
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_ilm_disabled(self):
"""
Test ilm disabled to not load ilm related components
"""
self.render_config(ilm={"enabled": False})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_policy_name(self):
"""
Test setting ilm policy name
"""
policy_name = self.beat_name + "_foo"
self.render_config(ilm={"enabled": True, "policy_name": policy_name})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, policy_name, self.alias_name)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
self.idxmgmt.assert_policy_created(policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_rollover_alias(self):
"""
Test settings ilm rollover alias
"""
self.render_config(ilm={"enabled": True, "rollover_alias": self.custom_alias})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
self.idxmgmt.assert_docs_written_to_alias(self.custom_alias)
self.idxmgmt.assert_alias_created(self.custom_alias)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_pattern(self):
"""
Test setting ilm pattern
"""
pattern = "1"
self.render_config(ilm={"enabled": True, "pattern": pattern})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name, pattern=pattern)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=pattern)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_pattern_date(self):
"""
Test setting ilm pattern with date
"""
pattern = "'{now/d}'"
self.render_config(ilm={"enabled": True, "pattern": pattern})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
resolved_pattern = datetime.datetime.now().strftime("%Y.%m.%d")
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name, pattern=resolved_pattern)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=resolved_pattern)
class TestCommandSetupTemplate(BaseTest):
"""
Test beat command `setup` with focus on template
"""
def setUp(self):
super(TestCommandSetupTemplate, self).setUp()
# auto-derived default settings, if nothing else is set
self.setupCmd = "--index-management"
self.data_stream = self.beat_name + "-9.9.9"
self.policy_name = self.beat_name
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.data_stream)
self.idxmgmt.delete(indices=[self.data_stream],
policies=[self.policy_name])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(indices=[self.data_stream],
policies=[self.policy_name])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch=self.get_elasticsearch_template_config(), **kwargs)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup(self):
"""
Test setup cmd with template and ilm-policy subcommands
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_default(self):
"""
Test template setup with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_index_template_index_pattern(
self.data_stream, [self.data_stream])
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_disabled(self):
"""
Test template setup when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.setupCmd, "-E",
"setup.template.enabled=false"
])
assert exit_code == 0
self.idxmgmt.assert_index_template_not_loaded(self.data_stream)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_with_opts(self):
"""
Test template setup with config options
"""
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.setupCmd, "-E", "setup.ilm.enabled=false", "-E",
"setup.template.settings.index.number_of_shards=2"
])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
# check that settings are overwritten
resp = self.es.transport.perform_request(
'GET', '/_index_template/' + self.data_stream)
found = False
for index_template in resp["index_templates"]:
if self.data_stream == index_template["name"]:
found = True
index = index_template["index_template"]["template"][
"settings"]["index"]
assert index["number_of_shards"] == "2", index[
"number_of_shards"]
assert found
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_template_created_on_ilm_policy_created(self):
"""
Test template setup overwrites template when new ilm policy is created
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.setupCmd, "-E",
"setup.ilm.enabled=false"
])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_not_created(self.policy_name)
# ensure ilm policy is created, triggering overwriting existing template
exit_code = self.run_beat(extra_args=[
"setup", self.setupCmd, "-E", "setup.template.overwrite=false",
"-E", "setup.template.settings.index.number_of_shards=2"
])
assert exit_code == 0
self.idxmgmt.assert_policy_created(self.policy_name)
class TestRunILM(BaseTest):
def setUp(self):
super(TestRunILM, self).setUp()
self.data_stream = self.beat_name + "-9.9.9"
self.policy_name = self.beat_name
self.custom_policy = self.beat_name + "_bar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.data_stream)
self.idxmgmt.delete(indices=[],
policies=[self.policy_name, self.custom_policy],
data_streams=[self.data_stream])
def tearDown(self):
self.idxmgmt.delete(indices=[],
policies=[self.policy_name, self.custom_policy],
data_streams=[self.data_stream])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
es_template_name=self.data_stream,
**kwargs)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_ilm_default(self):
"""
Test ilm default settings to load ilm policy, data stream template
"""
self.render_config()
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains(MSG_ILM_POLICY_LOADED))
self.wait_until(lambda: self.log_contains(
"PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_data_stream_created(self.data_stream)
self.idxmgmt.assert_policy_created(self.policy_name)
self.idxmgmt.assert_docs_written_to_data_stream(self.data_stream)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_ilm_disabled(self):
"""
Test ilm disabled to not load ilm related components
"""
self.render_config(ilm={"enabled": False})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains(
"PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_not_created(self.policy_name)
self.idxmgmt.assert_docs_written_to_data_stream(self.data_stream)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_policy_name(self):
"""
Test setting ilm policy name
"""
policy_name = self.beat_name + "_foo"
self.render_config(ilm={"enabled": True, "policy_name": policy_name})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains(MSG_ILM_POLICY_LOADED))
self.wait_until(lambda: self.log_contains(
"PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_docs_written_to_data_stream(self.data_stream)
self.idxmgmt.assert_policy_created(policy_name)
class TestCommandSetupIndexManagement(BaseTest):
"""
Test beat command `setup` related to ILM policy
"""
def setUp(self):
super(TestCommandSetupIndexManagement, self).setUp()
self.cmd = "--index-management"
# auto-derived default settings, if nothing else is set
self.policy_name = self.beat_name
self.data_stream = self.beat_name + "-9.9.9"
self.custom_policy = self.beat_name + "_bar"
self.custom_template = self.beat_name + "_foobar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.data_stream)
self.idxmgmt.delete(indices=[],
policies=[self.policy_name, self.custom_policy],
data_streams=[self.data_stream])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(indices=[],
policies=[self.policy_name, self.custom_policy],
data_streams=[self.data_stream])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
es_template_name=self.data_stream,
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_default(self):
"""
Test setup --index-management with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_index_template_index_pattern(self.data_stream, [self.data_stream])
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_disabled(self):
"""
Test setup --index-management when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-e", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.template.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_not_loaded(self.data_stream+"ba")
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_ilm_disabled(self):
"""
Test setup --index-management when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_policy_name(self):
"""
Test setup --index-management when policy_name is configured
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.policy_name=" + self.custom_policy])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_created(self.custom_policy)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_ilm_policy_no_overwrite(self):
"""
Test setup --index-management respects overwrite configuration
"""
policy_name = "mockbeat-test"
# update policy to verify overwrite behaviour
self.es.transport.perform_request('PUT', '/_ilm/policy/' + policy_name,
body={
"policy": {
"phases": {
"delete": {
"actions": {
"delete": {}
}
}
}
}
})
resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
assert "delete" in resp[policy_name]["policy"]["phases"]
assert "hot" not in resp[policy_name]["policy"]["phases"]
# ensure ilm policy is not overwritten
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=true",
"-E", "setup.ilm.overwrite=false",
"-E", "setup.ilm.policy_name=" + policy_name])
assert exit_code == 0
resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
assert "delete" in resp[policy_name]["policy"]["phases"]
assert "hot" not in resp[policy_name]["policy"]["phases"]
# ensure ilm policy is overwritten
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=true",
"-E", "setup.ilm.overwrite=true",
"-E", "setup.ilm.policy_name=" + policy_name])
assert exit_code == 0
resp = self.es.transport.perform_request('GET', '/_ilm/policy/' + policy_name)
assert "delete" not in resp[policy_name]["policy"]["phases"]
assert "hot" in resp[policy_name]["policy"]["phases"]
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_name_and_pattern_on_ilm_disabled(self):
"""
Test setup --index-management respects template.name and template.pattern when ilm is disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.name=" + self.custom_template,
"-E", "setup.template.pattern=" + self.custom_template + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_template)
self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"])
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_with_opts(self):
"""
Test setup --index-management with config options
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.settings.index.number_of_shards=2"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
# check that settings are overwritten
resp = self.es.transport.perform_request('GET', '/_index_template/' + self.data_stream)
found = False
for index_template in resp["index_templates"]:
if self.data_stream == index_template["name"]:
found = True
index = index_template["index_template"]["template"]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
assert found
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_overwrite_template_on_ilm_policy_created(self):
"""
Test setup --index-management overwrites template when new ilm policy is created
"""
# ensure template with ilm rollover_alias name is created, but ilm policy not yet
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.priority=160",
"-E", "setup.template.name=" + self.custom_template,
"-E", "setup.template.pattern=" + self.custom_template + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_template)
self.idxmgmt.assert_policy_not_created(self.policy_name)
# ensure ilm policy is created, triggering overwriting existing template
exit_code = self.run_beat(extra_args=["setup", "-d", "*", self.cmd,
"-E", "setup.template.overwrite=true",
"-E", "setup.template.name=" + self.custom_template,
"-E", "setup.template.pattern=" + self.custom_template + "*",
"-E", "setup.template.settings.index.number_of_shards=2"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_template)
self.idxmgmt.assert_policy_created(self.policy_name)
# check that template was overwritten
resp = self.es.transport.perform_request('GET', '/_index_template/' + self.custom_template)
found = False
for index_template in resp["index_templates"]:
if index_template["name"] == self.custom_template:
found = True
index = index_template["index_template"]["template"]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
assert found
class TestCommandSetupIndexManagement(BaseTest):
"""
Test beat command `setup` related to ILM policy
"""
def setUp(self):
super(TestCommandSetupIndexManagement, self).setUp()
self.cmd = "--index-management"
# auto-derived default settings, if nothing else is set
self.policy_name = self.beat_name
self.index_name = self.alias_name = self.beat_name + "-9.9.9"
self.custom_alias = self.beat_name + "_foo"
self.custom_policy = self.beat_name + "_bar"
self.custom_template = self.beat_name + "_foobar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.index_name)
self.idxmgmt.delete(
indices=[self.custom_alias, self.index_name, self.custom_policy],
policies=[self.policy_name, self.custom_policy])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(
indices=[self.custom_alias, self.index_name, self.custom_policy],
policies=[self.policy_name, self.custom_policy])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
es_template_name=self.index_name,
**kwargs)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_default(self):
"""
Test setup --index-management with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name,
self.policy_name,
self.alias_name)
self.idxmgmt.assert_index_template_index_pattern(
self.index_name, [self.index_name + "-*"])
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_default(self):
"""
Test setup --index-management with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name,
self.policy_name,
self.alias_name)
self.idxmgmt.assert_index_template_index_pattern(
self.index_name, [self.index_name + "-*"])
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
# try deleting policy needs to raise an error as it is in use
with pytest.raises(RequestError):
self.idxmgmt.delete_policy(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_disabled(self):
"""
Test setup --index-management when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E",
"setup.template.enabled=false"
])
assert exit_code == 0
self.idxmgmt.assert_index_template_not_loaded(self.index_name)
self.idxmgmt.assert_alias_created(self.index_name)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_ilm_disabled(self):
"""
Test setup --index-management when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd, "-E", "setup.ilm.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_policy_name(self):
"""
Test setup --index-management when policy_name is configured
"""
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E",
"setup.ilm.policy_name=" + self.custom_policy
])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name,
self.custom_policy,
self.alias_name)
self.idxmgmt.assert_policy_created(self.custom_policy)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_ilm_policy_no_overwrite(self):
"""
Test setup --index-management respects overwrite configuration
"""
policy_name = "mockbeat-test"
# update policy to verify overwrite behaviour
self.es.transport.perform_request('PUT',
'/_ilm/policy/' + policy_name,
body={
"policy": {
"phases": {
"delete": {
"actions": {
"delete": {}
}
}
}
}
})
resp = self.es.transport.perform_request('GET',
'/_ilm/policy/' + policy_name)
assert "delete" in resp[policy_name]["policy"]["phases"]
assert "hot" not in resp[policy_name]["policy"]["phases"]
# ensure ilm policy is not overwritten
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E",
"setup.ilm.enabled=true", "-E",
"setup.ilm.overwrite=false", "-E",
"setup.ilm.policy_name=" + policy_name
])
assert exit_code == 0
resp = self.es.transport.perform_request('GET',
'/_ilm/policy/' + policy_name)
assert "delete" in resp[policy_name]["policy"]["phases"]
assert "hot" not in resp[policy_name]["policy"]["phases"]
# ensure ilm policy is overwritten
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E",
"setup.ilm.enabled=true", "-E",
"setup.ilm.overwrite=true", "-E",
"setup.ilm.policy_name=" + policy_name
])
assert exit_code == 0
resp = self.es.transport.perform_request('GET',
'/_ilm/policy/' + policy_name)
assert "delete" not in resp[policy_name]["policy"]["phases"]
assert "hot" in resp[policy_name]["policy"]["phases"]
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_rollover_alias(self):
"""
Test setup --index-management when ilm.rollover_alias is configured
"""
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E",
"setup.ilm.rollover_alias=" + self.custom_alias
])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias,
self.policy_name,
self.custom_alias)
self.idxmgmt.assert_index_template_index_pattern(
self.custom_alias, [self.custom_alias + "-*"])
self.idxmgmt.assert_alias_created(self.custom_alias)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_rollover_alias_with_fieldref(self):
"""
Test setup --index-management when ilm.rollover_alias is configured and using field reference.
"""
aliasFieldRef = "%{[agent.name]}-myalias"
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E",
"setup.ilm.rollover_alias=" + aliasFieldRef
])
self.custom_alias = self.beat_name + "-myalias"
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias,
self.policy_name,
self.custom_alias)
self.idxmgmt.assert_index_template_index_pattern(
self.custom_alias, [self.custom_alias + "-*"])
self.idxmgmt.assert_alias_created(self.custom_alias)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_name_and_pattern(self):
"""
Test setup --index-management ignores template.name and template.pattern when ilm is enabled
"""
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E",
"setup.template.name=" + self.custom_template, "-E",
"setup.template.pattern=" + self.custom_template + "*"
])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name,
self.policy_name,
self.alias_name)
self.idxmgmt.assert_index_template_index_pattern(
self.alias_name, [self.alias_name + "-*"])
self.idxmgmt.assert_alias_created(self.alias_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_name_and_pattern_on_ilm_disabled(self):
"""
Test setup --index-management respects template.name and template.pattern when ilm is disabled
"""
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E", "setup.ilm.enabled=false", "-E",
"setup.template.name=" + self.custom_template, "-E",
"setup.template.pattern=" + self.custom_template + "*"
])
assert exit_code == 0
self.idxmgmt.assert_legacy_index_template_loaded(self.custom_template)
self.idxmgmt.assert_index_template_index_pattern(
self.custom_template, [self.custom_template + "*"])
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_template_with_opts(self):
"""
Test setup --index-management with config options
"""
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E", "setup.ilm.enabled=false", "-E",
"setup.template.settings.index.number_of_shards=2"
])
assert exit_code == 0
self.idxmgmt.assert_legacy_index_template_loaded(self.index_name)
# check that settings are overwritten
resp = self.es.transport.perform_request(
'GET', '/_template/' + self.index_name)
assert self.index_name in resp
index = resp[self.index_name]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_overwrite_template_on_ilm_policy_created(self):
"""
Test setup --index-management overwrites template when new ilm policy is created
"""
# ensure template with ilm rollover_alias name is created, but ilm policy not yet
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.cmd, "-E", "setup.ilm.enabled=false", "-E",
"setup.template.name=" + self.custom_alias, "-E",
"setup.template.pattern=" + self.custom_alias + "*"
])
assert exit_code == 0
self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias)
self.idxmgmt.assert_policy_not_created(self.policy_name)
# ensure ilm policy is created, triggering overwriting existing template
exit_code = self.run_beat(extra_args=[
"setup", self.cmd, "-E", "setup.template.overwrite=false", "-E",
"setup.template.settings.index.number_of_shards=2", "-E",
"setup.ilm.rollover_alias=" + self.custom_alias
])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias,
self.policy_name,
self.custom_alias)
self.idxmgmt.assert_policy_created(self.policy_name)
# check that template was overwritten
resp = self.es.transport.perform_request(
'GET', '/_template/' + self.custom_alias)
assert self.custom_alias in resp
index = resp[self.custom_alias]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
class TestCommandSetupIndexManagement(BaseTest):
"""
Test beat command `setup` related to ILM policy
"""
def setUp(self):
super(TestCommandSetupIndexManagement, self).setUp()
self.cmd = "--index-management"
# auto-derived default settings, if nothing else is set
self.index_name = self.alias_name = self.policy_name = self.beat_name + "-9.9.9"
self.custom_alias = self.beat_name + "_foo"
self.custom_policy = self.beat_name + "_bar"
self.custom_template = self.beat_name + "_foobar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.index_name)
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
es_template_name=self.index_name,
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_default(self):
"""
Test setup --index-management with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"])
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
@raises(RequestError)
def test_setup_default(self):
"""
Test setup --index-management with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"])
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
# try deleting policy needs to raise an error as it is in use
self.idxmgmt.delete_policy(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_disabled(self):
"""
Test setup --index-management when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.template.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_not_loaded(self.index_name)
self.idxmgmt.assert_alias_created(self.index_name)
self.idxmgmt.assert_policy_created(self.index_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_ilm_disabled(self):
"""
Test setup --index-management when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_policy_name(self):
"""
Test setup --index-management when policy_name is configured
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.policy_name=" + self.custom_policy])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name)
self.idxmgmt.assert_policy_created(self.custom_policy)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_rollover_alias(self):
"""
Test setup --index-management when ilm.rollover_alias is configured
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.rollover_alias=" + self.custom_alias])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"])
self.idxmgmt.assert_docs_written_to_alias(self.custom_alias)
self.idxmgmt.assert_alias_created(self.custom_alias)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_name_and_pattern(self):
"""
Test setup --index-management ignores template.name and template.pattern when ilm is enabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.template.name=" + self.custom_template,
"-E", "setup.template.pattern=" + self.custom_template + "*"])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"])
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_name_and_pattern_on_ilm_disabled(self):
"""
Test setup --index-management respects template.name and template.pattern when ilm is disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.name=" + self.custom_template,
"-E", "setup.template.pattern=" + self.custom_template + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_template)
self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"])
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_with_opts(self):
"""
Test setup --index-management with config options
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.settings.index.number_of_shards=2"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
# check that settings are overwritten
resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name)
assert self.index_name in resp
index = resp[self.index_name]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_overwrite_template_on_ilm_policy_created(self):
"""
Test setup --index-management overwrites template when new ilm policy is created
"""
# ensure template with ilm rollover_alias name is created, but ilm policy not yet
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.cmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.name=" + self.custom_alias,
"-E", "setup.template.pattern=" + self.custom_alias + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_alias)
self.idxmgmt.assert_policy_not_created(self.index_name)
# ensure ilm policy is created, triggering overwriting existing template
exit_code = self.run_beat(extra_args=["setup", self.cmd,
"-E", "setup.template.overwrite=false",
"-E", "setup.template.settings.index.number_of_shards=2",
"-E", "setup.ilm.rollover_alias=" + self.custom_alias])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias)
self.idxmgmt.assert_policy_created(self.index_name)
# check that template was overwritten
resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias)
assert self.custom_alias in resp
index = resp[self.custom_alias]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
class TestCommandSetupILMPolicy(BaseTest):
"""
Test beat command `setup` related to ILM policy
"""
def setUp(self):
super(TestCommandSetupILMPolicy, self).setUp()
self.setupCmd = "--index-management"
self.data_stream = self.beat_name + "-9.9.9"
self.policy_name = self.beat_name
self.custom_policy = self.beat_name + "_bar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.data_stream)
self.idxmgmt.delete(indices=[],
policies=[self.policy_name, self.custom_policy],
data_streams=[self.data_stream])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(indices=[],
policies=[self.policy_name, self.custom_policy],
data_streams=[self.data_stream])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch=self.get_elasticsearch_template_config(),
es_template_name=self.data_stream,
**kwargs)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_ilm_default(self):
"""
Test ilm policy setup with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd])
assert exit_code == 0
self.idxmgmt.assert_index_template_index_pattern(
self.data_stream, [self.data_stream])
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_setup_ilm_disabled(self):
"""
Test ilm policy setup when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.setupCmd, "-E",
"setup.ilm.enabled=false"
])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@pytest.mark.tag('integration')
def test_policy_name(self):
"""
Test ilm policy setup when policy_name is configured
"""
self.render_config()
exit_code = self.run_beat(
logging_args=["-v", "-d", "*"],
extra_args=[
"setup", self.setupCmd, "-E",
"setup.ilm.policy_name=" + self.custom_policy
])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.data_stream)
self.idxmgmt.assert_policy_created(self.custom_policy)
class TestCommandSetupILMPolicy(BaseTest):
"""
Test beat command `setup` related to ILM policy
"""
def setUp(self):
super(TestCommandSetupILMPolicy, self).setUp()
self.setupCmd = "--ilm-policy"
self.alias_name = self.policy_name = self.index_name = self.beat_name + "-9.9.9"
self.custom_alias = self.beat_name + "_foo"
self.custom_policy = self.beat_name + "_bar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.index_name)
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
es_template_name=self.index_name,
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_ilm_policy_and_template(self):
"""
Test combination of ilm policy and template setup
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd, "--template"])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_ilm_default(self):
"""
Test ilm policy setup with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"])
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_ilm_disabled(self):
"""
Test ilm policy setup when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_policy_name(self):
"""
Test ilm policy setup when policy_name is configured
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.policy_name=" + self.custom_policy])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name)
self.idxmgmt.assert_policy_created(self.custom_policy)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_rollover_alias(self):
"""
Test ilm policy setup when rollover_alias is configured
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.rollover_alias=" + self.custom_alias])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
self.idxmgmt.assert_docs_written_to_alias(self.custom_alias)
self.idxmgmt.assert_alias_created(self.custom_alias)
class TestRunILM(BaseTest):
def setUp(self):
super(TestRunILM, self).setUp()
self.alias_name = self.policy_name = self.index_name = self.beat_name + "-9.9.9"
self.custom_alias = self.beat_name + "_foo"
self.custom_policy = self.beat_name + "_bar"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.index_name)
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
def tearDown(self):
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
es_template_name=self.index_name,
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_ilm_default(self):
"""
Test ilm default settings to load ilm policy, write alias and ilm template
"""
self.render_config()
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name)
self.idxmgmt.assert_policy_created(self.policy_name)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_ilm_disabled(self):
"""
Test ilm disabled to not load ilm related components
"""
self.render_config(ilm={"enabled": False})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_index_template_loaded(self.index_name)
self.idxmgmt.assert_alias_not_created(self.alias_name)
self.idxmgmt.assert_policy_not_created(self.policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_policy_name(self):
"""
Test setting ilm policy name
"""
policy_name = self.beat_name + "_foo"
self.render_config(ilm={"enabled": True, "policy_name": policy_name})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, policy_name, self.alias_name)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name)
self.idxmgmt.assert_policy_created(policy_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_rollover_alias(self):
"""
Test settings ilm rollover alias
"""
self.render_config(ilm={"enabled": True, "rollover_alias": self.custom_alias})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
self.idxmgmt.assert_docs_written_to_alias(self.custom_alias)
self.idxmgmt.assert_alias_created(self.custom_alias)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_pattern(self):
"""
Test setting ilm pattern
"""
pattern = "1"
self.render_config(ilm={"enabled": True, "pattern": pattern})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name, pattern=pattern)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=pattern)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_pattern_date(self):
"""
Test setting ilm pattern with date
"""
pattern = "'{now/d}'"
self.render_config(ilm={"enabled": True, "pattern": pattern})
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("ILM policy successfully loaded"))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()
resolved_pattern = datetime.datetime.now().strftime("%Y.%m.%d")
self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name)
self.idxmgmt.assert_alias_created(self.alias_name, pattern=resolved_pattern)
self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=resolved_pattern)
class TestCommandSetupTemplate(BaseTest):
"""
Test beat command `setup` with focus on template
"""
def setUp(self):
super(TestCommandSetupTemplate, self).setUp()
# auto-derived default settings, if nothing else is set
self.setupCmd = "--template"
self.index_name = self.beat_name + "-9.9.9"
self.custom_alias = self.beat_name + "_foo"
self.es = self.es_client()
self.idxmgmt = IdxMgmt(self.es, self.index_name)
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name])
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
def tearDown(self):
self.idxmgmt.delete(indices=[self.custom_alias, self.index_name])
def render_config(self, **kwargs):
self.render_config_template(
elasticsearch={"hosts": self.get_elasticsearch_url()},
**kwargs
)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup(self):
"""
Test setup cmd with template and ilm-policy subcommands
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd, "--ilm-policy"])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name)
self.idxmgmt.assert_alias_created(self.index_name)
self.idxmgmt.assert_policy_created(self.index_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_default(self):
"""
Test template setup with default config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.index_name, self.index_name)
self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"])
# when running `setup --template`
# write_alias and rollover_policy related to ILM are also created
self.idxmgmt.assert_alias_created(self.index_name)
self.idxmgmt.assert_policy_created(self.index_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_disabled(self):
"""
Test template setup when ilm disabled
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.template.enabled=false"])
assert exit_code == 0
self.idxmgmt.assert_index_template_not_loaded(self.index_name)
# when running `setup --template` and `setup.template.enabled=false`
# write_alias and rollover_policy related to ILM are still created
self.idxmgmt.assert_alias_created(self.index_name)
self.idxmgmt.assert_policy_created(self.index_name)
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_with_opts(self):
"""
Test template setup with config options
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.settings.index.number_of_shards=2"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.index_name)
# check that settings are overwritten
resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name)
assert self.index_name in resp
index = resp[self.index_name]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_setup_template_with_ilm_changed_pattern(self):
"""
Test template setup with changed ilm.rollover_alias config
"""
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.rollover_alias=" + self.custom_alias])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias)
self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"])
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
@attr('integration')
def test_template_created_on_ilm_policy_created(self):
"""
Test template setup overwrites template when new ilm policy is created
"""
# ensure template with ilm rollover_alias name is created, but ilm policy not yet
self.render_config()
exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
extra_args=["setup", self.setupCmd,
"-E", "setup.ilm.enabled=false",
"-E", "setup.template.name=" + self.custom_alias,
"-E", "setup.template.pattern=" + self.custom_alias + "*"])
assert exit_code == 0
self.idxmgmt.assert_index_template_loaded(self.custom_alias)
self.idxmgmt.assert_policy_not_created(self.index_name)
# ensure ilm policy is created, triggering overwriting existing template
exit_code = self.run_beat(extra_args=["setup", self.setupCmd,
"-E", "setup.template.overwrite=false",
"-E", "setup.template.settings.index.number_of_shards=2",
"-E", "setup.ilm.rollover_alias=" + self.custom_alias])
assert exit_code == 0
self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias)
self.idxmgmt.assert_policy_created(self.index_name)
# check that template was overwritten
resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias)
assert self.custom_alias in resp
index = resp[self.custom_alias]["settings"]["index"]
assert index["number_of_shards"] == "2", index["number_of_shards"]
