def test_create_suppliers_for_es_and_plugin_source_build(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "source.revision",
"elasticsearch:abc,community-plugin:current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "node", "src.root.dir",
"/opt/rally/src")
cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir",
"elasticsearch")
cfg.add(config.Scope.application, "source", "remote.repo.url",
"https://github.com/elastic/elasticsearch.git")
cfg.add(config.Scope.application, "source",
"plugin.community-plugin.src.subdir",
"elasticsearch-extra/community-plugin")
car = team.Car("default",
root_path=None,
config_paths=[],
variables={
"clean_command": "./gradlew clean",
"build_command": "./gradlew assemble",
"build.jdk": "11"
})
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin",
core_plugin=False)
# --revision="elasticsearch:abc,community-plugin:effab"
composite_supplier = supplier.create(
cfg,
sources=True,
distribution=False,
build=True,
car=car,
plugins=[core_plugin, external_plugin])
self.assertEqual(3, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0],
supplier.ElasticsearchSourceSupplier)
self.assertIsInstance(composite_supplier.suppliers[1],
supplier.CorePluginSourceSupplier)
self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin)
self.assertIsInstance(composite_supplier.suppliers[2],
supplier.ExternalPluginSourceSupplier)
self.assertEqual(external_plugin,
composite_supplier.suppliers[2].plugin)
self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_create_suppliers_for_es_distribution_plugin_source_build(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "distribution.version",
"6.0.0")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision",
"community-plugin:current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application,
"distributions",
"release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "node", "src.root.dir",
"/opt/rally/src")
cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir",
"elasticsearch")
cfg.add(config.Scope.application, "source",
"plugin.community-plugin.src.dir",
"/home/user/Projects/community-plugin")
car = team.Car("default",
root_path=None,
config_paths=[],
variables={"build.jdk": "10"})
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin",
core_plugin=False)
# --revision="community-plugin:effab" --distribution-version="6.0.0"
composite_supplier = supplier.create(
cfg,
sources=False,
distribution=True,
car=car,
plugins=[core_plugin, external_plugin])
assert len(composite_supplier.suppliers) == 3
assert isinstance(composite_supplier.suppliers[0],
supplier.ElasticsearchDistributionSupplier)
assert isinstance(composite_supplier.suppliers[1],
supplier.PluginDistributionSupplier)
assert composite_supplier.suppliers[1].plugin == core_plugin
assert isinstance(composite_supplier.suppliers[2].source_supplier,
supplier.ExternalPluginSourceSupplier)
assert composite_supplier.suppliers[
2].source_supplier.plugin == external_plugin
assert composite_supplier.suppliers[
2].source_supplier.builder is not None
def test_create_suppliers_for_es_distribution_plugin_source_build(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "distribution.version",
"6.0.0")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision",
"community-plugin:current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "runtime", "java10.home",
"/usr/local/bin/java10/")
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "node", "src.root.dir",
"/opt/rally/src")
cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir",
"elasticsearch")
cfg.add(config.Scope.application, "source",
"plugin.community-plugin.src.dir",
"/home/user/Projects/community-plugin")
car = team.Car("default", root_path=None, config_paths=[])
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin",
core_plugin=False)
# --revision="community-plugin:effab" --distribution-version="6.0.0"
composite_supplier = supplier.create(
cfg,
sources=False,
distribution=True,
build=False,
challenge_root_path="/",
car=car,
plugins=[core_plugin, external_plugin])
self.assertEqual(3, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0],
supplier.ElasticsearchDistributionSupplier)
self.assertIsInstance(composite_supplier.suppliers[1],
supplier.PluginDistributionSupplier)
self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin)
self.assertIsInstance(composite_supplier.suppliers[2],
supplier.ExternalPluginSourceSupplier)
self.assertEqual(external_plugin,
composite_supplier.suppliers[2].plugin)
self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_derive_supply_requirements_es_and_plugin_source_build(self):
# corresponds to --revision="elasticsearch:abc,community-plugin:effab"
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
requirements = supplier._supply_requirements(sources=True, distribution=False, build=True, plugins=[core_plugin, external_plugin],
revisions={"elasticsearch": "abc", "all": "abc", "community-plugin": "effab"},
distribution_version=None)
self.assertDictEqual({
"elasticsearch": ("source", "abc", True),
# core plugin configuration is forced to be derived from ES
"analysis-icu": ("source", "abc", True),
"community-plugin": ("source", "effab", True),
}, requirements)
def test_derive_supply_requirements_es_distribution_and_plugin_source_skip(self):
# corresponds to --from-sources-skip-build --revision="community-plugin:current" --distribution-version="6.0.0"
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
requirements = supplier._supply_requirements(sources=True, distribution=False, build=False, plugins=[core_plugin, external_plugin],
revisions={"community-plugin": "current"},
distribution_version="6.0.0")
self.assertDictEqual({
"elasticsearch": ("distribution", "6.0.0", False),
# core plugin configuration is forced to be derived from ES
"analysis-icu": ("distribution", "6.0.0", False),
"community-plugin": ("source", "current", False),
}, requirements)
def test_create_suppliers_for_es_distribution_plugin_source_skip(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "distribution.version",
"6.0.0")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision",
"community-plugin:current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "source",
"plugin.community-plugin.src.dir",
"/home/user/Projects/community-plugin")
car = team.Car("default", root_path=None, config_paths=[])
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin",
core_plugin=False,
variables={"enabled": True})
# --pipeline=from-sources-skip-build
composite_supplier = supplier.create(
cfg,
sources=True,
distribution=False,
build=False,
car=car,
plugins=[core_plugin, external_plugin])
self.assertEqual(3, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0],
supplier.ElasticsearchDistributionSupplier)
self.assertIsInstance(composite_supplier.suppliers[1],
supplier.PluginDistributionSupplier)
self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin)
self.assertIsInstance(composite_supplier.suppliers[2].source_supplier,
supplier.ExternalPluginSourceSupplier)
self.assertEqual(
external_plugin,
composite_supplier.suppliers[2].source_supplier.plugin)
self.assertIsNone(
composite_supplier.suppliers[2].source_supplier.builder)
def test_pass_plugin_properties(self):
plugin = team.PluginDescriptor(name="unit-test-plugin", config="default", config_paths=["/etc/plugin"], variables={"active": True})
installer = provisioner.PluginInstaller(plugin, hook_handler_class=PluginInstallerTests.NoopHookHandler)
self.assertEqual("unit-test-plugin", installer.plugin_name)
self.assertEqual({"active": True}, installer.variables)
self.assertEqual(["/etc/plugin"], installer.config_source_paths)
def test_invokes_hook_with_java_home(self):
plugin = team.PluginDescriptor(
name="unit-test-plugin",
config="default",
config_paths=["/etc/plugin"],
variables={
"active": True,
},
)
installer = provisioner.PluginInstaller(
plugin,
java_home="/usr/local/javas/java8",
hook_handler_class=NoopHookHandler)
assert len(installer.hook_handler.hook_calls) == 0
installer.invoke_install_hook(team.BootstrapPhase.post_install,
{"foo": "bar"})
assert len(installer.hook_handler.hook_calls) == 1
assert installer.hook_handler.hook_calls["post_install"][
"variables"] == {
"foo": "bar"
}
assert installer.hook_handler.hook_calls["post_install"]["kwargs"] == {
"env": {
"JAVA_HOME": "/usr/local/javas/java8"
}
}
def test_invokes_hook(self):
plugin = team.PluginDescriptor(name="unit-test-plugin", config="default", config_paths=["/etc/plugin"], variables={"active": True})
installer = provisioner.PluginInstaller(plugin, hook_handler_class=PluginInstallerTests.NoopHookHandler)
self.assertEqual(0, len(installer.hook_handler.hook_calls))
installer.invoke_install_hook(provisioner.ProvisioningPhase.post_install, {"foo": "bar"})
self.assertEqual(1, len(installer.hook_handler.hook_calls))
self.assertEqual({"foo": "bar"}, installer.hook_handler.hook_calls["post_install"])
def test_resolve_plugin_binary(self):
s = supplier.CorePluginSourceSupplier(plugin=team.PluginDescriptor("core-plugin", core_plugin=True),
# built separately
es_src_dir="/src/elasticsearch",
builder=None)
binaries = {}
s.add(binaries)
self.assertDictEqual(binaries, {"core-plugin": "file:///src/elasticsearch/core-plugin/build/distributions/core-plugin.zip"})
def test_loads_core_plugin(self):
self.assertEqual(
team.PluginDescriptor(name="my-analysis-plugin",
core_plugin=True,
variables={"dbg": True}),
self.loader.load_plugin("my-analysis-plugin",
config_names=None,
plugin_params={"dbg": True}))
def test_install_plugin_successfully(self, installer_subprocess):
installer_subprocess.return_value = 0
plugin = team.PluginDescriptor(name="unit-test-plugin", config="default", variables={"active": True})
installer = provisioner.PluginInstaller(plugin, hook_handler_class=PluginInstallerTests.NoopHookHandler)
installer.install(es_home_path="/opt/elasticsearch")
installer_subprocess.assert_called_with('/opt/elasticsearch/bin/elasticsearch-plugin install --batch "unit-test-plugin"')
def test_resolve_plugin_url(self):
v = {"plugin_custom-analyzer_release_url": "http://example.org/elasticearch/custom-analyzer-{{VERSION}}.zip"}
s = supplier.PluginDistributionSupplier(repo=supplier.DistributionRepository(name="release",
distribution_config=v,
version="6.3.0"),
plugin=team.PluginDescriptor("custom-analyzer"))
binaries = {}
s.add(binaries)
self.assertDictEqual(binaries, {"custom-analyzer": "http://example.org/elasticearch/custom-analyzer-6.3.0.zip"})
def test_cannot_register_for_unknown_phase(self):
plugin = team.PluginDescriptor("unittest-plugin")
hook = self.UnitTestHook(phase="this_is_an_unknown_install_phase")
handler = team.BootstrapHookHandler(plugin, loader_class=self.UnitTestComponentLoader)
handler.loader.registration_function = hook
with pytest.raises(exceptions.SystemSetupError) as exc:
handler.load()
assert exc.value.args[0] == "Unknown bootstrap phase [this_is_an_unknown_install_phase]. Valid phases are: ['post_install']."
def test_cannot_register_for_unknown_phase(self):
plugin = team.PluginDescriptor("unittest-plugin")
hook = InstallHookHandlerTests.UnitTestHook(phase="this_is_an_unknown_install_phase")
handler = provisioner.InstallHookHandler(plugin, loader_class=InstallHookHandlerTests.UnitTestComponentLoader)
handler.loader.registration_function = hook
with self.assertRaises(exceptions.SystemSetupError) as ctx:
handler.load()
self.assertEqual("Provisioning phase [this_is_an_unknown_install_phase] is unknown. Valid phases are: ['post_install'].",
ctx.exception.args[0])
def setUp(self):
self.along_es = supplier.ExternalPluginSourceSupplier(plugin=team.PluginDescriptor("some-plugin", core_plugin=False),
revision="abc",
# built along-side ES
src_dir="/src",
src_config={
"plugin.some-plugin.src.subdir": "elasticsearch-extra/some-plugin",
"plugin.some-plugin.build.artifact.subdir": "plugin/build/distributions"
},
builder=None)
self.standalone = supplier.ExternalPluginSourceSupplier(plugin=team.PluginDescriptor("some-plugin", core_plugin=False),
revision="abc",
# built separately
src_dir=None,
src_config={
"plugin.some-plugin.src.dir": "/Projects/src/some-plugin",
"plugin.some-plugin.build.artifact.subdir": "build/distributions"
},
builder=None)
def test_derive_supply_requirements_es_distribution_and_plugin_source_build(
self):
# corresponds to --revision="community-plugin:effab" --distribution-version="6.0.0"
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin",
core_plugin=False)
requirements = supplier._supply_requirements(
sources=False,
distribution=True,
plugins=[core_plugin, external_plugin],
revisions={"community-plugin": "effab"},
distribution_version="6.0.0",
)
# core plugin is not contained, its configured is forced to be derived by ES
assert requirements == {
"elasticsearch": ("distribution", "6.0.0", False),
# core plugin configuration is forced to be derived from ES
"analysis-icu": ("distribution", "6.0.0", False),
"community-plugin": ("source", "effab", True),
}
def test_install_unknown_plugin(self, installer_subprocess):
# unknown plugin
installer_subprocess.return_value = 64
plugin = team.PluginDescriptor(name="unknown")
installer = provisioner.PluginInstaller(plugin, hook_handler_class=PluginInstallerTests.NoopHookHandler)
with self.assertRaises(exceptions.SystemSetupError) as ctx:
installer.install(es_home_path="/opt/elasticsearch")
self.assertEqual("Unknown plugin [unknown]", ctx.exception.args[0])
installer_subprocess.assert_called_with('/opt/elasticsearch/bin/elasticsearch-plugin install --batch "unknown"')
def test_loads_module(self):
plugin = team.PluginDescriptor("unittest-plugin")
hook = self.UnitTestHook()
handler = team.BootstrapHookHandler(plugin, loader_class=self.UnitTestComponentLoader)
handler.loader.registration_function = hook
handler.load()
handler.invoke("post_install", variables={"increment": 4})
# we registered our hook twice. Check that it has been called twice.
assert hook.call_counter == 2 * 4
def test_loads_module(self):
plugin = team.PluginDescriptor("unittest-plugin")
hook = InstallHookHandlerTests.UnitTestHook()
handler = provisioner.InstallHookHandler(plugin, loader_class=InstallHookHandlerTests.UnitTestComponentLoader)
handler.loader.registration_function = hook
handler.load()
handler.invoke("post_install", {"increment": 4})
# we registered our hook twice. Check that it has been called twice.
self.assertEqual(hook.call_counter, 2 * 4)
def test_install_plugin_with_io_error(self, installer_subprocess):
# I/O error
installer_subprocess.return_value = 74
plugin = team.PluginDescriptor(name="simple")
installer = provisioner.PluginInstaller(plugin, hook_handler_class=PluginInstallerTests.NoopHookHandler)
with self.assertRaises(exceptions.SupplyError) as ctx:
installer.install(es_home_path="/opt/elasticsearch")
self.assertEqual("I/O error while trying to install [simple]", ctx.exception.args[0])
installer_subprocess.assert_called_with('/opt/elasticsearch/bin/elasticsearch-plugin install --batch "simple"')
def test_invalid_config_duplicate_source(self):
with self.assertRaisesRegex(exceptions.SystemSetupError,
"Can only specify one of plugin.duplicate.src.dir and plugin.duplicate.src.subdir but both are set."):
supplier.ExternalPluginSourceSupplier(plugin=team.PluginDescriptor("duplicate", core_plugin=False),
revision="abc",
src_dir=None,
src_config={
"plugin.duplicate.src.subdir": "elasticsearch-extra/some-plugin",
"plugin.duplicate.src.dir": "/Projects/src/some-plugin",
"plugin.duplicate.build.artifact.subdir": "build/distributions"
},
builder=None)
def test_invalid_config_no_source(self):
with self.assertRaisesRegex(exceptions.SystemSetupError,
"Neither plugin.some-plugin.src.dir nor plugin.some-plugin.src.subdir are set for plugin some-plugin."):
supplier.ExternalPluginSourceSupplier(plugin=team.PluginDescriptor("some-plugin", core_plugin=False),
revision="abc",
# built separately
src_dir=None,
src_config={
# but no source config
# "plugin.some-plugin.src.dir": "/Projects/src/some-plugin",
"plugin.some-plugin.build.artifact.subdir": "build/distributions"
},
builder=None)
def test_install_plugin_with_unknown_error(self, installer_subprocess):
# some other error
installer_subprocess.return_value = 12987
plugin = team.PluginDescriptor(name="simple")
installer = provisioner.PluginInstaller(plugin, hook_handler_class=PluginInstallerTests.NoopHookHandler)
with self.assertRaises(exceptions.RallyError) as ctx:
installer.install(es_home_path="/opt/elasticsearch")
self.assertEqual("Unknown error while trying to install [simple] (installer return code [12987]). Please check the logs.",
ctx.exception.args[0])
installer_subprocess.assert_called_with('/opt/elasticsearch/bin/elasticsearch-plugin install --batch "simple"')
def test_create_suppliers_for_es_missing_distribution_plugin_source_skip(self):
cfg = config.Config()
# no distribution version!
# cfg.add(config.Scope.application, "mechanic", "distribution.version", "")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current")
cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release")
cfg.add(config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
cfg.add(config.Scope.application, "distributions", "release.cache", True)
cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/")
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
# --from-sources-skip-build --revision="community-plugin:current" (distribution version is missing!)
with self.assertRaises(exceptions.SystemSetupError) as ctx:
supplier.create(cfg, sources=True, distribution=False, build=False, challenge_root_path="/", plugins=[
core_plugin,
external_plugin
])
self.assertRegex(ctx.exception.args[0], r"Could not determine version..*")
def test_invokes_hook_no_java_home(self):
plugin = team.PluginDescriptor(name="unit-test-plugin",
config="default",
config_paths=["/etc/plugin"],
variables={"active": True})
installer = provisioner.PluginInstaller(plugin,
java_home=None,
hook_handler_class=NoopHookHandler)
self.assertEqual(0, len(installer.hook_handler.hook_calls))
installer.invoke_install_hook(team.BootstrapPhase.post_install, {"foo": "bar"})
self.assertEqual(1, len(installer.hook_handler.hook_calls))
self.assertEqual({"foo": "bar"}, installer.hook_handler.hook_calls["post_install"]["variables"])
self.assertEqual({"env": {}}, installer.hook_handler.hook_calls["post_install"]["kwargs"])
def test_pass_plugin_properties(self):
plugin = team.PluginDescriptor(
name="unit-test-plugin",
config="default",
config_paths=["/etc/plugin"],
variables={
"active": True,
},
)
installer = provisioner.PluginInstaller(
plugin,
java_home="/usr/local/javas/java8",
hook_handler_class=NoopHookHandler)
assert installer.plugin_name == "unit-test-plugin"
assert installer.variables == {"active": True}
assert installer.config_source_paths == ["/etc/plugin"]
def test_install_unknown_plugin(self, installer_subprocess):
# unknown plugin
installer_subprocess.return_value = 64
plugin = team.PluginDescriptor(name="unknown")
installer = provisioner.PluginInstaller(
plugin,
java_home="/usr/local/javas/java8",
hook_handler_class=NoopHookHandler)
with pytest.raises(exceptions.SystemSetupError) as exc:
installer.install(es_home_path="/opt/elasticsearch")
assert exc.value.args[0] == "Unknown plugin [unknown]"
installer_subprocess.assert_called_with(
'/opt/elasticsearch/bin/elasticsearch-plugin install --batch "unknown"',
env={"JAVA_HOME": "/usr/local/javas/java8"},
)
def test_resolve_plugin_url(self):
v = {
"plugin_custom-analyzer_release_url":
"http://example.org/elasticearch/custom-analyzer-{{VERSION}}.zip"
}
renderer = supplier.TemplateRenderer(version="6.3.0")
s = supplier.PluginDistributionSupplier(
repo=supplier.DistributionRepository(name="release",
distribution_config=v,
template_renderer=renderer),
plugin=team.PluginDescriptor("custom-analyzer"),
)
binaries = {}
s.add(binaries)
assert binaries == {
"custom-analyzer":
"http://example.org/elasticearch/custom-analyzer-6.3.0.zip",
}
def test_install_plugin_with_unknown_error(self, installer_subprocess):
# some other error
installer_subprocess.return_value = 12987
plugin = team.PluginDescriptor(name="simple")
installer = provisioner.PluginInstaller(
plugin,
java_home="/usr/local/javas/java8",
hook_handler_class=NoopHookHandler)
with pytest.raises(exceptions.RallyError) as exc:
installer.install(es_home_path="/opt/elasticsearch")
assert exc.value.args[
0] == "Unknown error while trying to install [simple] (installer return code [12987]). Please check the logs."
installer_subprocess.assert_called_with(
'/opt/elasticsearch/bin/elasticsearch-plugin install --batch "simple"',
env={"JAVA_HOME": "/usr/local/javas/java8"},
)
