def test_create_suppliers_for_es_missing_distribution_plugin_source_skip(
self):
cfg = config.Config()
# no distribution version!
# cfg.add(config.Scope.application, "mechanic", "distribution.version", "")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision",
"community-plugin:current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "runtime", "java10.home",
"/usr/local/bin/java10/")
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin",
core_plugin=False)
# --from-sources-skip-build --revision="community-plugin:current" (distribution version is missing!)
with self.assertRaises(exceptions.SystemSetupError) as ctx:
supplier.create(cfg,
sources=True,
distribution=False,
build=False,
challenge_root_path="/",
plugins=[core_plugin, external_plugin])
self.assertRegex(ctx.exception.args[0],
r"Could not determine version..*")
def create(cfg, metrics_store, node_ip, node_http_port, all_node_ips, all_node_ids, sources=False, distribution=False,
external=False, docker=False):
race_root_path = paths.race_root(cfg)
node_ids = cfg.opts("provisioning", "node.ids", mandatory=False)
node_name_prefix = cfg.opts("provisioning", "node.name.prefix")
car, plugins = load_team(cfg, external)
if sources or distribution:
s = supplier.create(cfg, sources, distribution, car, plugins)
p = []
all_node_names = ["%s-%s" % (node_name_prefix, n) for n in all_node_ids]
for node_id in node_ids:
node_name = "%s-%s" % (node_name_prefix, node_id)
p.append(
provisioner.local(cfg, car, plugins, node_ip, node_http_port, all_node_ips,
all_node_names, race_root_path, node_name))
l = launcher.ProcessLauncher(cfg)
elif external:
raise exceptions.RallyAssertionError("Externally provisioned clusters should not need to be managed by Rally's mechanic")
elif docker:
if len(plugins) > 0:
raise exceptions.SystemSetupError("You cannot specify any plugins for Docker clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
s = lambda: None
p = []
for node_id in node_ids:
node_name = "%s-%s" % (node_name_prefix, node_id)
p.append(provisioner.docker(cfg, car, node_ip, node_http_port, race_root_path, node_name))
l = launcher.DockerLauncher(cfg)
else:
# It is a programmer error (and not a user error) if this function is called with wrong parameters
raise RuntimeError("One of sources, distribution, docker or external must be True")
return Mechanic(cfg, metrics_store, s, p, l)
def install(cfg):
root_path = paths.install_root(cfg)
car, plugins = load_team(cfg, external=False)
# A non-empty distribution-version is provided
distribution = bool(cfg.opts("mechanic", "distribution.version", mandatory=False))
sources = not distribution
build_type = cfg.opts("mechanic", "build.type")
ip = cfg.opts("mechanic", "network.host")
http_port = int(cfg.opts("mechanic", "network.http.port"))
node_name = cfg.opts("mechanic", "node.name")
master_nodes = cfg.opts("mechanic", "master.nodes")
seed_hosts = cfg.opts("mechanic", "seed.hosts")
if build_type == "tar":
binary_supplier = supplier.create(cfg, sources, distribution, car, plugins)
p = provisioner.local(cfg=cfg, car=car, plugins=plugins, ip=ip, http_port=http_port,
all_node_ips=seed_hosts, all_node_names=master_nodes, target_root=root_path,
node_name=node_name)
node_config = p.prepare(binary=binary_supplier())
elif build_type == "docker":
if len(plugins) > 0:
raise exceptions.SystemSetupError("You cannot specify any plugins for Docker clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
p = provisioner.docker(cfg=cfg, car=car, ip=ip, http_port=http_port, target_root=root_path, node_name=node_name)
# there is no binary for Docker that can be downloaded / built upfront
node_config = p.prepare(binary=None)
else:
raise exceptions.SystemSetupError("Unknown build type [{}]".format(build_type))
provisioner.save_node_configuration(root_path, node_config)
console.println(json.dumps({"installation-id": cfg.opts("system", "install.id")}, indent=2), force=True)
def test_create_suppliers_for_es_only_config(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "distribution.version",
"6.0.0")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision",
"current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "runtime", "java10.home",
"/usr/local/bin/java10/")
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
car = team.Car("default", root_path=None, config_paths=[])
composite_supplier = supplier.create(cfg,
sources=False,
distribution=True,
build=False,
challenge_root_path="/",
car=car)
self.assertEqual(1, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0],
supplier.ElasticsearchDistributionSupplier)
def test_create_suppliers_for_es_and_plugin_source_build(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "source.revision", "elasticsearch:abc,community-plugin:current")
cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release")
cfg.add(config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
cfg.add(config.Scope.application, "distributions", "release.cache", True)
cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/")
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src")
cfg.add(config.Scope.application, "build", "gradle.bin", "/opt/gradle")
cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch")
cfg.add(config.Scope.application, "source", "remote.repo.url", "https://github.com/elastic/elasticsearch.git")
cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.subdir", "elasticsearch-extra/community-plugin")
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
# --revision="elasticsearch:abc,community-plugin:effab"
composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=True, challenge_root_path="/", plugins=[
core_plugin,
external_plugin
])
self.assertEqual(3, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchSourceSupplier)
self.assertIsInstance(composite_supplier.suppliers[1], supplier.CorePluginSourceSupplier)
self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin)
self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier)
self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin)
self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_create_suppliers_for_es_distribution_plugin_source_skip(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current")
cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release")
cfg.add(config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
cfg.add(config.Scope.application, "distributions", "release.cache", True)
cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/")
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin")
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
# --pipeline=from-sources-skip-build
composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=False, challenge_root_path="/", plugins=[
core_plugin,
external_plugin
])
self.assertEqual(3, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier)
self.assertIsInstance(composite_supplier.suppliers[1], supplier.PluginDistributionSupplier)
self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin)
self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier)
self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin)
self.assertIsNone(composite_supplier.suppliers[2].builder)
def test_create_suppliers_for_es_distribution_plugin_source_build(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current")
cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release")
cfg.add(config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
cfg.add(config.Scope.application, "distributions", "release.cache", True)
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src")
cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch")
cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin")
car = team.Car("default", root_path=None, config_paths=[], variables={"build.jdk": "10"})
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
# --revision="community-plugin:effab" --distribution-version="6.0.0"
composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", car=car, plugins=[
core_plugin,
external_plugin
])
self.assertEqual(3, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier)
self.assertIsInstance(composite_supplier.suppliers[1], supplier.PluginDistributionSupplier)
self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin)
self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier)
self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin)
self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_create_suppliers_for_es_and_plugin_source_build(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "source.revision",
"elasticsearch:abc,community-plugin:current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application,
"distributions",
"release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "node", "src.root.dir",
"/opt/rally/src")
cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir",
"elasticsearch")
cfg.add(config.Scope.application, "source", "remote.repo.url",
"https://github.com/elastic/elasticsearch.git")
cfg.add(config.Scope.application, "source",
"plugin.community-plugin.src.subdir",
"elasticsearch-extra/community-plugin")
car = team.Car(
"default",
root_path=None,
config_paths=[],
variables={
"clean_command": "./gradlew clean",
"build_command": "./gradlew assemble",
"build.jdk": "11"
},
)
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin",
core_plugin=False)
# --revision="elasticsearch:abc,community-plugin:effab"
composite_supplier = supplier.create(
cfg,
sources=True,
distribution=False,
car=car,
plugins=[core_plugin, external_plugin])
assert len(composite_supplier.suppliers) == 3
assert isinstance(composite_supplier.suppliers[0].source_supplier,
supplier.ElasticsearchSourceSupplier)
assert isinstance(composite_supplier.suppliers[1].source_supplier,
supplier.CorePluginSourceSupplier)
assert composite_supplier.suppliers[
1].source_supplier.plugin == core_plugin
assert isinstance(composite_supplier.suppliers[2].source_supplier,
supplier.ExternalPluginSourceSupplier)
assert composite_supplier.suppliers[
2].source_supplier.plugin == external_plugin
assert composite_supplier.suppliers[
2].source_supplier.builder is not None
def create(cfg,
metrics_store,
all_node_ips,
cluster_settings=None,
sources=False,
build=False,
distribution=False,
external=False,
docker=False):
races_root = paths.races_root(cfg)
challenge_root_path = paths.race_root(cfg)
node_ids = cfg.opts("provisioning", "node.ids", mandatory=False)
car, plugins = load_team(cfg, external)
if sources or distribution:
s = supplier.create(cfg, sources, distribution, build,
challenge_root_path, car, plugins)
p = []
for node_id in node_ids:
p.append(
provisioner.local_provisioner(cfg, car, plugins,
cluster_settings, all_node_ips,
challenge_root_path, node_id))
l = launcher.InProcessLauncher(cfg, metrics_store, races_root)
elif external:
if cluster_settings:
logging.getLogger(__name__).warning(
"Cannot apply challenge-specific cluster settings [%s] for an externally provisioned cluster. Please ensure that the cluster "
"settings are present or the benchmark may fail or behave unexpectedly."
% cluster_settings)
if len(plugins) > 0:
raise exceptions.SystemSetupError(
"You cannot specify any plugins for externally provisioned clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
s = lambda: None
p = [provisioner.no_op_provisioner()]
l = launcher.ExternalLauncher(cfg, metrics_store)
elif docker:
if len(plugins) > 0:
raise exceptions.SystemSetupError(
"You cannot specify any plugins for Docker clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
s = lambda: None
p = []
for node_id in node_ids:
p.append(
provisioner.docker_provisioner(cfg, car, cluster_settings,
challenge_root_path, node_id))
l = launcher.DockerLauncher(cfg, metrics_store)
else:
# It is a programmer error (and not a user error) if this function is called with wrong parameters
raise RuntimeError(
"One of sources, distribution, docker or external must be True")
return Mechanic(s, p, l)
def download(cfg):
car, plugins = load_team(cfg, external=False)
s = supplier.create(cfg,
sources=False,
distribution=True,
build=False,
car=car,
plugins=plugins)
binaries = s()
console.println(json.dumps(binaries, indent=2), force=True)
