def test_create_suppliers_for_es_missing_distribution_plugin_source_skip( self): cfg = config.Config() # no distribution version! # cfg.add(config.Scope.application, "mechanic", "distribution.version", "") # default value from command line cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add( config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" ) cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --from-sources-skip-build --revision="community-plugin:current" (distribution version is missing!) with self.assertRaises(exceptions.SystemSetupError) as ctx: supplier.create(cfg, sources=True, distribution=False, build=False, challenge_root_path="/", plugins=[core_plugin, external_plugin]) self.assertRegex(ctx.exception.args[0], r"Could not determine version..*")
def create(cfg, metrics_store, node_ip, node_http_port, all_node_ips, all_node_ids, sources=False, distribution=False, external=False, docker=False): race_root_path = paths.race_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) node_name_prefix = cfg.opts("provisioning", "node.name.prefix") car, plugins = load_team(cfg, external) if sources or distribution: s = supplier.create(cfg, sources, distribution, car, plugins) p = [] all_node_names = ["%s-%s" % (node_name_prefix, n) for n in all_node_ids] for node_id in node_ids: node_name = "%s-%s" % (node_name_prefix, node_id) p.append( provisioner.local(cfg, car, plugins, node_ip, node_http_port, all_node_ips, all_node_names, race_root_path, node_name)) l = launcher.ProcessLauncher(cfg) elif external: raise exceptions.RallyAssertionError("Externally provisioned clusters should not need to be managed by Rally's mechanic") elif docker: if len(plugins) > 0: raise exceptions.SystemSetupError("You cannot specify any plugins for Docker clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [] for node_id in node_ids: node_name = "%s-%s" % (node_name_prefix, node_id) p.append(provisioner.docker(cfg, car, node_ip, node_http_port, race_root_path, node_name)) l = launcher.DockerLauncher(cfg) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters raise RuntimeError("One of sources, distribution, docker or external must be True") return Mechanic(cfg, metrics_store, s, p, l)
def install(cfg): root_path = paths.install_root(cfg) car, plugins = load_team(cfg, external=False) # A non-empty distribution-version is provided distribution = bool(cfg.opts("mechanic", "distribution.version", mandatory=False)) sources = not distribution build_type = cfg.opts("mechanic", "build.type") ip = cfg.opts("mechanic", "network.host") http_port = int(cfg.opts("mechanic", "network.http.port")) node_name = cfg.opts("mechanic", "node.name") master_nodes = cfg.opts("mechanic", "master.nodes") seed_hosts = cfg.opts("mechanic", "seed.hosts") if build_type == "tar": binary_supplier = supplier.create(cfg, sources, distribution, car, plugins) p = provisioner.local(cfg=cfg, car=car, plugins=plugins, ip=ip, http_port=http_port, all_node_ips=seed_hosts, all_node_names=master_nodes, target_root=root_path, node_name=node_name) node_config = p.prepare(binary=binary_supplier()) elif build_type == "docker": if len(plugins) > 0: raise exceptions.SystemSetupError("You cannot specify any plugins for Docker clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") p = provisioner.docker(cfg=cfg, car=car, ip=ip, http_port=http_port, target_root=root_path, node_name=node_name) # there is no binary for Docker that can be downloaded / built upfront node_config = p.prepare(binary=None) else: raise exceptions.SystemSetupError("Unknown build type [{}]".format(build_type)) provisioner.save_node_configuration(root_path, node_config) console.println(json.dumps({"installation-id": cfg.opts("system", "install.id")}, indent=2), force=True)
def test_create_suppliers_for_es_only_config(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0") # default value from command line cfg.add(config.Scope.application, "mechanic", "source.revision", "current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add( config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" ) cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") car = team.Car("default", root_path=None, config_paths=[]) composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", car=car) self.assertEqual(1, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier)
def test_create_suppliers_for_es_and_plugin_source_build(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "source.revision", "elasticsearch:abc,community-plugin:current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") cfg.add(config.Scope.application, "build", "gradle.bin", "/opt/gradle") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") cfg.add(config.Scope.application, "source", "remote.repo.url", "https://github.com/elastic/elasticsearch.git") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.subdir", "elasticsearch-extra/community-plugin") core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --revision="elasticsearch:abc,community-plugin:effab" composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=True, challenge_root_path="/", plugins=[ core_plugin, external_plugin ]) self.assertEqual(3, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchSourceSupplier) self.assertIsInstance(composite_supplier.suppliers[1], supplier.CorePluginSourceSupplier) self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin) self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier) self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin) self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_create_suppliers_for_es_distribution_plugin_source_skip(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0") # default value from command line cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "runtime", "java9.home", "/usr/local/bin/java9/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin") core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --pipeline=from-sources-skip-build composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=False, challenge_root_path="/", plugins=[ core_plugin, external_plugin ]) self.assertEqual(3, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier) self.assertIsInstance(composite_supplier.suppliers[1], supplier.PluginDistributionSupplier) self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin) self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier) self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin) self.assertIsNone(composite_supplier.suppliers[2].builder)
def test_create_suppliers_for_es_distribution_plugin_source_build(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0") # default value from command line cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin") car = team.Car("default", root_path=None, config_paths=[], variables={"build.jdk": "10"}) core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --revision="community-plugin:effab" --distribution-version="6.0.0" composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", car=car, plugins=[ core_plugin, external_plugin ]) self.assertEqual(3, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier) self.assertIsInstance(composite_supplier.suppliers[1], supplier.PluginDistributionSupplier) self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin) self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier) self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin) self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_create_suppliers_for_es_and_plugin_source_build(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "source.revision", "elasticsearch:abc,community-plugin:current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add( config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz", ) cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") cfg.add(config.Scope.application, "source", "remote.repo.url", "https://github.com/elastic/elasticsearch.git") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.subdir", "elasticsearch-extra/community-plugin") car = team.Car( "default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "build_command": "./gradlew assemble", "build.jdk": "11" }, ) core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --revision="elasticsearch:abc,community-plugin:effab" composite_supplier = supplier.create( cfg, sources=True, distribution=False, car=car, plugins=[core_plugin, external_plugin]) assert len(composite_supplier.suppliers) == 3 assert isinstance(composite_supplier.suppliers[0].source_supplier, supplier.ElasticsearchSourceSupplier) assert isinstance(composite_supplier.suppliers[1].source_supplier, supplier.CorePluginSourceSupplier) assert composite_supplier.suppliers[ 1].source_supplier.plugin == core_plugin assert isinstance(composite_supplier.suppliers[2].source_supplier, supplier.ExternalPluginSourceSupplier) assert composite_supplier.suppliers[ 2].source_supplier.plugin == external_plugin assert composite_supplier.suppliers[ 2].source_supplier.builder is not None
def create(cfg, metrics_store, all_node_ips, cluster_settings=None, sources=False, build=False, distribution=False, external=False, docker=False): races_root = paths.races_root(cfg) challenge_root_path = paths.race_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) car, plugins = load_team(cfg, external) if sources or distribution: s = supplier.create(cfg, sources, distribution, build, challenge_root_path, car, plugins) p = [] for node_id in node_ids: p.append( provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, challenge_root_path, node_id)) l = launcher.InProcessLauncher(cfg, metrics_store, races_root) elif external: if cluster_settings: logging.getLogger(__name__).warning( "Cannot apply challenge-specific cluster settings [%s] for an externally provisioned cluster. Please ensure that the cluster " "settings are present or the benchmark may fail or behave unexpectedly." % cluster_settings) if len(plugins) > 0: raise exceptions.SystemSetupError( "You cannot specify any plugins for externally provisioned clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [provisioner.no_op_provisioner()] l = launcher.ExternalLauncher(cfg, metrics_store) elif docker: if len(plugins) > 0: raise exceptions.SystemSetupError( "You cannot specify any plugins for Docker clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [] for node_id in node_ids: p.append( provisioner.docker_provisioner(cfg, car, cluster_settings, challenge_root_path, node_id)) l = launcher.DockerLauncher(cfg, metrics_store) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters raise RuntimeError( "One of sources, distribution, docker or external must be True") return Mechanic(s, p, l)
def download(cfg): car, plugins = load_team(cfg, external=False) s = supplier.create(cfg, sources=False, distribution=True, build=False, car=car, plugins=plugins) binaries = s() console.println(json.dumps(binaries, indent=2), force=True)