def create(cfg, metrics_store, node_ip, node_http_port, all_node_ips, all_node_ids, sources=False, distribution=False, external=False, docker=False): race_root_path = paths.race_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) node_name_prefix = cfg.opts("provisioning", "node.name.prefix") car, plugins = load_team(cfg, external) if sources or distribution: s = supplier.create(cfg, sources, distribution, car, plugins) p = [] all_node_names = ["%s-%s" % (node_name_prefix, n) for n in all_node_ids] for node_id in node_ids: node_name = "%s-%s" % (node_name_prefix, node_id) p.append( provisioner.local(cfg, car, plugins, node_ip, node_http_port, all_node_ips, all_node_names, race_root_path, node_name)) l = launcher.ProcessLauncher(cfg) elif external: raise exceptions.RallyAssertionError("Externally provisioned clusters should not need to be managed by Rally's mechanic") elif docker: if len(plugins) > 0: raise exceptions.SystemSetupError("You cannot specify any plugins for Docker clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [] for node_id in node_ids: node_name = "%s-%s" % (node_name_prefix, node_id) p.append(provisioner.docker(cfg, car, node_ip, node_http_port, race_root_path, node_name)) l = launcher.DockerLauncher(cfg) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters raise RuntimeError("One of sources, distribution, docker or external must be True") return Mechanic(cfg, metrics_store, s, p, l)
def create(cfg, metrics_store, single_machine=True, sources=False, build=False, distribution=False, external=False, docker=False): challenge_root_path = paths.race_root(cfg) install_dir = "%s/install" % challenge_root_path log_dir = "%s/logs" % challenge_root_path io.ensure_dir(log_dir) if sources: try: src_dir = cfg.opts("source", "local.src.dir") except config.ConfigError: logger.exception("Cannot determine source directory") raise exceptions.SystemSetupError( "You cannot benchmark Elasticsearch from sources. Are you missing Gradle 2.13? Please install" " all prerequisites and reconfigure Rally with %s configure" % PROGRAM_NAME) remote_url = cfg.opts("source", "remote.repo.url") revision = cfg.opts("mechanic", "source.revision") gradle = cfg.opts("build", "gradle.bin") java_home = cfg.opts("runtime", "java8.home") s = lambda: supplier.from_sources(remote_url, src_dir, revision, gradle, java_home, log_dir, build) p = provisioner.local_provisioner(cfg, install_dir, single_machine) l = launcher.InProcessLauncher(cfg, metrics_store, challenge_root_path, log_dir) elif distribution: version = cfg.opts("mechanic", "distribution.version") repo_name = cfg.opts("mechanic", "distribution.repository") distributions_root = "%s/%s" % (cfg.opts( "node", "root.dir"), cfg.opts("source", "distribution.dir")) s = lambda: supplier.from_distribution(version=version, repo_name=repo_name, distributions_root= distributions_root) p = provisioner.local_provisioner(cfg, install_dir, single_machine) l = launcher.InProcessLauncher(cfg, metrics_store, challenge_root_path, log_dir) elif external: s = lambda: None p = provisioner.no_op_provisioner(cfg) l = launcher.ExternalLauncher(cfg, metrics_store) elif docker: s = lambda: None p = provisioner.docker_provisioner(cfg, install_dir) l = launcher.DockerLauncher(cfg, metrics_store) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters raise RuntimeError( "One of sources, distribution, docker or external must be True") return Mechanic(s, p, l)
def create(cfg, metrics_store, all_node_ips, cluster_settings=None, sources=False, build=False, distribution=False, external=False, docker=False): races_root = paths.races_root(cfg) challenge_root_path = paths.race_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) car, plugins = load_team(cfg, external) if sources or distribution: s = supplier.create(cfg, sources, distribution, build, challenge_root_path, car, plugins) p = [] for node_id in node_ids: p.append( provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, challenge_root_path, node_id)) l = launcher.InProcessLauncher(cfg, metrics_store, races_root) elif external: if cluster_settings: logging.getLogger(__name__).warning( "Cannot apply challenge-specific cluster settings [%s] for an externally provisioned cluster. Please ensure that the cluster " "settings are present or the benchmark may fail or behave unexpectedly." % cluster_settings) if len(plugins) > 0: raise exceptions.SystemSetupError( "You cannot specify any plugins for externally provisioned clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [provisioner.no_op_provisioner()] l = launcher.ExternalLauncher(cfg, metrics_store) elif docker: if len(plugins) > 0: raise exceptions.SystemSetupError( "You cannot specify any plugins for Docker clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [] for node_id in node_ids: p.append( provisioner.docker_provisioner(cfg, car, cluster_settings, challenge_root_path, node_id)) l = launcher.DockerLauncher(cfg, metrics_store) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters raise RuntimeError( "One of sources, distribution, docker or external must be True") return Mechanic(s, p, l)
def download(cfg): challenge_root_path = paths.race_root(cfg) car, plugins = load_team(cfg, external=False) s = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path=challenge_root_path, car=car, plugins=plugins) binaries = s() console.println(json.dumps(binaries, indent=2), force=True)
def create(cfg, metrics_store, all_node_ips, cluster_settings=None, sources=False, build=False, distribution=False, external=False, docker=False): races_root = paths.races_root(cfg) challenge_root_path = paths.race_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) repo = team.team_repo(cfg) # externally provisioned clusters do not support cars / plugins if external: car = None plugins = [] else: car = team.load_car(repo, cfg.opts("mechanic", "car.names")) plugins = team.load_plugins(repo, cfg.opts("mechanic", "car.plugins")) if sources: try: src_dir = cfg.opts("node", "src.root.dir") except config.ConfigError: logger.exception("Cannot determine source directory") raise exceptions.SystemSetupError( "You cannot benchmark Elasticsearch from sources. Did you install Gradle? Please install" " all prerequisites and reconfigure Rally with %s configure" % PROGRAM_NAME) remote_url = cfg.opts("source", "remote.repo.url") revision = cfg.opts("mechanic", "source.revision") gradle = cfg.opts("build", "gradle.bin") java_home = cfg.opts("runtime", "java.home") src_config = cfg.all_opts("source") s = lambda: supplier.from_sources( remote_url, src_dir, revision, gradle, java_home, challenge_root_path, plugins, src_config, build) p = [] for node_id in node_ids: p.append( provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, challenge_root_path, node_id)) l = launcher.InProcessLauncher(cfg, metrics_store, races_root) elif distribution: version = cfg.opts("mechanic", "distribution.version") repo_name = cfg.opts("mechanic", "distribution.repository") distributions_root = "%s/%s" % (cfg.opts( "node", "root.dir"), cfg.opts("source", "distribution.dir")) distribution_cfg = cfg.all_opts("distributions") s = lambda: supplier.from_distribution( version=version, repo_name=repo_name, distribution_config=distribution_cfg, distributions_root=distributions_root, plugins=plugins) p = [] for node_id in node_ids: p.append( provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, challenge_root_path, node_id)) l = launcher.InProcessLauncher(cfg, metrics_store, races_root) elif external: if cluster_settings: logger.warning( "Cannot apply challenge-specific cluster settings [%s] for an externally provisioned cluster. Please ensure " "that the cluster settings are present or the benchmark may fail or behave unexpectedly." % cluster_settings) if len(plugins) > 0: raise exceptions.SystemSetupError( "You cannot specify any plugins for externally provisioned clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [provisioner.no_op_provisioner()] l = launcher.ExternalLauncher(cfg, metrics_store) elif docker: if len(plugins) > 0: raise exceptions.SystemSetupError( "You cannot specify any plugins for Docker clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [] for node_id in node_ids: p.append( provisioner.docker_provisioner(cfg, car, cluster_settings, challenge_root_path, node_id)) l = launcher.DockerLauncher(cfg, metrics_store) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters raise RuntimeError( "One of sources, distribution, docker or external must be True") return Mechanic(s, p, l)
def create(cfg, metrics_store, all_node_ips, cluster_settings=None, sources=False, build=False, distribution=False, external=False, docker=False): races_root = paths.races_root(cfg) challenge_root_path = paths.race_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) repo = team.team_repo(cfg) # externally provisioned clusters do not support cars / plugins if external: car = None plugins = [] else: car = team.load_car(repo, cfg.opts("mechanic", "car.names")) plugins = team.load_plugins(repo, cfg.opts("mechanic", "car.plugins")) if sources: try: src_dir = cfg.opts("node", "src.root.dir") except config.ConfigError: logger.exception("Cannot determine source directory") raise exceptions.SystemSetupError("You cannot benchmark Elasticsearch from sources. Did you install Gradle? Please install" " all prerequisites and reconfigure Rally with %s configure" % PROGRAM_NAME) remote_url = cfg.opts("source", "remote.repo.url") revision = cfg.opts("mechanic", "source.revision") gradle = cfg.opts("build", "gradle.bin") java_home = cfg.opts("runtime", "java.home") src_config = cfg.all_opts("source") s = lambda: supplier.from_sources(remote_url, src_dir, revision, gradle, java_home, challenge_root_path, plugins, src_config, build) p = [] for node_id in node_ids: p.append(provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, challenge_root_path, node_id)) l = launcher.InProcessLauncher(cfg, metrics_store, races_root) elif distribution: version = cfg.opts("mechanic", "distribution.version") repo_name = cfg.opts("mechanic", "distribution.repository") distributions_root = "%s/%s" % (cfg.opts("node", "root.dir"), cfg.opts("source", "distribution.dir")) distribution_cfg = cfg.all_opts("distributions") s = lambda: supplier.from_distribution(version=version, repo_name=repo_name, distribution_config=distribution_cfg, distributions_root=distributions_root, plugins=plugins) p = [] for node_id in node_ids: p.append(provisioner.local_provisioner(cfg, car, plugins, cluster_settings, all_node_ips, challenge_root_path, node_id)) l = launcher.InProcessLauncher(cfg, metrics_store, races_root) elif external: if cluster_settings: logger.warning("Cannot apply challenge-specific cluster settings [%s] for an externally provisioned cluster. Please ensure " "that the cluster settings are present or the benchmark may fail or behave unexpectedly." % cluster_settings) if len(plugins) > 0: raise exceptions.SystemSetupError("You cannot specify any plugins for externally provisioned clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [provisioner.no_op_provisioner()] l = launcher.ExternalLauncher(cfg, metrics_store) elif docker: if len(plugins) > 0: raise exceptions.SystemSetupError("You cannot specify any plugins for Docker clusters. Please remove " "\"--elasticsearch-plugins\" and try again.") s = lambda: None p = [] for node_id in node_ids: p.append(provisioner.docker_provisioner(cfg, car, cluster_settings, challenge_root_path, node_id)) l = launcher.DockerLauncher(cfg, metrics_store) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters raise RuntimeError("One of sources, distribution, docker or external must be True") return Mechanic(s, p, l)