def _java_home(car): build_jdk = car.mandatory_var("build.jdk") try: _, path = jvm.resolve_path(int(build_jdk)) return path except ValueError: raise exceptions.SystemSetupError( "Car config key \"build.jdk\" is invalid: \"{}\" (must be int)". format(build_jdk))
def _java10_home(cfg): from esrally import config try: return cfg.opts("runtime", "java10.home") except config.ConfigError: raise exceptions.SystemSetupError( "No JDK 10 is configured. You cannot benchmark source builds of Elasticsearch on this machine. " "Please install a JDK 10 and reconfigure Rally with %s configure" % PROGRAM_NAME)
def _render_template(env, variables, file_name): try: template = env.get_template(io.basename(file_name)) # force a new line at the end. Jinja seems to remove it. return template.render(variables) + "\n" except jinja2.exceptions.TemplateSyntaxError as e: raise exceptions.InvalidSyntax("%s in %s" % (str(e), file_name)) except BaseException as e: raise exceptions.SystemSetupError("%s in %s" % (str(e), file_name))
def _determine_runtime_jdks(self, car): if self.override_runtime_jdk: return [self.override_runtime_jdk] else: runtime_jdks = car.mandatory_var("runtime.jdk") try: return [int(v) for v in runtime_jdks.split(",")] except ValueError: raise exceptions.SystemSetupError("Car config key \"runtime.jdk\" is invalid: \"{}\" (must be int)".format(runtime_jdks))
def _supply_requirements(sources, distribution, build, plugins, revisions, distribution_version): # per artifact (elasticsearch or a specific plugin): # * key: artifact # * value: ("source" | "distribution", distribution_version | revision, build = True | False) supply_requirements = {} # can only build Elasticsearch with source-related pipelines -> ignore revision in that case if "elasticsearch" in revisions and sources: supply_requirements["elasticsearch"] = ("source", _required_revision( revisions, "elasticsearch", "Elasticsearch"), build) else: # no revision given or explicitly specified that it's from a distribution -> must use a distribution supply_requirements["elasticsearch"] = ( "distribution", _required_version(distribution_version), False) for plugin in plugins: if plugin.core_plugin: # core plugins are entirely dependent upon Elasticsearch. supply_requirements[ plugin.name] = supply_requirements["elasticsearch"] else: # allow catch-all only if we're generally building from sources. If it is mixed, the user should tell explicitly. if plugin.name in revisions or ("all" in revisions and sources): # this plugin always needs to built unless we explicitly disable it; we cannot solely rely on the Rally pipeline. # We either have: # # * --pipeline=from-sources --distribution-version=X.Y.Z where the plugin should not be built but ES should be # a distributed version. # * --distribution-version=X.Y.Z --revision="my-plugin:abcdef" where the plugin should be built from sources. # pylint: disable=consider-using-ternary plugin_needs_build = (sources and build) or distribution # be a bit more lenient when checking for plugin revisions. This allows users to specify `--revision="current"` and # rely on Rally to do the right thing. try: plugin_revision = revisions[plugin.name] except KeyError: # maybe we can use the catch-all revision (only if it's not a git revision) plugin_revision = revisions.get("all") if not plugin_revision or SourceRepository.is_commit_hash( plugin_revision): raise exceptions.SystemSetupError( "No revision specified for plugin [%s]." % plugin.name) else: logging.getLogger(__name__).info( "Revision for [%s] is not explicitly defined. Using catch-all revision [%s].", plugin.name, plugin_revision) supply_requirements[plugin.name] = ("source", plugin_revision, plugin_needs_build) else: supply_requirements[plugin.name] = ( distribution, _required_version(distribution_version), False) return supply_requirements
def _src_dir(cfg, mandatory=True): # Don't let this spread across the whole module try: return cfg.opts("node", "src.root.dir", mandatory=mandatory) except exceptions.ConfigError: raise exceptions.SystemSetupError( "You cannot benchmark Elasticsearch from sources. Did you install Gradle? Please install" " all prerequisites and reconfigure Rally with %s configure" % PROGRAM_NAME)
def select_challenge(config, t): selected_challenge = config.opts("benchmarks", "challenge") for challenge in t.challenges: if challenge.name == selected_challenge: return challenge raise exceptions.SystemSetupError( "Unknown challenge [%s] for track [%s]. You can list the available tracks and their " "challenges with %s list tracks." % (selected_challenge, t.name, PROGRAM_NAME))
def check_can_handle_source_distribution(ctx): try: ctx.config.opts("source", "local.src.dir") except config.ConfigError: logging.exception("Rally is not configured to build from sources") raise exceptions.SystemSetupError( "Rally is not setup to build from sources. You can either benchmark a binary distribution or " "install the required software and reconfigure Rally with %s --configure." % PROGRAM_NAME)
def _try_init(self, may_skip_init=False): if not git.is_working_copy(self.src_dir): if self.has_remote(): self.logger.info("Downloading sources for %s from %s to %s.", self.name, self.remote_url, self.src_dir) git.clone(self.src_dir, self.remote_url) elif os.path.isdir(self.src_dir) and may_skip_init: self.logger.info("Skipping repository initialization for %s.", self.name) else: exceptions.SystemSetupError("A remote repository URL is mandatory for %s" % self.name)
def setup(self, sources=False): # to load the track we need to know the correct cluster distribution version. Usually, this value should be set # but there are rare cases (external pipeline and user did not specify the distribution version) where we need # to derive it ourselves. For source builds we always assume "master" if not sources and not self.cfg.exists("mechanic", "distribution.version"): distribution_version = mechanic.cluster_distribution_version( self.cfg) self.logger.info("Automatically derived distribution version [%s]", distribution_version) self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version) min_es_version = versions.Version.from_string( version.minimum_es_version()) specified_version = versions.Version.from_string( distribution_version) if specified_version < min_es_version: raise exceptions.SystemSetupError( f"Cluster version must be at least [{min_es_version}] but was [{distribution_version}]" ) self.current_track = track.load_track(self.cfg) self.track_revision = self.cfg.opts("track", "repository.revision", mandatory=False) challenge_name = self.cfg.opts("track", "challenge.name") self.current_challenge = self.current_track.find_challenge_or_default( challenge_name) if self.current_challenge is None: raise exceptions.SystemSetupError( "Track [{}] does not provide challenge [{}]. List the available tracks with {} list tracks." .format(self.current_track.name, challenge_name, PROGRAM_NAME)) if self.current_challenge.user_info: console.info(self.current_challenge.user_info) self.race = metrics.create_race(self.cfg, self.current_track, self.current_challenge, self.track_revision) self.metrics_store = metrics.metrics_store( self.cfg, track=self.race.track_name, challenge=self.race.challenge_name, read_only=False) self.race_store = metrics.race_store(self.cfg)
def from_distribution(version, repo_name, distributions_root): if version.strip() == "": raise exceptions.SystemSetupError( "Could not determine version. Please specify the Elasticsearch distribution " "to download with the command line parameter --distribution-version. " "E.g. --distribution-version=5.0.0") io.ensure_dir(distributions_root) distribution_path = "%s/elasticsearch-%s.tar.gz" % (distributions_root, version) try: repo = distribution_repos[repo_name] except KeyError: raise exceptions.SystemSetupError( "Unknown distribution repository [%s]. Valid values are: [%s]" % (repo_name, ",".join(distribution_repos.keys()))) download_url = repo.download_url(version) logger.info("Resolved download URL [%s] for version [%s]" % (download_url, version)) if not os.path.isfile(distribution_path) or repo.must_download: try: logger.info("Starting download of Elasticsearch [%s]" % version) progress = net.Progress("[INFO] Downloading Elasticsearch %s" % version) net.download(download_url, distribution_path, progress_indicator=progress) progress.finish() logger.info("Successfully downloaded Elasticsearch [%s]." % version) except urllib.error.HTTPError: console.println("[FAILED]") logging.exception( "Cannot download Elasticsearch distribution for version [%s] from [%s]." % (version, download_url)) raise exceptions.SystemSetupError( "Cannot download Elasticsearch distribution from [%s]. Please check that the specified " "version [%s] is correct." % (download_url, version)) else: logger.info( "Skipping download for version [%s]. Found an existing binary locally at [%s]." % (version, distribution_path)) return distribution_path
def run(cfg): logger = logging.getLogger(__name__) name = cfg.opts("race", "pipeline") race_id = cfg.opts("system", "race.id") logger.info("Race id [%s]", race_id) if len(name) == 0: # assume from-distribution pipeline if distribution.version has been specified and --pipeline cli arg not set if cfg.exists("mechanic", "distribution.version"): name = "from-distribution" else: name = "from-sources" logger.info( "User specified no pipeline. Automatically derived pipeline [%s].", name) cfg.add(config.Scope.applicationOverride, "race", "pipeline", name) else: logger.info("User specified pipeline [%s].", name) if os.environ.get("RALLY_RUNNING_IN_DOCKER", "").upper() == "TRUE": # in this case only benchmarking remote Elasticsearch clusters makes sense if name != "benchmark-only": raise exceptions.SystemSetupError( "Only the [benchmark-only] pipeline is supported by the Rally Docker image.\n" "Add --pipeline=benchmark-only in your Rally arguments and try again.\n" "For more details read the docs for the benchmark-only pipeline in {}\n" .format(doc_link("pipelines.html#benchmark-only"))) try: pipeline = pipelines[name] except KeyError: raise exceptions.SystemSetupError( "Unknown pipeline [%s]. List the available pipelines with %s list pipelines." % (name, PROGRAM_NAME)) try: pipeline(cfg) except exceptions.RallyError as e: # just pass on our own errors. It should be treated differently on top-level raise e except KeyboardInterrupt: logger.info("User has cancelled the benchmark.") except BaseException: tb = sys.exc_info()[2] raise exceptions.RallyError( "This race ended with a fatal crash.").with_traceback(tb)
def compare(cfg): baseline_ts = cfg.opts("reporting", "baseline.timestamp") contender_ts = cfg.opts("reporting", "contender.timestamp") if not baseline_ts or not contender_ts: raise exceptions.SystemSetupError( "compare needs baseline and a contender") race_store = metrics.race_store(cfg) ComparisonReporter(cfg).report(race_store.find_by_timestamp(baseline_ts), race_store.find_by_timestamp(contender_ts))
def _java_home(cfg): from esrally import config try: return cfg.opts("runtime", "java.home") except config.ConfigError: logger.exception("Cannot determine Java home.") raise exceptions.SystemSetupError( "No JDK is configured. You cannot benchmark Elasticsearch on this machine. Please install" " all prerequisites and reconfigure Rally with %s configure" % PROGRAM_NAME)
def download_benchmark_candidate(ctx): version = ctx.config.opts("source", "distribution.version") repo_name = ctx.config.opts("source", "distribution.repository") if version.strip() == "": raise exceptions.SystemSetupError( "Could not determine version. Please specify the Elasticsearch distribution " "to download with the command line parameter --distribution-version. " "E.g. --distribution-version=5.0.0") distributions_root = "%s/%s" % (ctx.config.opts( "system", "root.dir"), ctx.config.opts("source", "distribution.dir")) io.ensure_dir(distributions_root) distribution_path = "%s/elasticsearch-%s.tar.gz" % (distributions_root, version) try: repo = distribution_repos[repo_name] except KeyError: raise exceptions.SystemSetupError( "Unknown distribution repository [%s]. Valid values are: [%s]" % (repo_name, ",".join(distribution_repos.keys()))) download_url = repo.download_url(version) logger.info("Resolved download URL [%s] for version [%s]" % (download_url, version)) if not os.path.isfile(distribution_path) or repo.must_download: logger.info("Downloading distribution for version [%s]." % version) try: print("Downloading Elasticsearch %s ..." % version) net.download(download_url, distribution_path) except urllib.error.HTTPError: logging.exception( "Cannot download Elasticsearch distribution for version [%s] from [%s]." % (version, download_url)) raise exceptions.SystemSetupError( "Cannot download Elasticsearch distribution from [%s]. Please check that the specified " "version [%s] is correct." % (download_url, version)) else: logger.info( "Skipping download for version [%s]. Found an existing binary locally at [%s]." % (version, distribution_path)) ctx.config.add(config.Scope.invocation, "builder", "candidate.bin.path", distribution_path)
def select_challenge(config, t): challenge_name = config.opts("track", "challenge.name") selected_challenge = t.find_challenge_or_default(challenge_name) if not selected_challenge: raise exceptions.SystemSetupError( "Unknown challenge [%s] for track [%s]. You can list the available tracks and their " "challenges with %s list tracks." % (challenge_name, t.name, PROGRAM_NAME)) return selected_challenge
def compare(cfg): baseline_id = cfg.opts("reporting", "baseline.id") contender_id = cfg.opts("reporting", "contender.id") if not baseline_id or not contender_id: raise exceptions.SystemSetupError( "compare needs baseline and a contender") race_store = metrics.race_store(cfg) ComparisonReporter(cfg).report(race_store.find_by_race_id(baseline_id), race_store.find_by_race_id(contender_id))
def determine_runtime_jdks(): override_runtime_jdk = cfg.opts("mechanic", "runtime.jdk") if override_runtime_jdk: return [override_runtime_jdk] else: try: return [int(v) for v in car_runtime_jdks.split(",")] except ValueError: raise exceptions.SystemSetupError( "Car config key \"runtime.jdk\" is invalid: \"{}\" (must be int)" .format(car_runtime_jdks))
def _es_log_config(self): logging_yml_path = "%s/config/logging.yml" % self.binary_path log4j2_properties_path = "%s/config/log4j2.properties" % self.binary_path if os.path.isfile(logging_yml_path): return "logging.yml", logging_yml_path elif os.path.isfile(log4j2_properties_path): return "log4j2.properties", log4j2_properties_path else: raise exceptions.SystemSetupError( "Unrecognized Elasticsearch log config file format")
def probe(src, *args, **kwargs): # Probe for -C if not process.exit_status_as_bool(lambda: process.run_subprocess_with_logging( "git -C {} --version".format(src), level=logging.DEBUG), quiet=True): version = process.run_subprocess_with_output("git --version") if version: version = str(version).strip() else: version = "Unknown" raise exceptions.SystemSetupError("Your git version is [%s] but Rally requires at least git 1.9. Please update git." % version) return f(src, *args, **kwargs)
def _data_paths(self): if "data_paths" in self.car.variables: data_paths = self.car.variables["data_paths"] if isinstance(data_paths, str): return [data_paths] elif isinstance(data_paths, list): return data_paths else: raise exceptions.SystemSetupError("Expected [data_paths] to be either a string or a list but was [%s]." % type(data_paths)) else: return [os.path.join(self.es_home_path, "data")]
def convert_hosts(configured_host_list): hosts = [] try: for authority in configured_host_list: host, port = authority.split(":") hosts.append({"host": host, "port": port}) return hosts except ValueError: msg = "Could not convert hosts [%s] (expected a list of host:port pairs e.g. host1:9200,host2:9200)." % configured_host_list logger.exception(msg) raise exceptions.SystemSetupError(msg)
def start(cfg): root_path = paths.install_root(cfg) race_id = cfg.opts("system", "race.id") # avoid double-launching - we expect that the node file is absent with contextlib.suppress(FileNotFoundError): _load_node_file(root_path) install_id = cfg.opts("system", "install.id") raise exceptions.SystemSetupError("A node with this installation id is already running. Please stop it first " "with {} stop --installation-id={}".format(PROGRAM_NAME, install_id)) node_config = provisioner.load_node_configuration(root_path) if node_config.build_type == "tar": node_launcher = launcher.ProcessLauncher(cfg) elif node_config.build_type == "docker": node_launcher = launcher.DockerLauncher(cfg) else: raise exceptions.SystemSetupError("Unknown build type [{}]".format(node_config.build_type)) nodes = node_launcher.start([node_config]) _store_node_file(root_path, (nodes, race_id))
def register(self, phase, hook): logger.info( "Registering install hook [%s] for phase [%s] in plugin [%s]" % (hook.__name__, phase, self.plugin.name)) if not ProvisioningPhase.valid(phase): raise exceptions.SystemSetupError( "Provisioning phase [%s] is unknown. Valid phases are: %s." % (phase, ProvisioningPhase.names())) if phase not in self.hooks: self.hooks[phase] = [] self.hooks[phase].append(hook)
def register(self, phase, hook): self.logger.info( "Registering bootstrap hook [%s] for phase [%s] in component [%s]", hook.__name__, phase, self.component.name) if not BootstrapPhase.valid(phase): raise exceptions.SystemSetupError( "Unknown bootstrap phase [{}]. Valid phases are: {}.".format( phase, BootstrapPhase.names())) if phase not in self.hooks: self.hooks[phase] = [] self.hooks[phase].append(hook)
def install(cfg): root_path = paths.install_root(cfg) car, plugins = load_team(cfg, external=False) # A non-empty distribution-version is provided distribution = bool(cfg.opts("mechanic", "distribution.version", mandatory=False)) sources = not distribution build_type = cfg.opts("mechanic", "build.type") ip = cfg.opts("mechanic", "network.host") http_port = int(cfg.opts("mechanic", "network.http.port")) node_name = cfg.opts("mechanic", "node.name") master_nodes = cfg.opts("mechanic", "master.nodes") seed_hosts = cfg.opts("mechanic", "seed.hosts") if build_type == "tar": binary_supplier = supplier.create(cfg, sources, distribution, car, plugins) p = provisioner.local( cfg=cfg, car=car, plugins=plugins, ip=ip, http_port=http_port, all_node_ips=seed_hosts, all_node_names=master_nodes, target_root=root_path, node_name=node_name, ) node_config = p.prepare(binary=binary_supplier()) elif build_type == "docker": if len(plugins) > 0: raise exceptions.SystemSetupError( "You cannot specify any plugins for Docker clusters. Please remove " '"--elasticsearch-plugins" and try again.' ) p = provisioner.docker(cfg=cfg, car=car, ip=ip, http_port=http_port, target_root=root_path, node_name=node_name) # there is no binary for Docker that can be downloaded / built upfront node_config = p.prepare(binary=None) else: raise exceptions.SystemSetupError("Unknown build type [{}]".format(build_type)) provisioner.save_node_configuration(root_path, node_config) console.println(json.dumps({"installation-id": cfg.opts("system", "install.id")}, indent=2), force=True)
def load_car(repo, name, car_params=None): class Component: def __init__(self, root_path, entry_point): self.root_path = root_path self.entry_point = entry_point root_path = None # preserve order as we append to existing config files later during provisioning. all_config_paths = [] all_config_base_vars = {} all_car_vars = {} for n in name: descriptor = CarLoader(repo).load_car(n, car_params) for p in descriptor.config_paths: if p not in all_config_paths: all_config_paths.append(p) for p in descriptor.root_paths: # probe whether we have a root path if BootstrapHookHandler( Component(root_path=p, entry_point=Car.entry_point)).can_load(): if not root_path: root_path = p # multiple cars are based on the same hook elif root_path != p: raise exceptions.SystemSetupError( "Invalid car: {}. Multiple bootstrap hooks are forbidden." .format(name)) all_config_base_vars.update(descriptor.config_base_variables) all_car_vars.update(descriptor.variables) if len(all_config_paths) == 0: raise exceptions.SystemSetupError( "At least one config base is required for car {}".format(name)) variables = {} # car variables *always* take precedence over config base variables variables.update(all_config_base_vars) variables.update(all_car_vars) return Car(name, root_path, all_config_paths, variables)
def _render_template(self, loader, template_name, variables): try: env = jinja2.Environment(loader=loader) for k, v in variables.items(): env.globals[k] = v template = env.get_template(template_name) return template.render() except jinja2.exceptions.TemplateSyntaxError as e: raise exceptions.InvalidSyntax("%s in %s" % (str(e), template_name)) except BaseException as e: raise exceptions.SystemSetupError("%s in %s" % (str(e), template_name))
def load(self): root_module = self.loader.load() try: # every module needs to have a register() method root_module.register(self) except exceptions.RallyError: # just pass our own exceptions transparently. raise except BaseException: msg = "Could not load install hooks in [%s]" % self.loader.root_path logger.exception(msg) raise exceptions.SystemSetupError(msg)
def _url_for(self, user_defined_key, default_key, mandatory=True): try: if user_defined_key in self.cfg: url_template = self.cfg[user_defined_key] else: url_template = self.cfg[default_key] except KeyError: if mandatory: raise exceptions.SystemSetupError("Neither config key [{}] nor [{}] is defined.".format(user_defined_key, default_key)) else: return None return self.template_renderer.render(url_template)