def _java_home(car):
build_jdk = car.mandatory_var("build.jdk")
try:
_, path = jvm.resolve_path(int(build_jdk))
return path
except ValueError:
raise exceptions.SystemSetupError(
"Car config key \"build.jdk\" is invalid: \"{}\" (must be int)".
format(build_jdk))
def _java10_home(cfg):
from esrally import config
try:
return cfg.opts("runtime", "java10.home")
except config.ConfigError:
raise exceptions.SystemSetupError(
"No JDK 10 is configured. You cannot benchmark source builds of Elasticsearch on this machine. "
"Please install a JDK 10 and reconfigure Rally with %s configure" %
PROGRAM_NAME)
def _render_template(env, variables, file_name):
try:
template = env.get_template(io.basename(file_name))
# force a new line at the end. Jinja seems to remove it.
return template.render(variables) + "\n"
except jinja2.exceptions.TemplateSyntaxError as e:
raise exceptions.InvalidSyntax("%s in %s" % (str(e), file_name))
except BaseException as e:
raise exceptions.SystemSetupError("%s in %s" % (str(e), file_name))
def _determine_runtime_jdks(self, car):
if self.override_runtime_jdk:
return [self.override_runtime_jdk]
else:
runtime_jdks = car.mandatory_var("runtime.jdk")
try:
return [int(v) for v in runtime_jdks.split(",")]
except ValueError:
raise exceptions.SystemSetupError("Car config key \"runtime.jdk\" is invalid: \"{}\" (must be int)".format(runtime_jdks))
def _supply_requirements(sources, distribution, build, plugins, revisions,
distribution_version):
# per artifact (elasticsearch or a specific plugin):
# * key: artifact
# * value: ("source" | "distribution", distribution_version | revision, build = True | False)
supply_requirements = {}
# can only build Elasticsearch with source-related pipelines -> ignore revision in that case
if "elasticsearch" in revisions and sources:
supply_requirements["elasticsearch"] = ("source",
_required_revision(
revisions, "elasticsearch",
"Elasticsearch"), build)
else:
# no revision given or explicitly specified that it's from a distribution -> must use a distribution
supply_requirements["elasticsearch"] = (
"distribution", _required_version(distribution_version), False)
for plugin in plugins:
if plugin.core_plugin:
# core plugins are entirely dependent upon Elasticsearch.
supply_requirements[
plugin.name] = supply_requirements["elasticsearch"]
else:
# allow catch-all only if we're generally building from sources. If it is mixed, the user should tell explicitly.
if plugin.name in revisions or ("all" in revisions and sources):
# this plugin always needs to built unless we explicitly disable it; we cannot solely rely on the Rally pipeline.
# We either have:
#
# * --pipeline=from-sources --distribution-version=X.Y.Z where the plugin should not be built but ES should be
# a distributed version.
# * --distribution-version=X.Y.Z --revision="my-plugin:abcdef" where the plugin should be built from sources.
# pylint: disable=consider-using-ternary
plugin_needs_build = (sources and build) or distribution
# be a bit more lenient when checking for plugin revisions. This allows users to specify `--revision="current"` and
# rely on Rally to do the right thing.
try:
plugin_revision = revisions[plugin.name]
except KeyError:
# maybe we can use the catch-all revision (only if it's not a git revision)
plugin_revision = revisions.get("all")
if not plugin_revision or SourceRepository.is_commit_hash(
plugin_revision):
raise exceptions.SystemSetupError(
"No revision specified for plugin [%s]." %
plugin.name)
else:
logging.getLogger(__name__).info(
"Revision for [%s] is not explicitly defined. Using catch-all revision [%s].",
plugin.name, plugin_revision)
supply_requirements[plugin.name] = ("source", plugin_revision,
plugin_needs_build)
else:
supply_requirements[plugin.name] = (
distribution, _required_version(distribution_version),
False)
return supply_requirements
def _src_dir(cfg, mandatory=True):
# Don't let this spread across the whole module
try:
return cfg.opts("node", "src.root.dir", mandatory=mandatory)
except exceptions.ConfigError:
raise exceptions.SystemSetupError(
"You cannot benchmark Elasticsearch from sources. Did you install Gradle? Please install"
" all prerequisites and reconfigure Rally with %s configure" %
PROGRAM_NAME)
def select_challenge(config, t):
selected_challenge = config.opts("benchmarks", "challenge")
for challenge in t.challenges:
if challenge.name == selected_challenge:
return challenge
raise exceptions.SystemSetupError(
"Unknown challenge [%s] for track [%s]. You can list the available tracks and their "
"challenges with %s list tracks." %
(selected_challenge, t.name, PROGRAM_NAME))
def check_can_handle_source_distribution(ctx):
try:
ctx.config.opts("source", "local.src.dir")
except config.ConfigError:
logging.exception("Rally is not configured to build from sources")
raise exceptions.SystemSetupError(
"Rally is not setup to build from sources. You can either benchmark a binary distribution or "
"install the required software and reconfigure Rally with %s --configure."
% PROGRAM_NAME)
def _try_init(self, may_skip_init=False):
if not git.is_working_copy(self.src_dir):
if self.has_remote():
self.logger.info("Downloading sources for %s from %s to %s.", self.name, self.remote_url, self.src_dir)
git.clone(self.src_dir, self.remote_url)
elif os.path.isdir(self.src_dir) and may_skip_init:
self.logger.info("Skipping repository initialization for %s.", self.name)
else:
exceptions.SystemSetupError("A remote repository URL is mandatory for %s" % self.name)
def setup(self, sources=False):
# to load the track we need to know the correct cluster distribution version. Usually, this value should be set
# but there are rare cases (external pipeline and user did not specify the distribution version) where we need
# to derive it ourselves. For source builds we always assume "master"
if not sources and not self.cfg.exists("mechanic",
"distribution.version"):
distribution_version = mechanic.cluster_distribution_version(
self.cfg)
self.logger.info("Automatically derived distribution version [%s]",
distribution_version)
self.cfg.add(config.Scope.benchmark, "mechanic",
"distribution.version", distribution_version)
min_es_version = versions.Version.from_string(
version.minimum_es_version())
specified_version = versions.Version.from_string(
distribution_version)
if specified_version < min_es_version:
raise exceptions.SystemSetupError(
f"Cluster version must be at least [{min_es_version}] but was [{distribution_version}]"
)
self.current_track = track.load_track(self.cfg)
self.track_revision = self.cfg.opts("track",
"repository.revision",
mandatory=False)
challenge_name = self.cfg.opts("track", "challenge.name")
self.current_challenge = self.current_track.find_challenge_or_default(
challenge_name)
if self.current_challenge is None:
raise exceptions.SystemSetupError(
"Track [{}] does not provide challenge [{}]. List the available tracks with {} list tracks."
.format(self.current_track.name, challenge_name, PROGRAM_NAME))
if self.current_challenge.user_info:
console.info(self.current_challenge.user_info)
self.race = metrics.create_race(self.cfg, self.current_track,
self.current_challenge,
self.track_revision)
self.metrics_store = metrics.metrics_store(
self.cfg,
track=self.race.track_name,
challenge=self.race.challenge_name,
read_only=False)
self.race_store = metrics.race_store(self.cfg)
def from_distribution(version, repo_name, distributions_root):
if version.strip() == "":
raise exceptions.SystemSetupError(
"Could not determine version. Please specify the Elasticsearch distribution "
"to download with the command line parameter --distribution-version. "
"E.g. --distribution-version=5.0.0")
io.ensure_dir(distributions_root)
distribution_path = "%s/elasticsearch-%s.tar.gz" % (distributions_root,
version)
try:
repo = distribution_repos[repo_name]
except KeyError:
raise exceptions.SystemSetupError(
"Unknown distribution repository [%s]. Valid values are: [%s]" %
(repo_name, ",".join(distribution_repos.keys())))
download_url = repo.download_url(version)
logger.info("Resolved download URL [%s] for version [%s]" %
(download_url, version))
if not os.path.isfile(distribution_path) or repo.must_download:
try:
logger.info("Starting download of Elasticsearch [%s]" % version)
progress = net.Progress("[INFO] Downloading Elasticsearch %s" %
version)
net.download(download_url,
distribution_path,
progress_indicator=progress)
progress.finish()
logger.info("Successfully downloaded Elasticsearch [%s]." %
version)
except urllib.error.HTTPError:
console.println("[FAILED]")
logging.exception(
"Cannot download Elasticsearch distribution for version [%s] from [%s]."
% (version, download_url))
raise exceptions.SystemSetupError(
"Cannot download Elasticsearch distribution from [%s]. Please check that the specified "
"version [%s] is correct." % (download_url, version))
else:
logger.info(
"Skipping download for version [%s]. Found an existing binary locally at [%s]."
% (version, distribution_path))
return distribution_path
def run(cfg):
logger = logging.getLogger(__name__)
name = cfg.opts("race", "pipeline")
race_id = cfg.opts("system", "race.id")
logger.info("Race id [%s]", race_id)
if len(name) == 0:
# assume from-distribution pipeline if distribution.version has been specified and --pipeline cli arg not set
if cfg.exists("mechanic", "distribution.version"):
name = "from-distribution"
else:
name = "from-sources"
logger.info(
"User specified no pipeline. Automatically derived pipeline [%s].",
name)
cfg.add(config.Scope.applicationOverride, "race", "pipeline", name)
else:
logger.info("User specified pipeline [%s].", name)
if os.environ.get("RALLY_RUNNING_IN_DOCKER", "").upper() == "TRUE":
# in this case only benchmarking remote Elasticsearch clusters makes sense
if name != "benchmark-only":
raise exceptions.SystemSetupError(
"Only the [benchmark-only] pipeline is supported by the Rally Docker image.\n"
"Add --pipeline=benchmark-only in your Rally arguments and try again.\n"
"For more details read the docs for the benchmark-only pipeline in {}\n"
.format(doc_link("pipelines.html#benchmark-only")))
try:
pipeline = pipelines[name]
except KeyError:
raise exceptions.SystemSetupError(
"Unknown pipeline [%s]. List the available pipelines with %s list pipelines."
% (name, PROGRAM_NAME))
try:
pipeline(cfg)
except exceptions.RallyError as e:
# just pass on our own errors. It should be treated differently on top-level
raise e
except KeyboardInterrupt:
logger.info("User has cancelled the benchmark.")
except BaseException:
tb = sys.exc_info()[2]
raise exceptions.RallyError(
"This race ended with a fatal crash.").with_traceback(tb)
def compare(cfg):
baseline_ts = cfg.opts("reporting", "baseline.timestamp")
contender_ts = cfg.opts("reporting", "contender.timestamp")
if not baseline_ts or not contender_ts:
raise exceptions.SystemSetupError(
"compare needs baseline and a contender")
race_store = metrics.race_store(cfg)
ComparisonReporter(cfg).report(race_store.find_by_timestamp(baseline_ts),
race_store.find_by_timestamp(contender_ts))
def _java_home(cfg):
from esrally import config
try:
return cfg.opts("runtime", "java.home")
except config.ConfigError:
logger.exception("Cannot determine Java home.")
raise exceptions.SystemSetupError(
"No JDK is configured. You cannot benchmark Elasticsearch on this machine. Please install"
" all prerequisites and reconfigure Rally with %s configure" %
PROGRAM_NAME)
def download_benchmark_candidate(ctx):
version = ctx.config.opts("source", "distribution.version")
repo_name = ctx.config.opts("source", "distribution.repository")
if version.strip() == "":
raise exceptions.SystemSetupError(
"Could not determine version. Please specify the Elasticsearch distribution "
"to download with the command line parameter --distribution-version. "
"E.g. --distribution-version=5.0.0")
distributions_root = "%s/%s" % (ctx.config.opts(
"system", "root.dir"), ctx.config.opts("source", "distribution.dir"))
io.ensure_dir(distributions_root)
distribution_path = "%s/elasticsearch-%s.tar.gz" % (distributions_root,
version)
try:
repo = distribution_repos[repo_name]
except KeyError:
raise exceptions.SystemSetupError(
"Unknown distribution repository [%s]. Valid values are: [%s]" %
(repo_name, ",".join(distribution_repos.keys())))
download_url = repo.download_url(version)
logger.info("Resolved download URL [%s] for version [%s]" %
(download_url, version))
if not os.path.isfile(distribution_path) or repo.must_download:
logger.info("Downloading distribution for version [%s]." % version)
try:
print("Downloading Elasticsearch %s ..." % version)
net.download(download_url, distribution_path)
except urllib.error.HTTPError:
logging.exception(
"Cannot download Elasticsearch distribution for version [%s] from [%s]."
% (version, download_url))
raise exceptions.SystemSetupError(
"Cannot download Elasticsearch distribution from [%s]. Please check that the specified "
"version [%s] is correct." % (download_url, version))
else:
logger.info(
"Skipping download for version [%s]. Found an existing binary locally at [%s]."
% (version, distribution_path))
ctx.config.add(config.Scope.invocation, "builder", "candidate.bin.path",
distribution_path)
def select_challenge(config, t):
challenge_name = config.opts("track", "challenge.name")
selected_challenge = t.find_challenge_or_default(challenge_name)
if not selected_challenge:
raise exceptions.SystemSetupError(
"Unknown challenge [%s] for track [%s]. You can list the available tracks and their "
"challenges with %s list tracks." %
(challenge_name, t.name, PROGRAM_NAME))
return selected_challenge
def compare(cfg):
baseline_id = cfg.opts("reporting", "baseline.id")
contender_id = cfg.opts("reporting", "contender.id")
if not baseline_id or not contender_id:
raise exceptions.SystemSetupError(
"compare needs baseline and a contender")
race_store = metrics.race_store(cfg)
ComparisonReporter(cfg).report(race_store.find_by_race_id(baseline_id),
race_store.find_by_race_id(contender_id))
def determine_runtime_jdks():
override_runtime_jdk = cfg.opts("mechanic", "runtime.jdk")
if override_runtime_jdk:
return [override_runtime_jdk]
else:
try:
return [int(v) for v in car_runtime_jdks.split(",")]
except ValueError:
raise exceptions.SystemSetupError(
"Car config key \"runtime.jdk\" is invalid: \"{}\" (must be int)"
.format(car_runtime_jdks))
def _es_log_config(self):
logging_yml_path = "%s/config/logging.yml" % self.binary_path
log4j2_properties_path = "%s/config/log4j2.properties" % self.binary_path
if os.path.isfile(logging_yml_path):
return "logging.yml", logging_yml_path
elif os.path.isfile(log4j2_properties_path):
return "log4j2.properties", log4j2_properties_path
else:
raise exceptions.SystemSetupError(
"Unrecognized Elasticsearch log config file format")
def probe(src, *args, **kwargs):
# Probe for -C
if not process.exit_status_as_bool(lambda: process.run_subprocess_with_logging(
"git -C {} --version".format(src), level=logging.DEBUG), quiet=True):
version = process.run_subprocess_with_output("git --version")
if version:
version = str(version).strip()
else:
version = "Unknown"
raise exceptions.SystemSetupError("Your git version is [%s] but Rally requires at least git 1.9. Please update git." % version)
return f(src, *args, **kwargs)
def _data_paths(self):
if "data_paths" in self.car.variables:
data_paths = self.car.variables["data_paths"]
if isinstance(data_paths, str):
return [data_paths]
elif isinstance(data_paths, list):
return data_paths
else:
raise exceptions.SystemSetupError("Expected [data_paths] to be either a string or a list but was [%s]." % type(data_paths))
else:
return [os.path.join(self.es_home_path, "data")]
def convert_hosts(configured_host_list):
hosts = []
try:
for authority in configured_host_list:
host, port = authority.split(":")
hosts.append({"host": host, "port": port})
return hosts
except ValueError:
msg = "Could not convert hosts [%s] (expected a list of host:port pairs e.g. host1:9200,host2:9200)." % configured_host_list
logger.exception(msg)
raise exceptions.SystemSetupError(msg)
def start(cfg):
root_path = paths.install_root(cfg)
race_id = cfg.opts("system", "race.id")
# avoid double-launching - we expect that the node file is absent
with contextlib.suppress(FileNotFoundError):
_load_node_file(root_path)
install_id = cfg.opts("system", "install.id")
raise exceptions.SystemSetupError("A node with this installation id is already running. Please stop it first "
"with {} stop --installation-id={}".format(PROGRAM_NAME, install_id))
node_config = provisioner.load_node_configuration(root_path)
if node_config.build_type == "tar":
node_launcher = launcher.ProcessLauncher(cfg)
elif node_config.build_type == "docker":
node_launcher = launcher.DockerLauncher(cfg)
else:
raise exceptions.SystemSetupError("Unknown build type [{}]".format(node_config.build_type))
nodes = node_launcher.start([node_config])
_store_node_file(root_path, (nodes, race_id))
def register(self, phase, hook):
logger.info(
"Registering install hook [%s] for phase [%s] in plugin [%s]" %
(hook.__name__, phase, self.plugin.name))
if not ProvisioningPhase.valid(phase):
raise exceptions.SystemSetupError(
"Provisioning phase [%s] is unknown. Valid phases are: %s." %
(phase, ProvisioningPhase.names()))
if phase not in self.hooks:
self.hooks[phase] = []
self.hooks[phase].append(hook)
def register(self, phase, hook):
self.logger.info(
"Registering bootstrap hook [%s] for phase [%s] in component [%s]",
hook.__name__, phase, self.component.name)
if not BootstrapPhase.valid(phase):
raise exceptions.SystemSetupError(
"Unknown bootstrap phase [{}]. Valid phases are: {}.".format(
phase, BootstrapPhase.names()))
if phase not in self.hooks:
self.hooks[phase] = []
self.hooks[phase].append(hook)
def install(cfg):
root_path = paths.install_root(cfg)
car, plugins = load_team(cfg, external=False)
# A non-empty distribution-version is provided
distribution = bool(cfg.opts("mechanic", "distribution.version", mandatory=False))
sources = not distribution
build_type = cfg.opts("mechanic", "build.type")
ip = cfg.opts("mechanic", "network.host")
http_port = int(cfg.opts("mechanic", "network.http.port"))
node_name = cfg.opts("mechanic", "node.name")
master_nodes = cfg.opts("mechanic", "master.nodes")
seed_hosts = cfg.opts("mechanic", "seed.hosts")
if build_type == "tar":
binary_supplier = supplier.create(cfg, sources, distribution, car, plugins)
p = provisioner.local(
cfg=cfg,
car=car,
plugins=plugins,
ip=ip,
http_port=http_port,
all_node_ips=seed_hosts,
all_node_names=master_nodes,
target_root=root_path,
node_name=node_name,
)
node_config = p.prepare(binary=binary_supplier())
elif build_type == "docker":
if len(plugins) > 0:
raise exceptions.SystemSetupError(
"You cannot specify any plugins for Docker clusters. Please remove " '"--elasticsearch-plugins" and try again.'
)
p = provisioner.docker(cfg=cfg, car=car, ip=ip, http_port=http_port, target_root=root_path, node_name=node_name)
# there is no binary for Docker that can be downloaded / built upfront
node_config = p.prepare(binary=None)
else:
raise exceptions.SystemSetupError("Unknown build type [{}]".format(build_type))
provisioner.save_node_configuration(root_path, node_config)
console.println(json.dumps({"installation-id": cfg.opts("system", "install.id")}, indent=2), force=True)
def load_car(repo, name, car_params=None):
class Component:
def __init__(self, root_path, entry_point):
self.root_path = root_path
self.entry_point = entry_point
root_path = None
# preserve order as we append to existing config files later during provisioning.
all_config_paths = []
all_config_base_vars = {}
all_car_vars = {}
for n in name:
descriptor = CarLoader(repo).load_car(n, car_params)
for p in descriptor.config_paths:
if p not in all_config_paths:
all_config_paths.append(p)
for p in descriptor.root_paths:
# probe whether we have a root path
if BootstrapHookHandler(
Component(root_path=p,
entry_point=Car.entry_point)).can_load():
if not root_path:
root_path = p
# multiple cars are based on the same hook
elif root_path != p:
raise exceptions.SystemSetupError(
"Invalid car: {}. Multiple bootstrap hooks are forbidden."
.format(name))
all_config_base_vars.update(descriptor.config_base_variables)
all_car_vars.update(descriptor.variables)
if len(all_config_paths) == 0:
raise exceptions.SystemSetupError(
"At least one config base is required for car {}".format(name))
variables = {}
# car variables *always* take precedence over config base variables
variables.update(all_config_base_vars)
variables.update(all_car_vars)
return Car(name, root_path, all_config_paths, variables)
def _render_template(self, loader, template_name, variables):
try:
env = jinja2.Environment(loader=loader)
for k, v in variables.items():
env.globals[k] = v
template = env.get_template(template_name)
return template.render()
except jinja2.exceptions.TemplateSyntaxError as e:
raise exceptions.InvalidSyntax("%s in %s" % (str(e), template_name))
except BaseException as e:
raise exceptions.SystemSetupError("%s in %s" % (str(e), template_name))
def load(self):
root_module = self.loader.load()
try:
# every module needs to have a register() method
root_module.register(self)
except exceptions.RallyError:
# just pass our own exceptions transparently.
raise
except BaseException:
msg = "Could not load install hooks in [%s]" % self.loader.root_path
logger.exception(msg)
raise exceptions.SystemSetupError(msg)
def _url_for(self, user_defined_key, default_key, mandatory=True):
try:
if user_defined_key in self.cfg:
url_template = self.cfg[user_defined_key]
else:
url_template = self.cfg[default_key]
except KeyError:
if mandatory:
raise exceptions.SystemSetupError("Neither config key [{}] nor [{}] is defined.".format(user_defined_key, default_key))
else:
return None
return self.template_renderer.render(url_template)