def load_car(self, name, car_params=None):
car_config_file = self._car_file(name)
if not io.exists(car_config_file):
raise exceptions.SystemSetupError("Unknown car [{}]. List the available cars with {} list cars.".format(name, PROGRAM_NAME))
config = self._config_loader(car_config_file)
root_paths = []
config_paths = []
config_base_vars = {}
description = self._value(config, ["meta", "description"], default="")
car_type = self._value(config, ["meta", "type"], default="car")
config_bases = self._value(config, ["config", "base"], default="").split(",")
for base in config_bases:
if base:
root_path = os.path.join(self.cars_dir, base)
root_paths.append(root_path)
config_paths.append(os.path.join(root_path, "templates"))
config_file = os.path.join(root_path, "config.ini")
if io.exists(config_file):
base_config = self._config_loader(config_file)
self._copy_section(base_config, "variables", config_base_vars)
# it's possible that some cars don't have a config base, e.g. mixins which only override variables
if len(config_paths) == 0:
self.logger.info("Car [%s] does not define any config paths. Assuming that it is used as a mixin.", name)
variables = self._copy_section(config, "variables", {})
# add all car params here to override any defaults
if car_params:
variables.update(car_params)
env = self._copy_section(config, "env", {})
return CarDescriptor(name, description, car_type, root_paths, config_paths, config_base_vars, variables, env)
def load_plugin(self, name, config_names, plugin_params={}):
root_path = self._plugin_root_path(name)
if not config_names:
# maybe we only have a config folder but nothing else (e.g. if there is only an install hook)
if io.exists(root_path):
return PluginDescriptor(name=name, config=config_names, root_path=root_path)
else:
core_plugin = self._core_plugin(name)
if core_plugin:
return core_plugin
# If we just have a plugin name then we assume that this is a community plugin and the user has specified a download URL
else:
logger.info("The plugin [%s] is neither a configured nor an official plugin. Assuming that this is a community "
"plugin not requiring any configuration and you have set a proper download URL." % name)
return PluginDescriptor(name)
else:
variables = {}
config_paths = []
# used for deduplication
known_config_bases = set()
# used to determine whether this is a core plugin
core_plugin = self._core_plugin(name)
for config_name in config_names:
config_file = self._plugin_file(name, config_name)
# Do we have an explicit configuration for this plugin?
if not io.exists(config_file):
if core_plugin:
raise exceptions.SystemSetupError("Plugin [%s] does not provide configuration [%s]. List the available plugins "
"and configurations with %s list elasticsearch-plugins "
"--distribution-version=VERSION." % (name, config_name, PROGRAM_NAME))
else:
raise exceptions.SystemSetupError("Unknown plugin [%s]. List the available plugins with %s list "
"elasticsearch-plugins --distribution-version=VERSION." % (name, PROGRAM_NAME))
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
# Do not modify the case of option keys but read them as is
config.optionxform = lambda option: option
config.read(config_file)
if "config" in config and "base" in config["config"]:
config_bases = config["config"]["base"].split(",")
for base in config_bases:
if base and base not in known_config_bases:
config_paths.append(os.path.join(root_path, base))
known_config_bases.add(base)
if "variables" in config.sections():
for k, v in config["variables"].items():
variables[k] = v
# add all plugin params here to override any defaults
variables.update(plugin_params)
# maybe one of the configs is really just for providing variables. However, we still require one config base overall.
if len(config_paths) == 0:
raise exceptions.SystemSetupError("At least one config base is required for plugin [%s]" % name)
return PluginDescriptor(name=name, core_plugin=core_plugin is not None, config=config_names, root_path=root_path,
config_paths=config_paths, variables=variables)
def load_plugin(self, name, config_names):
root_path = self._plugin_root_path(name)
if not config_names:
# maybe we only have a config folder but nothing else (e.g. if there is only an install hook)
if io.exists(root_path):
return PluginDescriptor(name=name, config=config_names, root_path=root_path)
else:
core_plugin = self._core_plugin(name)
if core_plugin:
return core_plugin
# If we just have a plugin name then we assume that this is a community plugin and the user has specified a download URL
else:
logger.info("The plugin [%s] is neither a configured nor an official plugin. Assuming that this is a community "
"plugin not requiring any configuration and you have set a proper download URL." % name)
return PluginDescriptor(name)
else:
variables = {}
config_paths = []
# used for deduplication
known_config_bases = set()
# used to determine whether this is a core plugin
core_plugin = self._core_plugin(name)
for config_name in config_names:
config_file = self._plugin_file(name, config_name)
# Do we have an explicit configuration for this plugin?
if not io.exists(config_file):
if core_plugin:
raise exceptions.SystemSetupError("Plugin [%s] does not provide configuration [%s]. List the available plugins "
"and configurations with %s list elasticsearch-plugins "
"--distribution-version=VERSION." % (name, config_name, PROGRAM_NAME))
else:
raise exceptions.SystemSetupError("Unknown plugin [%s]. List the available plugins with %s list "
"elasticsearch-plugins --distribution-version=VERSION." % (name, PROGRAM_NAME))
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
# Do not modify the case of option keys but read them as is
config.optionxform = lambda option: option
config.read(config_file)
if "config" in config and "base" in config["config"]:
config_bases = config["config"]["base"].split(",")
for base in config_bases:
if base and base not in known_config_bases:
config_paths.append(os.path.join(root_path, base))
known_config_bases.add(base)
if "variables" in config.sections():
for k, v in config["variables"].items():
variables[k] = v
# maybe one of the configs is really just for providing variables. However, we still require one config base overall.
if len(config_paths) == 0:
raise exceptions.SystemSetupError("At least one config base is required for plugin [%s]" % name)
return PluginDescriptor(name=name, core_plugin=core_plugin is not None, config=config_names, root_path=root_path,
config_paths=config_paths, variables=variables)
def load_car(self, name):
car_config_file = self._car_file(name)
if not io.exists(car_config_file):
raise exceptions.SystemSetupError("Unknown car [%s]. List the available cars with %s list cars." % (name, PROGRAM_NAME))
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
# Do not modify the case of option keys but read them as is
config.optionxform = lambda option: option
config.read(car_config_file)
config_paths = []
description = ""
car_type = "car"
if "meta" in config:
description = config["meta"].get("description", description)
car_type = config["meta"].get("type", car_type)
if "config" in config and "base" in config["config"]:
config_bases = config["config"]["base"].split(",")
for base in config_bases:
if base:
config_paths.append(os.path.join(self.cars_dir, base))
# it's possible that some cars don't have a config base, e.g. mixins which only override variables
if len(config_paths) == 0:
logger.info("Car [%s] does not define any config paths. Assuming that it is used as a mixin." % name)
variables = {}
if "variables" in config.sections():
for k, v in config["variables"].items():
variables[k] = v
env = {}
if "env" in config.sections():
for k, v in config["env"].items():
env[k] = v
return CarDescriptor(name, description, car_type, config_paths, variables, env)
def __init__(self,
remote_url,
root_dir,
repo_name,
resource_name,
offline,
fetch=True):
# If no URL is found, we consider this a local only repo (but still require that it is a git repo)
self.url = remote_url
self.remote = self.url is not None and self.url.strip() != ""
self.repo_dir = os.path.join(root_dir, repo_name)
self.resource_name = resource_name
self.offline = offline
self.logger = logging.getLogger(__name__)
if self.remote and not self.offline and fetch:
# a normal git repo with a remote
if not git.is_working_copy(self.repo_dir):
git.clone(src=self.repo_dir, remote=self.url)
else:
try:
git.fetch(src=self.repo_dir)
except exceptions.SupplyError:
console.warn(
"Could not update %s. Continuing with your locally available state."
% self.resource_name)
else:
if not git.is_working_copy(self.repo_dir):
if io.exists(self.repo_dir):
raise exceptions.SystemSetupError(
"[{src}] must be a git repository.\n\nPlease run:\ngit -C {src} init"
.format(src=self.repo_dir))
else:
raise exceptions.SystemSetupError(
"Expected a git repository at [{src}] but the directory does not exist."
.format(src=self.repo_dir))
def load_car(self, name):
car_config_file = self._car_file(name)
if not io.exists(car_config_file):
raise exceptions.SystemSetupError("Unknown car [%s]. List the available cars with %s list cars." % (name, PROGRAM_NAME))
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
# Do not modify the case of option keys but read them as is
config.optionxform = lambda option: option
config.read(car_config_file)
config_paths = []
description = ""
car_type = "car"
if "meta" in config:
description = config["meta"].get("description", description)
car_type = config["meta"].get("type", car_type)
if "config" in config and "base" in config["config"]:
config_bases = config["config"]["base"].split(",")
for base in config_bases:
if base:
config_paths.append(os.path.join(self.cars_dir, base))
# it's possible that some cars don't have a config base, e.g. mixins which only override variables
if len(config_paths) == 0:
logger.info("Car [%s] does not define any config paths. Assuming that it is used as a mixin." % name)
variables = {}
if "variables" in config.sections():
for k, v in config["variables"].items():
variables[k] = v
env = {}
if "env" in config.sections():
for k, v in config["env"].items():
env[k] = v
return CarDescriptor(name, description, car_type, config_paths, variables, env)
def load_car(self, name):
car_config_file = self._car_file(name)
if not io.exists(car_config_file):
raise exceptions.SystemSetupError(
"Unknown car [%s]. List the available cars with %s list cars."
% (name, PROGRAM_NAME))
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
# Do not modify the case of option keys but read them as is
config.optionxform = lambda option: option
config.read(car_config_file)
config_paths = []
if "config" in config and "base" in config["config"]:
config_bases = config["config"]["base"].split(",")
for base in config_bases:
if base:
config_paths.append(os.path.join(self.cars_dir, base))
if len(config_paths) == 0:
raise exceptions.SystemSetupError(
"At least one config base is required for car [%s]" % name)
variables = {}
if "variables" in config.sections():
for k, v in config["variables"].items():
variables[k] = v
env = {}
if "env" in config.sections():
for k, v in config["env"].items():
env[k] = v
return Car(name, config_paths, variables, env)
def ask_property(self,
prompt,
mandatory=True,
check_path_exists=False,
check_pattern=None,
choices=None,
sensitive=False,
default_value=None):
if default_value is not None:
final_prompt = "%s (default: %s): " % (prompt, default_value)
elif not mandatory:
final_prompt = "%s (Press Enter to skip): " % prompt
else:
final_prompt = "%s: " % prompt
while True:
if self.assume_defaults and (default_value is not None
or not mandatory):
self.o(final_prompt)
value = None
elif sensitive:
value = self.sec_i(final_prompt)
else:
value = self.i(final_prompt)
if not value or value.strip() == "":
if mandatory and default_value is None:
self.o(" Value is required. Please retry.")
continue
else:
# suppress output when the default is empty
if default_value:
self.o(" Using default value '%s'" % default_value)
# this way, we can still check the path...
value = default_value
if mandatory or value is not None:
if check_path_exists and not io.exists(value):
self.o("'%s' does not exist. Please check and retry." %
value)
continue
if check_pattern is not None and not check_pattern.match(
str(value)):
self.o(
"Input does not match pattern [%s]. Please check and retry."
% check_pattern.pattern)
continue
if choices is not None and str(value) not in choices:
self.o(
"Input is not one of the valid choices %s. Please check and retry."
% choices)
continue
self.o("")
# user entered a valid value
return value
def install_default_log_config():
"""
Ensures a log configuration file is present on this machine. The default
log configuration is based on the template in resources/logging.json.
It also ensures that the default log path has been created so log files
can be successfully opened in that directory.
"""
log_config = log_config_path()
if not io.exists(log_config):
io.ensure_dir(io.dirname(log_config))
source_path = io.normalize_path(os.path.join(os.path.dirname(__file__), "resources", "logging.json"))
with open(log_config, "w", encoding="UTF-8") as target:
with open(source_path, "r", encoding="UTF-8") as src:
contents = src.read().replace("${LOG_PATH}", default_log_path())
target.write(contents)
io.ensure_dir(default_log_path())
def remove_obsolete_default_log_config():
"""
Log rotation is problematic because Rally uses multiple processes and there is a lurking race condition when
rolling log files. Hence, we do not rotate logs from within Rally and leverage established tools like logrotate for that.
Checks whether the user has a problematic out-of-the-box logging configuration delivered with Rally 1.0.0 which
used log rotation and removes it so it can be replaced by a new one in a later step.
"""
log_config = log_config_path()
if io.exists(log_config):
source_path = io.normalize_path(os.path.join(os.path.dirname(__file__), "resources", "logging_1_0_0.json"))
with open(source_path, "r", encoding="UTF-8") as src:
contents = src.read().replace("${LOG_PATH}", default_log_path())
source_hash = hashlib.sha512(contents.encode()).hexdigest()
with open(log_config, "r", encoding="UTF-8") as target:
target_hash = hashlib.sha512(target.read().encode()).hexdigest()
if source_hash == target_hash:
os.rename(log_config, "{}.bak".format(log_config))
def install_default_log_config():
"""
Ensures a log configuration file is present on this machine. The default
log configuration is based on the template in resources/logging.json.
It also ensures that the default log path has been created so log files
can be successfully opened in that directory.
"""
log_config = log_config_path()
if not io.exists(log_config):
io.ensure_dir(io.dirname(log_config))
source_path = io.normalize_path(
os.path.join(os.path.dirname(__file__), "resources",
"logging.json"))
with open(log_config, "w", encoding="UTF-8") as target:
with open(source_path, "r", encoding="UTF-8") as src:
# Ensure we have a trailing path separator as after LOG_PATH there will only be the file name
log_path = os.path.join(paths.logs(), "")
# the logging path might contain backslashes that we need to escape
log_path = io.escape_path(log_path)
contents = src.read().replace("${LOG_PATH}", log_path)
target.write(contents)
io.ensure_dir(paths.logs())
def migrate(config_file, current_version, target_version, out=print, i=input):
prompter = Prompter(i=i, o=out, assume_defaults=False)
logger.info("Upgrading configuration from version [%s] to [%s]." %
(current_version, target_version))
# Something is really fishy. We don't want to downgrade the configuration.
if current_version >= target_version:
raise ConfigError(
"The existing config file is available in a later version already. Expected version <= [%s] but found [%s]"
% (target_version, current_version))
# but first a backup...
config_file.backup()
config = config_file.load(interpolation=None)
if current_version == 0 and target_version > current_version:
logger.info("Migrating config from version [0] to [1]")
current_version = 1
config["meta"] = {}
config["meta"]["config.version"] = str(current_version)
# in version 1 we changed some directories from being absolute to being relative
config["system"]["log.root.dir"] = "logs"
config["provisioning"]["local.install.dir"] = "install"
config["reporting"]["report.base.dir"] = "reports"
if current_version == 1 and target_version > current_version:
logger.info("Migrating config from version [1] to [2]")
current_version = 2
config["meta"]["config.version"] = str(current_version)
# no need to ask the user now if we are about to upgrade to version 4
config["reporting"]["datastore.type"] = "in-memory"
config["reporting"]["datastore.host"] = ""
config["reporting"]["datastore.port"] = ""
config["reporting"]["datastore.secure"] = ""
config["reporting"]["datastore.user"] = ""
config["reporting"]["datastore.password"] = ""
config["system"]["env.name"] = "local"
if current_version == 2 and target_version > current_version:
logger.info("Migrating config from version [2] to [3]")
current_version = 3
config["meta"]["config.version"] = str(current_version)
# Remove obsolete settings
config["reporting"].pop("report.base.dir")
config["reporting"].pop("output.html.report.filename")
if current_version == 3 and target_version > current_version:
root_dir = config["system"]["root.dir"]
out("""
*****************************************************************************************
You have an old configuration of Rally. Rally has now a much simpler setup
routine which will autodetect lots of settings for you and it also does not
require you to setup a metrics store anymore.
Rally will now migrate your configuration but if you don't need advanced features
like a metrics store, then you should delete the configuration directory:
rm -rf {0}
and then rerun Rally's configuration routine:
{1} configure
Please also note you have {2:.1f} GB of data in your current benchmark directory at
{3}
You might want to clean up this directory also.
For more details please see {4}
*****************************************************************************************
Pausing for 10 seconds to let you consider this message.
""".format(
config_file.config_dir, PROGRAM_NAME,
convert.bytes_to_gb(io.get_size(root_dir)), root_dir,
console.format.link(
"https://github.com/elastic/rally/blob/master/CHANGELOG.md#030"
)))
time.sleep(10)
logger.info("Migrating config from version [3] to [4]")
current_version = 4
config["meta"]["config.version"] = str(current_version)
if len(config["reporting"]["datastore.host"]) > 0:
config["reporting"]["datastore.type"] = "elasticsearch"
else:
config["reporting"]["datastore.type"] = "in-memory"
# Remove obsolete settings
config["build"].pop("maven.bin")
config["benchmarks"].pop("metrics.stats.disk.device")
if current_version == 4 and target_version > current_version:
config["tracks"] = {}
config["tracks"][
"default.url"] = "https://github.com/elastic/rally-tracks"
current_version = 5
config["meta"]["config.version"] = str(current_version)
if current_version == 5 and target_version > current_version:
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(False)
current_version = 6
config["meta"]["config.version"] = str(current_version)
if current_version == 6 and target_version > current_version:
# Remove obsolete settings
config.pop("provisioning")
config["system"].pop("log.root.dir")
current_version = 7
config["meta"]["config.version"] = str(current_version)
if current_version == 7 and target_version > current_version:
# move [system][root.dir] to [node][root.dir]
if "node" not in config:
config["node"] = {}
config["node"]["root.dir"] = config["system"].pop("root.dir")
# also move all references!
for section in config:
for k, v in config[section].items():
config[section][k] = v.replace("${system:root.dir}",
"${node:root.dir}")
current_version = 8
config["meta"]["config.version"] = str(current_version)
if current_version == 8 and target_version > current_version:
config["teams"] = {}
config["teams"][
"default.url"] = "https://github.com/elastic/rally-teams"
current_version = 9
config["meta"]["config.version"] = str(current_version)
if current_version == 9 and target_version > current_version:
config["distributions"] = {}
config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
"{{VERSION}}.tar.gz"
config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
"distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"][
"release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"]["release.cache"] = "true"
current_version = 10
config["meta"]["config.version"] = str(current_version)
if current_version == 10 and target_version > current_version:
config["runtime"]["java.home"] = config["runtime"].pop("java8.home")
current_version = 11
config["meta"]["config.version"] = str(current_version)
if current_version == 11 and target_version > current_version:
# As this is a rather complex migration, we log more than usual to understand potential migration problems better.
if "source" in config:
if "local.src.dir" in config["source"]:
previous_root = config["source"].pop("local.src.dir")
logger.info("Set [source][local.src.dir] to [%s]." %
previous_root)
# if this directory was Rally's default location, then move it on the file system because to allow for checkouts of plugins
# in the sibling directory.
if previous_root == os.path.join(config["node"]["root.dir"],
"src"):
new_root_dir_all_sources = previous_root
new_es_sub_dir = "elasticsearch"
new_root = os.path.join(new_root_dir_all_sources,
new_es_sub_dir)
# only attempt to move if the directory exists. It may be possible that users never ran a source benchmark although they
# have configured it. In that case the source directory will not yet exist.
if io.exists(previous_root):
logger.info(
"Previous source directory was at Rally's default location [%s]. Moving to [%s]."
% (previous_root, new_root))
try:
# we need to do this in two steps as we need to move the sources to a subdirectory
tmp_path = io.normalize_path(
os.path.join(new_root_dir_all_sources,
os.pardir, "tmp_src_mig"))
os.rename(previous_root, tmp_path)
io.ensure_dir(new_root)
os.rename(tmp_path, new_root)
except OSError:
logger.exception(
"Could not move source directory from [%s] to [%s]."
% (previous_root, new_root))
# A warning is sufficient as Rally should just do a fresh checkout if moving did not work.
console.warn(
"Elasticsearch source directory could not be moved from [%s] to [%s]. Please check the logs."
% (previous_root, new_root))
else:
logger.info(
"Source directory is configured at Rally's default location [%s] but does not exist yet."
% previous_root)
else:
logger.info(
"Previous source directory was the custom directory [%s]."
% previous_root)
new_root_dir_all_sources = io.normalize_path(
os.path.join(previous_root, os.path.pardir))
# name of the elasticsearch project directory.
new_es_sub_dir = io.basename(previous_root)
logger.info("Setting [node][src.root.dir] to [%s]." %
new_root_dir_all_sources)
config["node"]["src.root.dir"] = new_root_dir_all_sources
logger.info(
"Setting [source][elasticsearch.src.subdir] to [%s]" %
new_es_sub_dir)
config["source"]["elasticsearch.src.subdir"] = new_es_sub_dir
else:
logger.info(
"Key [local.src.dir] not found. Advancing without changes."
)
else:
logger.info(
"No section named [source] found in config. Advancing without changes."
)
current_version = 12
config["meta"]["config.version"] = str(current_version)
if current_version == 12 and target_version > current_version:
# the current configuration allows to benchmark from sources
if "build" in config and "gradle.bin" in config["build"]:
java_9_home = io.guess_java_home(major_version=9)
from esrally.utils import jvm
if java_9_home and not jvm.is_early_access_release(java_9_home):
logger.debug("Autodetected a JDK 9 installation at [%s]" %
java_9_home)
if "runtime" not in config:
config["runtime"] = {}
config["runtime"]["java9.home"] = java_9_home
else:
logger.debug(
"Could not autodetect a JDK 9 installation. Checking [java.home] already points to a JDK 9."
)
detected = False
if "runtime" in config:
java_home = config["runtime"]["java.home"]
if jvm.major_version(
java_home
) == 9 and not jvm.is_early_access_release(java_home):
config["runtime"]["java9.home"] = java_home
detected = True
if not detected:
logger.debug(
"Could not autodetect a JDK 9 installation. Asking user."
)
raw_java_9_home = prompter.ask_property(
"Enter the JDK 9 root directory",
check_path_exists=True,
mandatory=False)
if raw_java_9_home and jvm.major_version(
raw_java_9_home
) == 9 and not jvm.is_early_access_release(
raw_java_9_home):
java_9_home = io.normalize_path(
raw_java_9_home) if raw_java_9_home else None
config["runtime"]["java9.home"] = java_9_home
else:
out("********************************************************************************"
)
out("You don't have a valid JDK 9 installation and cannot benchmark source builds."
)
out("")
out("You can still benchmark binary distributions with e.g.:"
)
out("")
out(" %s --distribution-version=6.0.0" % PROGRAM_NAME)
out("********************************************************************************"
)
out("")
current_version = 13
config["meta"]["config.version"] = str(current_version)
# all migrations done
config_file.store(config)
logger.info("Successfully self-upgraded configuration to version [%s]" %
target_version)