def load(self, track_plugin_path):
plugin_name = io.basename(track_plugin_path)
logger.info("Loading track plugin [%s] from [%s]" % (plugin_name, track_plugin_path))
# search all paths within this directory for modules but exclude all directories starting with "_"
module_dirs = []
for dirpath, dirs, _ in os.walk(track_plugin_path):
module_dirs.append(dirpath)
ignore = []
for d in dirs:
if d.startswith("_"):
logger.debug("Removing [%s] from load path." % d)
ignore.append(d)
for d in ignore:
dirs.remove(d)
# load path is only the root of the package hierarchy
plugin_root_path = os.path.abspath(os.path.join(track_plugin_path, os.pardir))
logger.debug("Adding [%s] to Python load path." % plugin_root_path)
# needs to be at the beginning of the system path, otherwise import machinery tries to load application-internal modules
sys.path.insert(0, plugin_root_path)
try:
root_module = self._load_plugin(plugin_name, module_dirs, track_plugin_path)
# every module needs to have a register() method
root_module.register(self)
except BaseException:
msg = "Could not load track plugin [%s]" % plugin_name
logger.exception(msg)
raise exceptions.SystemSetupError(msg)
def _render_template_from_file(self, variables):
compose_file = os.path.join(self.rally_root, "resources",
"docker-compose.yml.j2")
return self._render_template(loader=jinja2.FileSystemLoader(
io.dirname(compose_file)),
template_name=io.basename(compose_file),
variables=variables)
def _render_template_from_file(self, variables):
compose_file = "%s/resources/docker-compose.yml" % (self.cfg.opts(
"system", "rally.root"))
return self._render_template(loader=jinja2.FileSystemLoader(
io.dirname(compose_file)),
template_name=io.basename(compose_file),
variables=variables)
def _render_template(env, variables, file_name):
try:
template = env.get_template(io.basename(file_name))
# force a new line at the end. Jinja seems to remove it.
return template.render(variables) + "\n"
except jinja2.exceptions.TemplateSyntaxError as e:
raise exceptions.InvalidSyntax("%s in %s" % (str(e), file_name))
except BaseException as e:
raise exceptions.SystemSetupError("%s in %s" % (str(e), file_name))
def __init__(self, track_path):
if not os.path.exists(track_path):
raise exceptions.SystemSetupError("Track path %s does not exist" % track_path)
if os.path.isdir(track_path):
self.track_name = io.basename(track_path)
self._track_dir = track_path
self._track_file = os.path.join(track_path, "track.json")
if not os.path.exists(self._track_file):
raise exceptions.SystemSetupError("Could not find track.json in %s" % track_path)
elif os.path.isfile(track_path):
if io.has_extension(track_path, ".json"):
self._track_dir = io.dirname(track_path)
self._track_file = track_path
self.track_name = io.splitext(io.basename(track_path))[0]
else:
raise exceptions.SystemSetupError("%s has to be a JSON file" % track_path)
else:
raise exceptions.SystemSetupError("%s is neither a file nor a directory" % track_path)
def render_template_from_file(template_file_name):
def relative_glob(start, f):
result = glob.glob(os.path.join(start, f))
if result:
return [os.path.relpath(p, start) for p in result]
else:
return []
base_path = io.dirname(template_file_name)
return render_template(loader=jinja2.FileSystemLoader(base_path),
template_name=io.basename(template_file_name),
glob_helper=lambda f: relative_glob(base_path, f))
def render_template_from_file(template_file_name):
def relative_glob(start, f):
result = glob.glob(os.path.join(start, f))
if result:
return [os.path.relpath(p, start) for p in result]
else:
return []
base_path = io.dirname(template_file_name)
return render_template(loader=jinja2.FileSystemLoader(base_path),
template_name=io.basename(template_file_name),
glob_helper=lambda f: relative_glob(base_path, f))
def _configured_plugins(self):
configured_plugins = []
# each directory is a plugin, each .ini is a config (just go one level deep)
for entry in os.listdir(self.plugins_root_path):
plugin_path = os.path.join(self.plugins_root_path, entry)
if os.path.isdir(plugin_path):
for child_entry in os.listdir(plugin_path):
if os.path.isfile(os.path.join(plugin_path, child_entry)) and io.has_extension(child_entry, ".ini"):
f, _ = io.splitext(child_entry)
plugin_name = self._file_to_plugin_name(entry)
config = io.basename(f)
configured_plugins.append(PluginDescriptor(name=plugin_name, config=config))
return configured_plugins
def _configured_plugins(self, variables=None):
configured_plugins = []
# each directory is a plugin, each .ini is a config (just go one level deep)
for entry in os.listdir(self.plugins_root_path):
plugin_path = os.path.join(self.plugins_root_path, entry)
if os.path.isdir(plugin_path):
for child_entry in os.listdir(plugin_path):
if os.path.isfile(os.path.join(plugin_path, child_entry)) and io.has_extension(child_entry, ".ini"):
f, _ = io.splitext(child_entry)
plugin_name = self._file_to_plugin_name(entry)
config = io.basename(f)
configured_plugins.append(PluginDescriptor(name=plugin_name, config=config, variables=variables))
return configured_plugins
def load(self, track_plugin_path):
plugin_name = io.basename(track_plugin_path)
logger.info("Loading track plugin [%s] from [%s]" % (plugin_name, track_plugin_path))
# search all paths within this directory for modules
module_dirs = [dirpath for dirpath, _, _ in os.walk(track_plugin_path)]
# load path is only the root of the package hierarchy
plugin_root_path = os.path.abspath(os.path.join(track_plugin_path, os.pardir))
logger.debug("Adding [%s] to Python load path." % plugin_root_path)
# needs to be at the beginning of the system path, otherwise import machinery tries to load application-internal modules
sys.path.insert(0, plugin_root_path)
try:
root_module = self._load_plugin(plugin_name, module_dirs, track_plugin_path)
# every module needs to have a register() method
root_module.register(self)
except BaseException:
msg = "Could not load track plugin [%s]" % plugin_name
logger.exception(msg)
raise exceptions.SystemSetupError(msg)
def load(self):
"""
Loads a component with the given component entry point.
Precondition: ``ComponentLoader#can_load() == True``.
:return: The root module.
"""
component_name = io.basename(self.root_path)
self.logger.info("Loading component [%s] from [%s]", component_name,
self.root_path)
module_dirs = []
# search all paths within this directory for modules but exclude all directories starting with "_"
if self.recurse:
for dirpath, dirs, _ in os.walk(self.root_path):
module_dirs.append(dirpath)
ignore = []
for d in dirs:
if d.startswith("_"):
self.logger.debug("Removing [%s] from load path.", d)
ignore.append(d)
for d in ignore:
dirs.remove(d)
else:
module_dirs.append(self.root_path)
# load path is only the root of the package hierarchy
component_root_path = os.path.abspath(
os.path.join(self.root_path, os.pardir))
self.logger.debug("Adding [%s] to Python load path.",
component_root_path)
# needs to be at the beginning of the system path, otherwise import machinery tries to load application-internal modules
sys.path.insert(0, component_root_path)
try:
root_module = self._load_component(component_name, module_dirs)
return root_module
except BaseException:
msg = "Could not load component [{}]".format(component_name)
self.logger.exception(msg)
raise exceptions.SystemSetupError(msg)
def load(self):
"""
Loads a component with the given component entry point.
Precondition: ``ComponentLoader#can_load() == True``.
:return: The root module.
"""
component_name = io.basename(self.root_path)
logger.info("Loading component [%s] from [%s]" % (component_name, self.root_path))
module_dirs = []
# search all paths within this directory for modules but exclude all directories starting with "_"
if self.recurse:
for dirpath, dirs, _ in os.walk(self.root_path):
module_dirs.append(dirpath)
ignore = []
for d in dirs:
if d.startswith("_"):
logger.debug("Removing [%s] from load path." % d)
ignore.append(d)
for d in ignore:
dirs.remove(d)
else:
module_dirs.append(self.root_path)
# load path is only the root of the package hierarchy
component_root_path = os.path.abspath(os.path.join(self.root_path, os.pardir))
logger.debug("Adding [%s] to Python load path." % component_root_path)
# needs to be at the beginning of the system path, otherwise import machinery tries to load application-internal modules
sys.path.insert(0, component_root_path)
try:
root_module = self._load_component(component_name, module_dirs)
return root_module
except BaseException:
msg = "Could not load component [%s]" % component_name
logger.exception(msg)
raise exceptions.SystemSetupError(msg)
def _render_template(env, variables, file_name):
template = env.get_template(io.basename(file_name))
# force a new line at the end. Jinja seems to remove it.
return template.render(variables) + "\n"
def create_config(self, config_file, advanced_config=False, assume_defaults=False):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param config_file:
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
:param assume_defaults: If True, assume the user accepted all values for which defaults are provided. Mainly intended for automatic
configuration in CI run. Default: False.
"""
self.prompter = Prompter(self.i, self.sec_i, self.o, assume_defaults)
if advanced_config:
self.o("Running advanced configuration. You can get additional help at:")
self.o("")
self.o(" %s" % console.format.link(doc_link("configuration.html")))
self.o("")
else:
self.o("Running simple configuration. Run the advanced configuration with:")
self.o("")
self.o(" %s configure --advanced-config" % PROGRAM_NAME)
self.o("")
if config_file.present:
self.o("\nWARNING: Will overwrite existing config file at [%s]\n" % config_file.location)
self.logger.debug("Detected an existing configuration file at [%s]", config_file.location)
else:
self.logger.debug("Did not detect a configuration file at [%s]. Running initial configuration routine.", config_file.location)
root_dir = io.normalize_path(os.path.abspath(os.path.join(config_file.config_dir, "benchmarks")))
if advanced_config:
root_dir = io.normalize_path(self._ask_property("Enter the benchmark root directory", default_value=root_dir))
else:
self.o("* Setting up benchmark root directory in %s" % root_dir)
# We try to autodetect an existing ES source directory
guess = self._guess_es_src_dir()
if guess:
source_dir = guess
self.logger.debug("Autodetected Elasticsearch project directory at [%s].", source_dir)
else:
default_src_dir = os.path.join(root_dir, "src", "elasticsearch")
self.logger.debug("Could not autodetect Elasticsearch project directory. Providing [%s] as default.", default_src_dir)
source_dir = default_src_dir
if advanced_config:
source_dir = io.normalize_path(self._ask_property("Enter your Elasticsearch project directory:",
default_value=source_dir))
if not advanced_config:
self.o("* Setting up benchmark source directory in %s" % source_dir)
self.o("")
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = "https://github.com/elastic/elasticsearch.git"
if advanced_config:
data_store_choice = self._ask_property("Where should metrics be kept?"
"\n\n"
"(1) In memory (simpler but less options for analysis)\n"
"(2) Elasticsearch (requires a separate ES instance, keeps all raw samples for analysis)"
"\n\n", default_value="1", choices=["1", "2"])
if data_store_choice == "1":
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "False", "", ""
else:
data_store_type = "elasticsearch"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store()
env_name = self._ask_env_name()
preserve_install = convert.to_bool(self._ask_property("Do you want Rally to keep the Elasticsearch benchmark candidate "
"installation including the index (will use several GB per race)?",
default_value=False))
else:
# Does not matter for an in-memory store
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "False", "", ""
preserve_install = False
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(Config.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["env.name"] = env_name
config["node"] = {}
config["node"]["root.dir"] = root_dir
final_source_dir = io.normalize_path(os.path.abspath(os.path.join(source_dir, os.pardir)))
config["node"]["src.root.dir"] = final_source_dir
config["source"] = {}
config["source"]["remote.repo.url"] = repo_url
# the Elasticsearch directory is just the last path component (relative to the source root directory)
config["source"]["elasticsearch.src.subdir"] = io.basename(source_dir)
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = os.path.join(root_dir, "data")
config["reporting"] = {}
config["reporting"]["datastore.type"] = data_store_type
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
config["tracks"] = {}
config["tracks"]["default.url"] = "https://github.com/elastic/rally-tracks"
config["teams"] = {}
config["teams"]["default.url"] = "https://github.com/elastic/rally-teams"
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(preserve_install)
config["distributions"] = {}
config["distributions"]["release.cache"] = "true"
config_file.store(config)
self.o("Configuration successfully written to %s. Happy benchmarking!" % config_file.location)
self.o("")
self.o("More info about Rally:")
self.o("")
self.o("* Type %s --help" % PROGRAM_NAME)
self.o("* Read the documentation at %s" % console.format.link(doc_link()))
self.o("* Ask a question on the forum at %s" % console.format.link("https://discuss.elastic.co/c/elasticsearch/rally"))
def _render_template_from_file(self, variables):
compose_file = "%s/resources/docker-compose.yml.j2" % self.rally_root
return self._render_template(loader=jinja2.FileSystemLoader(
io.dirname(compose_file)),
template_name=io.basename(compose_file),
variables=variables)
def _render_template_from_file(self, variables):
compose_file = "%s/resources/docker-compose.yml" % (self.cfg.opts("system", "rally.root"))
return self._render_template(loader=jinja2.FileSystemLoader(io.dirname(compose_file)),
template_name=io.basename(compose_file),
variables=variables)
def create_config(self, config_file, advanced_config=False, assume_defaults=False):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param config_file:
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
:param assume_defaults: If True, assume the user accepted all values for which defaults are provided. Mainly intended for automatic
configuration in CI run. Default: False.
"""
self.assume_defaults = assume_defaults
if advanced_config:
self.o("Running advanced configuration. You can get additional help at:")
self.o("")
self.o(" %s" % console.format.link("%sconfiguration.html" % DOC_LINK))
self.o("")
logger.info("Running advanced configuration routine.")
self.o("")
else:
self.o("Running simple configuration. Run the advanced configuration with:")
self.o("")
self.o(" %s configure --advanced-config" % PROGRAM_NAME)
self.o("")
logger.info("Running simple configuration routine.")
if config_file.present:
self.o("\nWARNING: Will overwrite existing config file at [%s]\n" % config_file.location)
logger.debug("Detected an existing configuration file at [%s]" % config_file.location)
else:
logger.debug("Did not detect a configuration file at [%s]. Running initial configuration routine." % config_file.location)
# Autodetect settings
self.o("* Autodetecting available third-party software")
git_path = io.guess_install_location("git")
gradle_bin = io.guess_install_location("gradle")
java_8_home = io.guess_java_home(major_version=8)
java_9_home = io.guess_java_home(major_version=9)
from esrally.utils import jvm
if java_8_home:
auto_detected_java_home = java_8_home
# Don't auto-detect an EA release and bring trouble to the user later on. They can still configure it manually if they want to.
elif java_9_home and not jvm.is_early_access_release(java_9_home):
auto_detected_java_home = java_9_home
else:
auto_detected_java_home = None
self.print_detection_result("git ", git_path)
self.print_detection_result("gradle ", gradle_bin)
self.print_detection_result("JDK ", auto_detected_java_home,
warn_if_missing=True,
additional_message="You cannot benchmark Elasticsearch without a JDK installation")
self.o("")
# users that don't have Gradle available cannot benchmark from sources
benchmark_from_sources = gradle_bin
if not benchmark_from_sources:
self.o("**********************************************************************************")
self.o("You don't have the necessary software to benchmark source builds of Elasticsearch.")
self.o("")
self.o("You can still benchmark binary distributions with e.g.:")
self.o("")
self.o(" %s --distribution-version=5.0.0" % PROGRAM_NAME)
self.o("**********************************************************************************")
self.o("")
root_dir = "%s/benchmarks" % config_file.config_dir
if advanced_config:
root_dir = io.normalize_path(self._ask_property("Enter the benchmark data directory:", default_value=root_dir))
else:
self.o("* Setting up benchmark data directory in [%s] (needs several GB)." % root_dir)
if benchmark_from_sources:
# We try to autodetect an existing ES source directory
guess = self._guess_es_src_dir()
if guess:
source_dir = guess
self.o("Autodetected Elasticsearch project directory at [%s]." % source_dir)
logger.debug("Autodetected Elasticsearch project directory at [%s]." % source_dir)
else:
default_src_dir = "%s/src/elasticsearch" % root_dir
logger.debug("Could not autodetect Elasticsearch project directory. Providing [%s] as default." % default_src_dir)
source_dir = io.normalize_path(self._ask_property("Enter your Elasticsearch project directory:",
default_value=default_src_dir))
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = "https://github.com/elastic/elasticsearch.git"
if auto_detected_java_home:
java_home = auto_detected_java_home
else:
self.o("")
java_home = io.normalize_path(self._ask_property("Enter the JDK root directory:", check_path_exists=True))
if advanced_config:
env_name = self._ask_env_name()
data_store_type = "elasticsearch"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store()
preserve_install = convert.to_bool(self._ask_property("Do you want Rally to keep the Elasticsearch benchmark candidate "
"installation including the index (will use lots of disk space)?",
default_value=False))
else:
# Does not matter too much for an in-memory store
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "", "", ""
preserve_install = False
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(Config.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["env.name"] = env_name
config["node"] = {}
config["node"]["root.dir"] = root_dir
if benchmark_from_sources:
# user has provided the Elasticsearch directory but the root for Elasticsearch and related plugins will be one level above
final_source_dir = io.normalize_path(os.path.abspath(os.path.join(source_dir, os.pardir)))
config["node"]["src.root.dir"] = final_source_dir
config["source"] = {}
config["source"]["remote.repo.url"] = repo_url
# the Elasticsearch directory is just the last path component (relative to the source root directory)
config["source"]["elasticsearch.src.subdir"] = io.basename(source_dir)
config["build"] = {}
config["build"]["gradle.bin"] = gradle_bin
config["runtime"] = {}
config["runtime"]["java.home"] = java_home
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = "${node:root.dir}/data"
config["reporting"] = {}
config["reporting"]["datastore.type"] = data_store_type
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
config["tracks"] = {}
config["tracks"]["default.url"] = "https://github.com/elastic/rally-tracks"
config["teams"] = {}
config["teams"]["default.url"] = "https://github.com/elastic/rally-teams"
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(preserve_install)
config["distributions"] = {}
config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
"{{VERSION}}.tar.gz"
config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
"distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"]["release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"]["release.cache"] = "true"
config_file.store(config)
self.o("Configuration successfully written to [%s]. Happy benchmarking!" % config_file.location)
self.o("")
if benchmark_from_sources:
self.o("To benchmark Elasticsearch with the default benchmark run:")
self.o("")
self.o(" %s" % PROGRAM_NAME)
else:
self.o("To benchmark Elasticsearch 5.0.0 with the default benchmark run:")
self.o("")
self.o(" %s --distribution-version=5.0.0" % PROGRAM_NAME)
self.o("")
self.o("For help, type %s --help or see the user documentation at %s"
% (PROGRAM_NAME, console.format.link(DOC_LINK)))
def __car_name(path):
p, _ = io.splitext(path)
return io.basename(p)
def render_template_from_file(template_file_name):
return render_template(loader=jinja2.FileSystemLoader(io.dirname(template_file_name)), template_name=io.basename(template_file_name))
def create_config(self,
config_file,
advanced_config=False,
assume_defaults=False,
use_gradle_wrapper=False,
java_home=None,
runtime_java_home=None):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param config_file:
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
:param assume_defaults: If True, assume the user accepted all values for which defaults are provided. Mainly intended for automatic
configuration in CI run. Default: False.
:param use_gradle_wrapper: If True, use the Gradle wrapper, otherwise use the system's Gradle version. Default: False.
"""
self.prompter = Prompter(self.i, self.sec_i, self.o, assume_defaults)
if advanced_config:
self.o(
"Running advanced configuration. You can get additional help at:"
)
self.o("")
self.o(" %s" %
console.format.link("%sconfiguration.html" % DOC_LINK))
self.o("")
else:
self.o(
"Running simple configuration. Run the advanced configuration with:"
)
self.o("")
self.o(" %s configure --advanced-config" % PROGRAM_NAME)
self.o("")
if config_file.present:
self.o("\nWARNING: Will overwrite existing config file at [%s]\n" %
config_file.location)
logger.debug("Detected an existing configuration file at [%s]" %
config_file.location)
else:
logger.debug(
"Did not detect a configuration file at [%s]. Running initial configuration routine."
% config_file.location)
# Autodetect settings
self.o("* Autodetecting available third-party software")
git_path = io.guess_install_location("git")
gradle_bin = "./gradlew" if use_gradle_wrapper else io.guess_install_location(
"gradle")
java_8_home = runtime_java_home if runtime_java_home else io.guess_java_home(
major_version=8)
java_9_home = java_home if java_home else io.guess_java_home(
major_version=9)
from esrally.utils import jvm
if java_8_home:
auto_detected_java_home = java_8_home
# Don't auto-detect an EA release and bring trouble to the user later on. They can still configure it manually if they want to.
elif java_9_home and not jvm.is_early_access_release(java_9_home):
auto_detected_java_home = java_9_home
else:
auto_detected_java_home = None
self.print_detection_result("git ", git_path)
self.print_detection_result("gradle ", gradle_bin)
self.print_detection_result(
"JDK ",
auto_detected_java_home,
warn_if_missing=True,
additional_message=
"You cannot benchmark Elasticsearch on this machine without a JDK."
)
self.o("")
# users that don't have Gradle available cannot benchmark from sources
benchmark_from_sources = gradle_bin
if not benchmark_from_sources:
self.o(
"********************************************************************************"
)
self.o(
"You don't have the required software to benchmark Elasticsearch source builds."
)
self.o("")
self.o("You can still benchmark binary distributions with e.g.:")
self.o("")
self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME)
self.o(
"********************************************************************************"
)
self.o("")
root_dir = io.normalize_path(
os.path.abspath(os.path.join(config_file.config_dir,
"benchmarks")))
if advanced_config:
root_dir = io.normalize_path(
self._ask_property("Enter the benchmark data directory",
default_value=root_dir))
else:
self.o("* Setting up benchmark data directory in %s" % root_dir)
if benchmark_from_sources:
if not java_9_home or jvm.is_early_access_release(java_9_home):
raw_java_9_home = self._ask_property(
"Enter the JDK 9 root directory",
check_path_exists=True,
mandatory=False)
if raw_java_9_home and jvm.major_version(
raw_java_9_home
) == 9 and not jvm.is_early_access_release(raw_java_9_home):
java_9_home = io.normalize_path(
raw_java_9_home) if raw_java_9_home else None
else:
benchmark_from_sources = False
self.o(
"********************************************************************************"
)
self.o(
"You don't have a valid JDK 9 installation and cannot benchmark source builds."
)
self.o("")
self.o(
"You can still benchmark binary distributions with e.g.:"
)
self.o("")
self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME)
self.o(
"********************************************************************************"
)
self.o("")
if benchmark_from_sources:
# We try to autodetect an existing ES source directory
guess = self._guess_es_src_dir()
if guess:
source_dir = guess
logger.debug(
"Autodetected Elasticsearch project directory at [%s]." %
source_dir)
else:
default_src_dir = os.path.join(root_dir, "src",
"elasticsearch")
logger.debug(
"Could not autodetect Elasticsearch project directory. Providing [%s] as default."
% default_src_dir)
source_dir = default_src_dir
if advanced_config:
source_dir = io.normalize_path(
self._ask_property(
"Enter your Elasticsearch project directory:",
default_value=source_dir))
if not advanced_config:
self.o("* Setting up benchmark source directory in %s" %
source_dir)
self.o("")
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = "https://github.com/elastic/elasticsearch.git"
if auto_detected_java_home:
java_home = auto_detected_java_home
local_benchmarks = True
else:
raw_java_home = self._ask_property(
"Enter the JDK root directory (version 8 or later)",
check_path_exists=True,
mandatory=False)
java_home = io.normalize_path(
raw_java_home) if raw_java_home else None
if not java_home:
local_benchmarks = False
self.o("")
self.o(
"********************************************************************************"
)
self.o(
"You don't have a JDK installed but Elasticsearch requires one to run. This means"
)
self.o(
"that you cannot benchmark Elasticsearch on this machine.")
self.o("")
self.o("You can still benchmark against remote machines e.g.:")
self.o("")
self.o(
" %s --pipeline=benchmark-only --target-host=\"NODE_IP:9200\""
% PROGRAM_NAME)
self.o("")
self.o("See %s for further info." %
console.format.link("%srecipes.html" % DOC_LINK))
self.o(
"********************************************************************************"
)
self.o("")
else:
local_benchmarks = True
if advanced_config:
data_store_choice = self._ask_property(
"Where should metrics be kept?"
"\n\n"
"(1) In memory (simpler but less options for analysis)\n"
"(2) Elasticsearch (requires a separate ES instance, keeps all raw samples for analysis)"
"\n\n",
default_value="1",
choices=["1", "2"])
if data_store_choice == "1":
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "", "", ""
else:
data_store_type = "elasticsearch"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store(
)
env_name = self._ask_env_name()
preserve_install = convert.to_bool(
self._ask_property(
"Do you want Rally to keep the Elasticsearch benchmark candidate "
"installation including the index (will use several GB per trial run)?",
default_value=False))
else:
# Does not matter for an in-memory store
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "", "", ""
preserve_install = False
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(Config.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["env.name"] = env_name
config["node"] = {}
config["node"]["root.dir"] = root_dir
if benchmark_from_sources:
# user has provided the Elasticsearch directory but the root for Elasticsearch and related plugins will be one level above
final_source_dir = io.normalize_path(
os.path.abspath(os.path.join(source_dir, os.pardir)))
config["node"]["src.root.dir"] = final_source_dir
config["source"] = {}
config["source"]["remote.repo.url"] = repo_url
# the Elasticsearch directory is just the last path component (relative to the source root directory)
config["source"]["elasticsearch.src.subdir"] = io.basename(
source_dir)
if gradle_bin:
config["build"] = {}
config["build"]["gradle.bin"] = gradle_bin
config["runtime"] = {}
if java_home:
config["runtime"]["java.home"] = java_home
if java_9_home:
config["runtime"]["java9.home"] = java_9_home
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = "${node:root.dir}/data"
config["reporting"] = {}
config["reporting"]["datastore.type"] = data_store_type
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
config["tracks"] = {}
config["tracks"][
"default.url"] = "https://github.com/elastic/rally-tracks"
config["teams"] = {}
config["teams"][
"default.url"] = "https://github.com/elastic/rally-teams"
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(
preserve_install)
config["distributions"] = {}
config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
"{{VERSION}}.tar.gz"
config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
"distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"][
"release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"]["release.cache"] = "true"
config_file.store(config)
self.o(
"Configuration successfully written to %s. Happy benchmarking!" %
config_file.location)
self.o("")
if local_benchmarks and benchmark_from_sources:
self.o(
"To benchmark Elasticsearch with the default benchmark, run:")
self.o("")
self.o(" %s" % PROGRAM_NAME)
self.o("")
elif local_benchmarks:
self.o(
"To benchmark Elasticsearch 6.0.0 with the default benchmark, run:"
)
self.o("")
self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME)
self.o("")
else:
# we've already printed an info for the user. No need to repeat that.
pass
self.o("More info about Rally:")
self.o("")
self.o("* Type %s --help" % PROGRAM_NAME)
self.o("* Read the documentation at %s" %
console.format.link(DOC_LINK))
self.o("* Ask a question on the forum at %s" % console.format.link(
"https://discuss.elastic.co/c/elasticsearch/rally"))
def migrate(config_file, current_version, target_version, out=print, i=input):
prompter = Prompter(i=i, o=out, assume_defaults=False)
logger.info("Upgrading configuration from version [%s] to [%s]." %
(current_version, target_version))
# Something is really fishy. We don't want to downgrade the configuration.
if current_version >= target_version:
raise ConfigError(
"The existing config file is available in a later version already. Expected version <= [%s] but found [%s]"
% (target_version, current_version))
# but first a backup...
config_file.backup()
config = config_file.load(interpolation=None)
if current_version == 0 and target_version > current_version:
logger.info("Migrating config from version [0] to [1]")
current_version = 1
config["meta"] = {}
config["meta"]["config.version"] = str(current_version)
# in version 1 we changed some directories from being absolute to being relative
config["system"]["log.root.dir"] = "logs"
config["provisioning"]["local.install.dir"] = "install"
config["reporting"]["report.base.dir"] = "reports"
if current_version == 1 and target_version > current_version:
logger.info("Migrating config from version [1] to [2]")
current_version = 2
config["meta"]["config.version"] = str(current_version)
# no need to ask the user now if we are about to upgrade to version 4
config["reporting"]["datastore.type"] = "in-memory"
config["reporting"]["datastore.host"] = ""
config["reporting"]["datastore.port"] = ""
config["reporting"]["datastore.secure"] = ""
config["reporting"]["datastore.user"] = ""
config["reporting"]["datastore.password"] = ""
config["system"]["env.name"] = "local"
if current_version == 2 and target_version > current_version:
logger.info("Migrating config from version [2] to [3]")
current_version = 3
config["meta"]["config.version"] = str(current_version)
# Remove obsolete settings
config["reporting"].pop("report.base.dir")
config["reporting"].pop("output.html.report.filename")
if current_version == 3 and target_version > current_version:
root_dir = config["system"]["root.dir"]
out("""
*****************************************************************************************
You have an old configuration of Rally. Rally has now a much simpler setup
routine which will autodetect lots of settings for you and it also does not
require you to setup a metrics store anymore.
Rally will now migrate your configuration but if you don't need advanced features
like a metrics store, then you should delete the configuration directory:
rm -rf {0}
and then rerun Rally's configuration routine:
{1} configure
Please also note you have {2:.1f} GB of data in your current benchmark directory at
{3}
You might want to clean up this directory also.
For more details please see {4}
*****************************************************************************************
Pausing for 10 seconds to let you consider this message.
""".format(
config_file.config_dir, PROGRAM_NAME,
convert.bytes_to_gb(io.get_size(root_dir)), root_dir,
console.format.link(
"https://github.com/elastic/rally/blob/master/CHANGELOG.md#030"
)))
time.sleep(10)
logger.info("Migrating config from version [3] to [4]")
current_version = 4
config["meta"]["config.version"] = str(current_version)
if len(config["reporting"]["datastore.host"]) > 0:
config["reporting"]["datastore.type"] = "elasticsearch"
else:
config["reporting"]["datastore.type"] = "in-memory"
# Remove obsolete settings
config["build"].pop("maven.bin")
config["benchmarks"].pop("metrics.stats.disk.device")
if current_version == 4 and target_version > current_version:
config["tracks"] = {}
config["tracks"][
"default.url"] = "https://github.com/elastic/rally-tracks"
current_version = 5
config["meta"]["config.version"] = str(current_version)
if current_version == 5 and target_version > current_version:
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(False)
current_version = 6
config["meta"]["config.version"] = str(current_version)
if current_version == 6 and target_version > current_version:
# Remove obsolete settings
config.pop("provisioning")
config["system"].pop("log.root.dir")
current_version = 7
config["meta"]["config.version"] = str(current_version)
if current_version == 7 and target_version > current_version:
# move [system][root.dir] to [node][root.dir]
if "node" not in config:
config["node"] = {}
config["node"]["root.dir"] = config["system"].pop("root.dir")
# also move all references!
for section in config:
for k, v in config[section].items():
config[section][k] = v.replace("${system:root.dir}",
"${node:root.dir}")
current_version = 8
config["meta"]["config.version"] = str(current_version)
if current_version == 8 and target_version > current_version:
config["teams"] = {}
config["teams"][
"default.url"] = "https://github.com/elastic/rally-teams"
current_version = 9
config["meta"]["config.version"] = str(current_version)
if current_version == 9 and target_version > current_version:
config["distributions"] = {}
config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
"{{VERSION}}.tar.gz"
config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
"distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"][
"release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"]["release.cache"] = "true"
current_version = 10
config["meta"]["config.version"] = str(current_version)
if current_version == 10 and target_version > current_version:
config["runtime"]["java.home"] = config["runtime"].pop("java8.home")
current_version = 11
config["meta"]["config.version"] = str(current_version)
if current_version == 11 and target_version > current_version:
# As this is a rather complex migration, we log more than usual to understand potential migration problems better.
if "source" in config:
if "local.src.dir" in config["source"]:
previous_root = config["source"].pop("local.src.dir")
logger.info("Set [source][local.src.dir] to [%s]." %
previous_root)
# if this directory was Rally's default location, then move it on the file system because to allow for checkouts of plugins
# in the sibling directory.
if previous_root == os.path.join(config["node"]["root.dir"],
"src"):
new_root_dir_all_sources = previous_root
new_es_sub_dir = "elasticsearch"
new_root = os.path.join(new_root_dir_all_sources,
new_es_sub_dir)
# only attempt to move if the directory exists. It may be possible that users never ran a source benchmark although they
# have configured it. In that case the source directory will not yet exist.
if io.exists(previous_root):
logger.info(
"Previous source directory was at Rally's default location [%s]. Moving to [%s]."
% (previous_root, new_root))
try:
# we need to do this in two steps as we need to move the sources to a subdirectory
tmp_path = io.normalize_path(
os.path.join(new_root_dir_all_sources,
os.pardir, "tmp_src_mig"))
os.rename(previous_root, tmp_path)
io.ensure_dir(new_root)
os.rename(tmp_path, new_root)
except OSError:
logger.exception(
"Could not move source directory from [%s] to [%s]."
% (previous_root, new_root))
# A warning is sufficient as Rally should just do a fresh checkout if moving did not work.
console.warn(
"Elasticsearch source directory could not be moved from [%s] to [%s]. Please check the logs."
% (previous_root, new_root))
else:
logger.info(
"Source directory is configured at Rally's default location [%s] but does not exist yet."
% previous_root)
else:
logger.info(
"Previous source directory was the custom directory [%s]."
% previous_root)
new_root_dir_all_sources = io.normalize_path(
os.path.join(previous_root, os.path.pardir))
# name of the elasticsearch project directory.
new_es_sub_dir = io.basename(previous_root)
logger.info("Setting [node][src.root.dir] to [%s]." %
new_root_dir_all_sources)
config["node"]["src.root.dir"] = new_root_dir_all_sources
logger.info(
"Setting [source][elasticsearch.src.subdir] to [%s]" %
new_es_sub_dir)
config["source"]["elasticsearch.src.subdir"] = new_es_sub_dir
else:
logger.info(
"Key [local.src.dir] not found. Advancing without changes."
)
else:
logger.info(
"No section named [source] found in config. Advancing without changes."
)
current_version = 12
config["meta"]["config.version"] = str(current_version)
if current_version == 12 and target_version > current_version:
# the current configuration allows to benchmark from sources
if "build" in config and "gradle.bin" in config["build"]:
java_9_home = io.guess_java_home(major_version=9)
from esrally.utils import jvm
if java_9_home and not jvm.is_early_access_release(java_9_home):
logger.debug("Autodetected a JDK 9 installation at [%s]" %
java_9_home)
if "runtime" not in config:
config["runtime"] = {}
config["runtime"]["java9.home"] = java_9_home
else:
logger.debug(
"Could not autodetect a JDK 9 installation. Checking [java.home] already points to a JDK 9."
)
detected = False
if "runtime" in config:
java_home = config["runtime"]["java.home"]
if jvm.major_version(
java_home
) == 9 and not jvm.is_early_access_release(java_home):
config["runtime"]["java9.home"] = java_home
detected = True
if not detected:
logger.debug(
"Could not autodetect a JDK 9 installation. Asking user."
)
raw_java_9_home = prompter.ask_property(
"Enter the JDK 9 root directory",
check_path_exists=True,
mandatory=False)
if raw_java_9_home and jvm.major_version(
raw_java_9_home
) == 9 and not jvm.is_early_access_release(
raw_java_9_home):
java_9_home = io.normalize_path(
raw_java_9_home) if raw_java_9_home else None
config["runtime"]["java9.home"] = java_9_home
else:
out("********************************************************************************"
)
out("You don't have a valid JDK 9 installation and cannot benchmark source builds."
)
out("")
out("You can still benchmark binary distributions with e.g.:"
)
out("")
out(" %s --distribution-version=6.0.0" % PROGRAM_NAME)
out("********************************************************************************"
)
out("")
current_version = 13
config["meta"]["config.version"] = str(current_version)
# all migrations done
config_file.store(config)
logger.info("Successfully self-upgraded configuration to version [%s]" %
target_version)
def render_template_from_file(template_file_name):
return render_template(loader=jinja2.FileSystemLoader(
io.dirname(template_file_name)),
template_name=io.basename(template_file_name))
def __car_name(path):
p, _ = io.splitext(path)
return io.basename(p)
def _render_template(env, variables, file_name):
template = env.get_template(io.basename(file_name))
# force a new line at the end. Jinja seems to remove it.
return template.render(variables) + "\n"