def create_config(self,
config_file,
advanced_config=False,
assume_defaults=False,
use_gradle_wrapper=False,
java_home=None,
runtime_java_home=None):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param config_file:
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
:param assume_defaults: If True, assume the user accepted all values for which defaults are provided. Mainly intended for automatic
configuration in CI run. Default: False.
:param use_gradle_wrapper: If True, use the Gradle wrapper, otherwise use the system's Gradle version. Default: False.
"""
self.prompter = Prompter(self.i, self.sec_i, self.o, assume_defaults)
if advanced_config:
self.o(
"Running advanced configuration. You can get additional help at:"
)
self.o("")
self.o(" %s" %
console.format.link("%sconfiguration.html" % DOC_LINK))
self.o("")
else:
self.o(
"Running simple configuration. Run the advanced configuration with:"
)
self.o("")
self.o(" %s configure --advanced-config" % PROGRAM_NAME)
self.o("")
if config_file.present:
self.o("\nWARNING: Will overwrite existing config file at [%s]\n" %
config_file.location)
logger.debug("Detected an existing configuration file at [%s]" %
config_file.location)
else:
logger.debug(
"Did not detect a configuration file at [%s]. Running initial configuration routine."
% config_file.location)
# Autodetect settings
self.o("* Autodetecting available third-party software")
git_path = io.guess_install_location("git")
gradle_bin = "./gradlew" if use_gradle_wrapper else io.guess_install_location(
"gradle")
java_8_home = runtime_java_home if runtime_java_home else io.guess_java_home(
major_version=8)
java_9_home = java_home if java_home else io.guess_java_home(
major_version=9)
from esrally.utils import jvm
if java_8_home:
auto_detected_java_home = java_8_home
# Don't auto-detect an EA release and bring trouble to the user later on. They can still configure it manually if they want to.
elif java_9_home and not jvm.is_early_access_release(java_9_home):
auto_detected_java_home = java_9_home
else:
auto_detected_java_home = None
self.print_detection_result("git ", git_path)
self.print_detection_result("gradle ", gradle_bin)
self.print_detection_result(
"JDK ",
auto_detected_java_home,
warn_if_missing=True,
additional_message=
"You cannot benchmark Elasticsearch on this machine without a JDK."
)
self.o("")
# users that don't have Gradle available cannot benchmark from sources
benchmark_from_sources = gradle_bin
if not benchmark_from_sources:
self.o(
"********************************************************************************"
)
self.o(
"You don't have the required software to benchmark Elasticsearch source builds."
)
self.o("")
self.o("You can still benchmark binary distributions with e.g.:")
self.o("")
self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME)
self.o(
"********************************************************************************"
)
self.o("")
root_dir = io.normalize_path(
os.path.abspath(os.path.join(config_file.config_dir,
"benchmarks")))
if advanced_config:
root_dir = io.normalize_path(
self._ask_property("Enter the benchmark data directory",
default_value=root_dir))
else:
self.o("* Setting up benchmark data directory in %s" % root_dir)
if benchmark_from_sources:
if not java_9_home or jvm.is_early_access_release(java_9_home):
raw_java_9_home = self._ask_property(
"Enter the JDK 9 root directory",
check_path_exists=True,
mandatory=False)
if raw_java_9_home and jvm.major_version(
raw_java_9_home
) == 9 and not jvm.is_early_access_release(raw_java_9_home):
java_9_home = io.normalize_path(
raw_java_9_home) if raw_java_9_home else None
else:
benchmark_from_sources = False
self.o(
"********************************************************************************"
)
self.o(
"You don't have a valid JDK 9 installation and cannot benchmark source builds."
)
self.o("")
self.o(
"You can still benchmark binary distributions with e.g.:"
)
self.o("")
self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME)
self.o(
"********************************************************************************"
)
self.o("")
if benchmark_from_sources:
# We try to autodetect an existing ES source directory
guess = self._guess_es_src_dir()
if guess:
source_dir = guess
logger.debug(
"Autodetected Elasticsearch project directory at [%s]." %
source_dir)
else:
default_src_dir = os.path.join(root_dir, "src",
"elasticsearch")
logger.debug(
"Could not autodetect Elasticsearch project directory. Providing [%s] as default."
% default_src_dir)
source_dir = default_src_dir
if advanced_config:
source_dir = io.normalize_path(
self._ask_property(
"Enter your Elasticsearch project directory:",
default_value=source_dir))
if not advanced_config:
self.o("* Setting up benchmark source directory in %s" %
source_dir)
self.o("")
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = "https://github.com/elastic/elasticsearch.git"
if auto_detected_java_home:
java_home = auto_detected_java_home
local_benchmarks = True
else:
raw_java_home = self._ask_property(
"Enter the JDK root directory (version 8 or later)",
check_path_exists=True,
mandatory=False)
java_home = io.normalize_path(
raw_java_home) if raw_java_home else None
if not java_home:
local_benchmarks = False
self.o("")
self.o(
"********************************************************************************"
)
self.o(
"You don't have a JDK installed but Elasticsearch requires one to run. This means"
)
self.o(
"that you cannot benchmark Elasticsearch on this machine.")
self.o("")
self.o("You can still benchmark against remote machines e.g.:")
self.o("")
self.o(
" %s --pipeline=benchmark-only --target-host=\"NODE_IP:9200\""
% PROGRAM_NAME)
self.o("")
self.o("See %s for further info." %
console.format.link("%srecipes.html" % DOC_LINK))
self.o(
"********************************************************************************"
)
self.o("")
else:
local_benchmarks = True
if advanced_config:
data_store_choice = self._ask_property(
"Where should metrics be kept?"
"\n\n"
"(1) In memory (simpler but less options for analysis)\n"
"(2) Elasticsearch (requires a separate ES instance, keeps all raw samples for analysis)"
"\n\n",
default_value="1",
choices=["1", "2"])
if data_store_choice == "1":
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "", "", ""
else:
data_store_type = "elasticsearch"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store(
)
env_name = self._ask_env_name()
preserve_install = convert.to_bool(
self._ask_property(
"Do you want Rally to keep the Elasticsearch benchmark candidate "
"installation including the index (will use several GB per trial run)?",
default_value=False))
else:
# Does not matter for an in-memory store
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "", "", ""
preserve_install = False
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(Config.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["env.name"] = env_name
config["node"] = {}
config["node"]["root.dir"] = root_dir
if benchmark_from_sources:
# user has provided the Elasticsearch directory but the root for Elasticsearch and related plugins will be one level above
final_source_dir = io.normalize_path(
os.path.abspath(os.path.join(source_dir, os.pardir)))
config["node"]["src.root.dir"] = final_source_dir
config["source"] = {}
config["source"]["remote.repo.url"] = repo_url
# the Elasticsearch directory is just the last path component (relative to the source root directory)
config["source"]["elasticsearch.src.subdir"] = io.basename(
source_dir)
if gradle_bin:
config["build"] = {}
config["build"]["gradle.bin"] = gradle_bin
config["runtime"] = {}
if java_home:
config["runtime"]["java.home"] = java_home
if java_9_home:
config["runtime"]["java9.home"] = java_9_home
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = "${node:root.dir}/data"
config["reporting"] = {}
config["reporting"]["datastore.type"] = data_store_type
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
config["tracks"] = {}
config["tracks"][
"default.url"] = "https://github.com/elastic/rally-tracks"
config["teams"] = {}
config["teams"][
"default.url"] = "https://github.com/elastic/rally-teams"
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(
preserve_install)
config["distributions"] = {}
config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
"{{VERSION}}.tar.gz"
config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
"distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"][
"release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"]["release.cache"] = "true"
config_file.store(config)
self.o(
"Configuration successfully written to %s. Happy benchmarking!" %
config_file.location)
self.o("")
if local_benchmarks and benchmark_from_sources:
self.o(
"To benchmark Elasticsearch with the default benchmark, run:")
self.o("")
self.o(" %s" % PROGRAM_NAME)
self.o("")
elif local_benchmarks:
self.o(
"To benchmark Elasticsearch 6.0.0 with the default benchmark, run:"
)
self.o("")
self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME)
self.o("")
else:
# we've already printed an info for the user. No need to repeat that.
pass
self.o("More info about Rally:")
self.o("")
self.o("* Type %s --help" % PROGRAM_NAME)
self.o("* Read the documentation at %s" %
console.format.link(DOC_LINK))
self.o("* Ask a question on the forum at %s" % console.format.link(
"https://discuss.elastic.co/c/elasticsearch/rally"))
def create_config(self,
config_file,
advanced_config=False,
assume_defaults=False):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param config_file:
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
:param assume_defaults: If True, assume the user accepted all values for which defaults are provided. Mainly intended for automatic
configuration in CI run. Default: False.
"""
self.assume_defaults = assume_defaults
if advanced_config:
self.o(
"Running advanced configuration. You can get additional help at:"
)
self.o("")
self.o(" %s" %
console.format.link("%sconfiguration.html" % DOC_LINK))
self.o("")
logger.info("Running advanced configuration routine.")
self.o("")
else:
self.o(
"Running simple configuration. Run the advanced configuration with:"
)
self.o("")
self.o(" %s configure --advanced-config" % PROGRAM_NAME)
self.o("")
logger.info("Running simple configuration routine.")
if config_file.present:
self.o("\nWARNING: Will overwrite existing config file at [%s]\n" %
config_file.location)
logger.debug("Detected an existing configuration file at [%s]" %
config_file.location)
else:
logger.debug(
"Did not detect a configuration file at [%s]. Running initial configuration routine."
% config_file.location)
# Autodetect settings
self.o("* Autodetecting available third-party software")
git_path = io.guess_install_location("git")
gradle_bin = io.guess_install_location("gradle")
java_9_home = io.guess_java_home(major_version=9)
from esrally.utils import jvm
# Don't auto-detect an EA release and bring trouble to the user later on. They can still configure it manually if they want to.
if java_9_home and not jvm.is_early_access_release(java_9_home):
auto_detected_java_home = java_9_home
else:
auto_detected_java_home = io.guess_java_home(major_version=8)
self.print_detection_result("git ", git_path)
self.print_detection_result("gradle ", gradle_bin)
self.print_detection_result(
"JDK ",
auto_detected_java_home,
warn_if_missing=True,
additional_message=
"You cannot benchmark Elasticsearch without a JDK installation")
self.o("")
# users that don't have Gradle available cannot benchmark from sources
benchmark_from_sources = gradle_bin
if not benchmark_from_sources:
self.o(
"**********************************************************************************"
)
self.o(
"You don't have the necessary software to benchmark source builds of Elasticsearch."
)
self.o("")
self.o("You can still benchmark binary distributions with e.g.:")
self.o("")
self.o(" %s --distribution-version=5.0.0" % PROGRAM_NAME)
self.o(
"**********************************************************************************"
)
self.o("")
root_dir = "%s/benchmarks" % config_file.config_dir
if advanced_config:
root_dir = io.normalize_path(
self._ask_property("Enter the benchmark data directory:",
default_value=root_dir))
else:
self.o(
"* Setting up benchmark data directory in [%s] (needs several GB)."
% root_dir)
if benchmark_from_sources:
# We try to autodetect an existing ES source directory
guess = self._guess_es_src_dir()
if guess:
source_dir = guess
self.o(
"Autodetected Elasticsearch project directory at [%s]." %
source_dir)
logger.debug(
"Autodetected Elasticsearch project directory at [%s]." %
source_dir)
else:
default_src_dir = "%s/src" % root_dir
logger.debug(
"Could not autodetect Elasticsearch project directory. Providing [%s] as default."
% default_src_dir)
source_dir = io.normalize_path(
self._ask_property(
"Enter your Elasticsearch project directory:",
default_value=default_src_dir))
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = "https://github.com/elastic/elasticsearch.git"
if auto_detected_java_home:
java_home = auto_detected_java_home
else:
self.o("")
java_home = io.normalize_path(
self._ask_property("Enter the JDK root directory:",
check_path_exists=True))
if advanced_config:
env_name = self._ask_env_name()
data_store_type = "elasticsearch"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store(
)
preserve_install = convert.to_bool(
self._ask_property(
"Do you want Rally to keep the Elasticsearch benchmark candidate "
"installation including the index (will use lots of disk space)?",
default_value=False))
else:
# Does not matter too much for an in-memory store
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "", "", ""
preserve_install = False
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(Config.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["env.name"] = env_name
config["node"] = {}
config["node"]["root.dir"] = root_dir
if benchmark_from_sources:
config["source"] = {}
config["source"]["local.src.dir"] = source_dir
config["source"]["remote.repo.url"] = repo_url
config["build"] = {}
config["build"]["gradle.bin"] = gradle_bin
config["runtime"] = {}
config["runtime"]["java.home"] = java_home
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = "${node:root.dir}/data"
config["reporting"] = {}
config["reporting"]["datastore.type"] = data_store_type
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
config["tracks"] = {}
config["tracks"][
"default.url"] = "https://github.com/elastic/rally-tracks"
config["teams"] = {}
config["teams"][
"default.url"] = "https://github.com/elastic/rally-teams"
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(
preserve_install)
config["distributions"] = {}
config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
"{{VERSION}}.tar.gz"
config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
"distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"][
"release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
config["distributions"]["release.cache"] = "true"
config_file.store(config)
self.o(
"Configuration successfully written to [%s]. Happy benchmarking!" %
config_file.location)
self.o("")
if benchmark_from_sources:
self.o(
"To benchmark Elasticsearch with the default benchmark run:")
self.o("")
self.o(" %s" % PROGRAM_NAME)
else:
self.o(
"To benchmark Elasticsearch 5.0.0 with the default benchmark run:"
)
self.o("")
self.o(" %s --distribution-version=5.0.0" % PROGRAM_NAME)
self.o("")
self.o("For help, type %s --help or see the user documentation at %s" %
(PROGRAM_NAME, console.format.link(DOC_LINK)))
def create_config(self, advanced_config=False):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
"""
if self.config_present():
print("\nWARNING: Will overwrite existing config file at [%s]\n" % self._config_file.location)
else:
print("Rally has not detected any configuration file at %s. It will ask a few questions about the environment.\n\n" %
self._config_file.location)
print("The benchmark root directory contains benchmark data, logs, etc. . It will consume several GB of free space depending on "
"which benchmarks are executed (expect at least 10 GB).")
benchmark_root_dir = self._ask_property("Enter the benchmark root directory (will be created automatically)")
env_name = self._ask_env_name()
only_binary = self._ask_property("Do you want to benchmark only official binary distributions of Elasticsearch? Note: If you "
"answer with 'Yes' you will not need certain software (like Gradle or git) but you will not "
"be able to benchmark the latest development versions from Elasticsearch unless you reconfigure "
"Rally with esrally --configure again.",
check_pattern=Config.BOOLEAN_PATTERN, default_value="No")
if not (self._to_bool(only_binary)):
default_src_dir = "%s/src" % benchmark_root_dir
source_dir = self._ask_property("Enter the directory where sources are located (your Elasticsearch project directory)",
default_value=default_src_dir)
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = self._ask_property("Enter the Elasticsearch repo URL", default_value="https://github.com/elastic/elasticsearch.git")
default_gradle_location = io.guess_install_location("gradle", fallback="/usr/local/bin/gradle")
gradle_bin = self._ask_property("Enter the full path to the Gradle binary", default_value=default_gradle_location,
check_path_exists=True)
if advanced_config:
default_mvn_location = io.guess_install_location("mvn", fallback="/usr/local/bin/mvn")
maven_bin = self._ask_property("Enter the full path to the Maven 3 binary", default_value=default_mvn_location,
check_path_exists=True)
else:
maven_bin = ""
default_jdk_8 = io.guess_java_home(major_version=8, fallback="")
jdk8_home = self._ask_property(
"Enter the JDK 8 root directory (e.g. something like /Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home on a Mac)",
default_value=default_jdk_8,
check_path_exists=True)
if advanced_config:
stats_disk_device = self._ask_property("Enter the HDD device name for stats (e.g. /dev/disk1)")
else:
stats_disk_device = ""
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store()
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(self.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["root.dir"] = benchmark_root_dir
config["system"]["log.root.dir"] = "logs"
config["system"]["env.name"] = env_name
if not (self._to_bool(only_binary)):
config["source"] = {}
config["source"]["local.src.dir"] = source_dir
config["source"]["remote.repo.url"] = repo_url
config["build"] = {}
config["build"]["gradle.bin"] = gradle_bin
config["build"]["maven.bin"] = maven_bin
config["provisioning"] = {}
config["provisioning"]["local.install.dir"] = "install"
config["runtime"] = {}
config["runtime"]["java8.home"] = jdk8_home
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = "${system:root.dir}/data"
config["benchmarks"]["metrics.stats.disk.device"] = stats_disk_device
config["reporting"] = {}
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
self._config_file.store(config)
print("\nConfiguration successfully written to '%s'. Please rerun esrally now." % self._config_file.location)
def create_config(self, config_file, advanced_config=False, assume_defaults=False):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param config_file:
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
:param assume_defaults: If True, assume the user accepted all values for which defaults are provided. Mainly intended for automatic
configuration in CI run. Default: False.
"""
self.assume_defaults = assume_defaults
if advanced_config:
self.o("Running advanced configuration. You can get additional help at:")
self.o("")
self.o(" %s" % console.format.link("%sconfiguration.html" % DOC_LINK))
self.o("")
logger.info("Running advanced configuration routine.")
self.o("")
else:
self.o("Running simple configuration. Run the advanced configuration with:")
self.o("")
self.o(" %s configure --advanced-config" % PROGRAM_NAME)
self.o("")
logger.info("Running simple configuration routine.")
if config_file.present:
self.o("\nWARNING: Will overwrite existing config file at [%s]\n" % config_file.location)
logger.debug("Detected an existing configuration file at [%s]" % config_file.location)
else:
logger.debug("Did not detect a configuration file at [%s]. Running initial configuration routine." % config_file.location)
# Autodetect settings
self.o("[✓] Autodetecting available third-party software")
git_path = io.guess_install_location("git")
gradle_bin = io.guess_install_location("gradle")
# default_jdk_7 = io.guess_java_home(major_version=7)
default_jdk_8 = io.guess_java_home(major_version=8)
self.print_detection_result("git ", git_path)
self.print_detection_result("gradle ", gradle_bin)
self.print_detection_result("JDK 8 ", default_jdk_8,
warn_if_missing=True,
additional_message="You cannot benchmark Elasticsearch 5.x without a JDK 8 installation")
# self.print_detection_result("JDK 9 ", default_jdk_9, warn_if_missing=True)
self.o("")
# users that don't have Gradle available cannot benchmark from sources
benchmark_from_sources = gradle_bin
if not benchmark_from_sources:
self.o("**********************************************************************************")
self.o("You don't have the necessary software to benchmark source builds of Elasticsearch.")
self.o("")
self.o("You can still benchmark binary distributions with e.g.:")
self.o("")
self.o(" %s --distribution-version=5.0.0" % PROGRAM_NAME)
self.o("**********************************************************************************")
self.o("")
root_dir = "%s/benchmarks" % config_file.config_dir
self.o("[✓] Setting up benchmark data directory in [%s] (needs several GB)." % root_dir)
if benchmark_from_sources:
# We try to autodetect an existing ES source directory
guess = self._guess_es_src_dir()
if guess:
source_dir = guess
self.o("[✓] Autodetected Elasticsearch project directory at [%s]." % source_dir)
logger.debug("Autodetected Elasticsearch project directory at [%s]." % source_dir)
else:
default_src_dir = "%s/src" % root_dir
logger.debug("Could not autodetect Elasticsearch project directory. Providing [%s] as default." % default_src_dir)
source_dir = io.normalize_path(self._ask_property("Enter your Elasticsearch project directory:",
default_value=default_src_dir))
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = "https://github.com/elastic/elasticsearch.git"
if default_jdk_8:
jdk8_home = default_jdk_8
else:
self.o("")
jdk8_home = io.normalize_path(self._ask_property("Enter the JDK 8 root directory:", check_path_exists=True))
if advanced_config:
env_name = self._ask_env_name()
data_store_type = "elasticsearch"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store()
preserve_install = convert.to_bool(self._ask_property("Do you want Rally to keep the Elasticsearch benchmark candidate "
"installation including the index (will use lots of disk space)?",
default_value=False))
else:
# Does not matter too much for an in-memory store
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "", "", ""
preserve_install = False
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(Config.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["root.dir"] = root_dir
config["system"]["log.root.dir"] = "logs"
config["system"]["env.name"] = env_name
if benchmark_from_sources:
config["source"] = {}
config["source"]["local.src.dir"] = source_dir
config["source"]["remote.repo.url"] = repo_url
config["build"] = {}
config["build"]["gradle.bin"] = gradle_bin
config["provisioning"] = {}
config["provisioning"]["local.install.dir"] = "install"
config["runtime"] = {}
config["runtime"]["java8.home"] = jdk8_home
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = "${system:root.dir}/data"
config["reporting"] = {}
config["reporting"]["datastore.type"] = data_store_type
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
config["tracks"] = {}
config["tracks"]["default.url"] = "https://github.com/elastic/rally-tracks"
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(preserve_install)
config_file.store(config)
self.o("[✓] Configuration successfully written to [%s]. Happy benchmarking!" % config_file.location)
self.o("")
if benchmark_from_sources:
self.o("To benchmark Elasticsearch with the default benchmark run:")
self.o("")
self.o(" %s" % PROGRAM_NAME)
else:
self.o("To benchmark Elasticsearch 5.0.0 with the default benchmark run:")
self.o("")
self.o(" %s --distribution-version=5.0.0" % PROGRAM_NAME)
self.o("")
self.o("For help, type %s --help or see the user documentation at %s"
% (PROGRAM_NAME, console.format.link(DOC_LINK)))
def create_config(self, config_file, advanced_config=False):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
"""
if advanced_config:
self.o(
"Running advanced configuration. You can get additional help at:"
)
self.o("")
self.o(" %s" % format.link(
"https://esrally.readthedocs.io/en/latest/configuration.html"))
self.o("")
logger.debug("Running advanced configuration routine.")
self.o("")
else:
self.o(
"Running simple configuration. You can run the advanced configuration with:"
)
self.o("")
self.o(" %s configure --advanced-config" % PROGRAM_NAME)
self.o("")
logger.debug("Running simple configuration routine.")
if config_file.present:
self.o("\nWARNING: Will overwrite existing config file at [%s]\n" %
config_file.location)
logger.debug("Detected an existing configuration file at [%s]" %
config_file.location)
else:
logger.debug(
"Did not detect a configuration file at [%s]. Running initial configuration routine."
% config_file.location)
# Autodetect settings
self.o("[✓] Autodetecting available third-party software")
git_path = io.guess_install_location("git")
gradle_bin = io.guess_install_location("gradle")
# default_jdk_7 = io.guess_java_home(major_version=7)
default_jdk_8 = io.guess_java_home(major_version=8)
self.print_detection_result("git ", git_path)
self.print_detection_result("gradle ", gradle_bin)
# self.print_detection_result("JDK 7 ", default_jdk_7, warn_if_missing=True, additional_message="Cannot benchmark Elasticsearch 2.x")
self.print_detection_result(
"JDK 8 ",
default_jdk_8,
warn_if_missing=True,
additional_message=
"You cannot benchmark Elasticsearch 5.x without a JDK 8 installation"
)
# self.print_detection_result("JDK 9 ", default_jdk_9, warn_if_missing=True)
self.o("")
# users that don't have Gradle available cannot benchmark from sources
benchmark_from_sources = gradle_bin
if not benchmark_from_sources:
self.o(
"**********************************************************************************"
)
self.o(
"You don't have the necessary software to benchmark source builds of Elasticsearch."
)
self.o("")
self.o("You can still benchmark binary distributions with e.g.:")
self.o("")
self.o(
" %s --pipeline=from-distribution --distribution-version=5.0.0-alpha2"
% PROGRAM_NAME)
self.o("")
self.o("See %s" % format.link(
"https://esrally.readthedocs.io/en/latest/pipelines.html#from-distribution"
))
self.o(
"**********************************************************************************"
)
self.o("")
root_dir = "%s/benchmarks" % config_file.config_dir
self.o(
"[✓] Setting up benchmark data directory in [%s] (needs several GB)."
% root_dir)
if benchmark_from_sources:
# We try to autodetect an existing ES source directory
guess = self._guess_es_src_dir()
if guess:
source_dir = guess
self.o(
"[✓] Autodetected Elasticsearch project directory at [%s]."
% source_dir)
logger.debug(
"Autodetected Elasticsearch project directory at [%s]." %
source_dir)
else:
default_src_dir = "%s/src" % root_dir
logger.debug(
"Could not autodetect Elasticsearch project directory. Providing [%s] as default."
% default_src_dir)
source_dir = io.normalize_path(
self._ask_property(
"Enter your Elasticsearch project directory:",
default_value=default_src_dir))
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = "https://github.com/elastic/elasticsearch.git"
if default_jdk_8:
jdk8_home = default_jdk_8
else:
self.o("")
jdk8_home = io.normalize_path(
self._ask_property("Enter the JDK 8 root directory:",
check_path_exists=True))
if advanced_config:
env_name = self._ask_env_name()
data_store_type = "elasticsearch"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store(
)
preserve_install = convert.to_bool(
self._ask_property(
"Do you want Rally to keep the Elasticsearch benchmark candidate "
"installation including all data (will use lots of disk space)",
default_value=False))
else:
# Does not matter too much for an in-memory store
env_name = "local"
data_store_type = "in-memory"
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "", "", ""
preserve_install = False
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(Config.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["root.dir"] = root_dir
config["system"]["log.root.dir"] = "logs"
config["system"]["env.name"] = env_name
if benchmark_from_sources:
config["source"] = {}
config["source"]["local.src.dir"] = source_dir
config["source"]["remote.repo.url"] = repo_url
config["build"] = {}
config["build"]["gradle.bin"] = gradle_bin
config["provisioning"] = {}
config["provisioning"]["local.install.dir"] = "install"
config["runtime"] = {}
config["runtime"]["java8.home"] = jdk8_home
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = "${system:root.dir}/data"
config["reporting"] = {}
config["reporting"]["datastore.type"] = data_store_type
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
config["tracks"] = {}
config["tracks"][
"default.url"] = "https://github.com/elastic/rally-tracks"
config["defaults"] = {}
config["defaults"]["preserve_benchmark_candidate"] = str(
preserve_install)
config_file.store(config)
self.o(
"[✓] Configuration successfully written to [%s]. Happy benchmarking!"
% config_file.location)
self.o("")
if benchmark_from_sources:
self.o(
"To benchmark the currently checked out version of Elasticsearch with the default benchmark run:"
)
self.o("")
self.o(" %s" % PROGRAM_NAME)
else:
self.o(
"To benchmark Elasticsearch 5.0.0-alpha2 with the default benchmark run:"
)
self.o("")
self.o(
" %s --pipeline=from-distribution --distribution-version=5.0.0-alpha2"
% PROGRAM_NAME)
self.o()
self.o("For help, type %s --help or see the user documentation at %s" %
(PROGRAM_NAME, format.link("https://esrally.readthedocs.io")))
def create_config(self, advanced_config=False):
"""
Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties
and writes them to the configuration file in ~/.rally/rally.ini.
:param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False.
"""
if self.config_present():
print("\nWARNING: Will overwrite existing config file at [%s]\n" %
self._config_file.location)
else:
print(
"Rally has not detected any configuration file at %s. It will ask a few questions about the environment.\n\n"
% self._config_file.location)
print(
"The benchmark root directory contains benchmark data, logs, etc. . It will consume several GB of free space depending on "
"which benchmarks are executed (expect at least 10 GB).")
benchmark_root_dir = self._ask_property(
"Enter the benchmark root directory (will be created automatically)"
)
env_name = self._ask_env_name()
only_binary = self._ask_property(
"Do you want to benchmark only official binary distributions of Elasticsearch? Note: If you "
"answer with 'Yes' you will not need certain software (like Gradle or git) but you will not "
"be able to benchmark the latest development versions from Elasticsearch unless you reconfigure "
"Rally with esrally --configure again.",
check_pattern=Config.BOOLEAN_PATTERN,
default_value="No")
if not (self._to_bool(only_binary)):
default_src_dir = "%s/src" % benchmark_root_dir
source_dir = self._ask_property(
"Enter the directory where sources are located (your Elasticsearch project directory)",
default_value=default_src_dir)
# Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody.
repo_url = self._ask_property(
"Enter the Elasticsearch repo URL",
default_value="https://github.com/elastic/elasticsearch.git")
default_gradle_location = io.guess_install_location(
"gradle", fallback="/usr/local/bin/gradle")
gradle_bin = self._ask_property(
"Enter the full path to the Gradle binary",
default_value=default_gradle_location,
check_path_exists=True)
if advanced_config:
default_mvn_location = io.guess_install_location(
"mvn", fallback="/usr/local/bin/mvn")
maven_bin = self._ask_property(
"Enter the full path to the Maven 3 binary",
default_value=default_mvn_location,
check_path_exists=True)
else:
maven_bin = ""
default_jdk_8 = io.guess_java_home(major_version=8, fallback="")
jdk8_home = self._ask_property(
"Enter the JDK 8 root directory (e.g. something like /Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home on a Mac)",
default_value=default_jdk_8,
check_path_exists=True)
if advanced_config:
stats_disk_device = self._ask_property(
"Enter the HDD device name for stats (e.g. /dev/disk1)")
else:
stats_disk_device = ""
data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = self._ask_data_store(
)
config = configparser.ConfigParser()
config["meta"] = {}
config["meta"]["config.version"] = str(self.CURRENT_CONFIG_VERSION)
config["system"] = {}
config["system"]["root.dir"] = benchmark_root_dir
config["system"]["log.root.dir"] = "logs"
config["system"]["env.name"] = env_name
if not (self._to_bool(only_binary)):
config["source"] = {}
config["source"]["local.src.dir"] = source_dir
config["source"]["remote.repo.url"] = repo_url
config["build"] = {}
config["build"]["gradle.bin"] = gradle_bin
config["build"]["maven.bin"] = maven_bin
config["provisioning"] = {}
config["provisioning"]["local.install.dir"] = "install"
config["runtime"] = {}
config["runtime"]["java8.home"] = jdk8_home
config["benchmarks"] = {}
config["benchmarks"]["local.dataset.cache"] = "${system:root.dir}/data"
config["benchmarks"]["metrics.stats.disk.device"] = stats_disk_device
config["reporting"] = {}
config["reporting"]["datastore.host"] = data_store_host
config["reporting"]["datastore.port"] = data_store_port
config["reporting"]["datastore.secure"] = data_store_secure
config["reporting"]["datastore.user"] = data_store_user
config["reporting"]["datastore.password"] = data_store_password
self._config_file.store(config)
print(
"\nConfiguration successfully written to '%s'. Please rerun esrally now."
% self._config_file.location)
