def add_rally_user(config_names, variables, **kwargs):
if "x-pack-security" not in config_names:
return False
logger = logging.getLogger(LOGGER_NAME)
users_binary = "elasticsearch-users"
install_root = variables["install_root_path"]
logger.info("Adding user 'rally'.")
users = resolve_binary(install_root, users_binary)
return_code = process.run_subprocess_with_logging(
'{users} useradd rally -p "rally-password"'.format(users=users),
env=kwargs.get("env"))
if return_code != 0:
logger.error("%s has exited with code [%d]", users_binary, return_code)
raise exceptions.SystemSetupError(
"Could not add user 'rally'. Please see the log for details.")
return_code = process.run_subprocess_with_logging(
'{users} roles rally -a superuser'.format(users=users),
env=kwargs.get("env"))
if return_code != 0:
logger.error("%s has exited with code [%d]", users_binary, return_code)
raise exceptions.SystemSetupError(
"Could not add role 'superuser' for user 'rally'. Please see the log for details."
)
return True
def detach_from_cluster(self, cluster):
data_paths = self.config.opts("provisioning", "local.data.paths", mandatory=False)
if data_paths is not None:
data_path = data_paths[0]
index_size_bytes = io.get_size(data_path)
self.metrics_store.put_count_cluster_level("final_index_size_bytes", index_size_bytes, "byte")
process.run_subprocess_with_logging("find %s -ls" % data_path, header="index files:")
def add_rally_user(config_names, variables, **kwargs):
if "security" not in config_names:
return False
install_root = variables["install_root_path"]
logger.info("Adding Rally user.")
users = os.path.join(install_root, "bin", "x-pack", "users")
# ./bin/x-pack/users useradd rally -p pw-rally-benchmark
return_code = process.run_subprocess_with_logging(
'{users} useradd rally -p "rally-password"'.format(users=users))
if return_code != 0:
logger.error("users has exited with code [%d]" % return_code)
raise exceptions.SystemSetupError(
"Could not add x-pack user 'rally'. Please see the log for details."
)
# ./bin/x-pack/users roles rally -a superuser
return_code = process.run_subprocess_with_logging(
'{users} roles rally -a superuser'.format(users=users))
if return_code != 0:
logger.error("users has exited with code [%d]" % return_code)
raise exceptions.SystemSetupError(
"Could not add role 'superuser' for user 'rally'. Please see the log for details."
)
return True
def stop(self, nodes):
logger.info("Stopping Docker container")
for node in nodes:
node.telemetry.detach_from_node(node, running=True)
process.run_subprocess_with_logging(
"docker-compose -f %s down" %
self.binary_paths[node.node_name])
node.telemetry.detach_from_node(node, running=False)
def check_prerequisites():
if process.run_subprocess_with_logging("docker ps") != 0:
raise AssertionError(
"Docker must be installed and the daemon must be up and running to run integration tests."
)
if process.run_subprocess_with_logging("docker-compose --help") != 0:
raise AssertionError(
"Docker Compose is required to run integration tests.")
def detach_from_node(self, node, running):
# we need to gather the file size after the node has terminated so we can be sure that it has written all its buffers.
if not running and self.attached and self.data_paths:
self.attached = False
index_size_bytes = 0
for data_path in self.data_paths:
index_size_bytes += io.get_size(data_path)
process.run_subprocess_with_logging("find %s -ls" % data_path, header="index files:")
self.metrics_store.put_count_node_level(node.node_name, "final_index_size_bytes", index_size_bytes, "byte")
def stop(self, nodes):
if self.keep_running:
logger.info("Keeping Docker container running.")
else:
logger.info("Stopping Docker container")
for node in nodes:
node.telemetry.detach_from_node(node, running=True)
process.run_subprocess_with_logging("docker-compose -f %s down" % self.binary_paths[node.node_name])
node.telemetry.detach_from_node(node, running=False)
def stop(self, nodes, metrics_store):
self.logger.info("Shutting down [%d] nodes running in Docker on this host.", len(nodes))
for node in nodes:
self.logger.info("Stopping node [%s].", node.node_name)
telemetry.add_metadata_for_node(metrics_store, node.node_name, node.host_name)
node.telemetry.detach_from_node(node, running=True)
process.run_subprocess_with_logging(self._docker_compose(node.binary_path, "down"))
node.telemetry.detach_from_node(node, running=False)
node.telemetry.store_system_metrics(node, metrics_store)
def detach_from_cluster(self, cluster):
if self.attached and self.data_paths:
self.attached = False
data_path = self.data_paths[0]
index_size_bytes = io.get_size(data_path)
self.metrics_store.put_count_cluster_level(
"final_index_size_bytes", index_size_bytes, "byte")
process.run_subprocess_with_logging("find %s -ls" % data_path,
header="index files:")
def stop(self, nodes):
if self.keep_running:
self.logger.info("Keeping Docker container running.")
else:
self.logger.info("Stopping Docker container")
for node in nodes:
node.telemetry.detach_from_node(node, running=True)
process.run_subprocess_with_logging(_get_docker_compose_cmd(self.binary_paths[node.node_name], "down"))
node.telemetry.detach_from_node(node, running=False)
def detach_from_cluster(self, cluster):
data_paths = self.config.opts("provisioning",
"local.data.paths",
mandatory=False)
if data_paths is not None:
data_path = data_paths[0]
index_size_bytes = io.get_size(data_path)
self.metrics_store.put_count_cluster_level(
"final_index_size_bytes", index_size_bytes, "byte")
process.run_subprocess_with_logging("find %s -ls" % data_path,
header="index files:")
def _prepare_env(self, car, node_name, t):
env = {}
env.update(os.environ)
java_home = self.cfg.opts("runtime", "java8.home")
# Unix specific!:
self._set_env(env, "PATH", "%s/bin" % java_home, separator=":")
# Don't merge here!
env["JAVA_HOME"] = java_home
# we just blindly trust telemetry here...
for k, v in t.instrument_candidate_env(car, node_name).items():
self._set_env(env, k, v)
# probe if this JVM supports +ExitOnOutOfMemoryError
if process.run_subprocess_with_logging("%s/bin/java -XX:+ExitOnOutOfMemoryError -version" % java_home):
logger.info("JVM supports +ExitOnOutOfMemoryError. Setting this option to detect out of memory errors during the benchmark.")
java_opts = "-XX:+ExitOnOutOfMemoryError "
else:
logger.info("JVM does not support +ExitOnOutOfMemoryError. Cannot detect out of memory errors. Please consider a JDK upgrade.")
java_opts = ""
if car.heap:
java_opts += "-Xms%s -Xmx%s " % (car.heap, car.heap)
if car.java_opts:
java_opts += car.java_opts
if len(java_opts) > 0:
self._set_env(env, "ES_JAVA_OPTS", java_opts)
self._set_env(env, "ES_GC_OPTS", car.gc_opts)
logger.info("ENV: %s" % str(env))
return env
def unzip(zip_name, target_directory):
"""
Decompresses the provided archive to the target directory. The following file extensions are supported:
* zip: Relies that the 'unzip' tool is available on the path
* bz2: Can be uncompressed using standard library facilities, so no external tool is required.
The decompression method is chosen based on the file extension.
:param zip_name: The full path name to the file that should be decompressed.
:param target_directory: The directory to which files should be decompressed. May or may not exist prior to calling this function.
"""
filename, extension = splitext(zip_name)
if extension == ".zip":
if not process.run_subprocess_with_logging(
"unzip %s -d %s" % (zip_name, target_directory)):
raise RuntimeError("Could not unzip %s to %s" %
(zip_name, target_directory))
elif extension == ".bz2":
# We rather avoid external tools as much as possible to simplify Rally's setup, hence we use the library functions
target_file = os.path.join(target_directory, filename)
with open(target_file, "wb") as extracted, bz2.BZ2File(zip_name,
"rb") as file:
for data in iter(lambda: file.read(100 * 1024), b''):
extracted.write(data)
else:
raise RuntimeError(
"Unsupported file extension '%s'. Cannot unzip '%s'" %
(extension, zip_name))
def clone(src, remote):
io.ensure_dir(src)
# Don't swallow subprocess output, user might need to enter credentials...
if process.run_subprocess_with_logging("git clone %s %s" %
(remote, io.escape_path(src))):
raise exceptions.SupplyError("Could not clone from [%s] to [%s]" %
(remote, src))
def install(self, es_home_path, plugin_url=None):
installer_binary_path = os.path.join(es_home_path, "bin",
"elasticsearch-plugin")
if plugin_url:
logger.info("Installing [%s] into [%s] from [%s]" %
(self.plugin_name, es_home_path, plugin_url))
install_cmd = '%s install --batch "%s"' % (installer_binary_path,
plugin_url)
else:
logger.info("Installing [%s] into [%s]" %
(self.plugin_name, es_home_path))
install_cmd = '%s install --batch "%s"' % (installer_binary_path,
self.plugin_name)
return_code = process.run_subprocess_with_logging(install_cmd)
# see: https://www.elastic.co/guide/en/elasticsearch/plugins/current/_other_command_line_parameters.html
if return_code == 0:
logger.info("Successfully installed [%s]." % self.plugin_name)
elif return_code == 64:
# most likely this is an unknown plugin
raise exceptions.SystemSetupError("Unknown plugin [%s]" %
self.plugin_name)
elif return_code == 74:
raise exceptions.SupplyError(
"I/O error while trying to install [%s]" % self.plugin_name)
else:
raise exceptions.RallyError(
"Unknown error while trying to install [%s] (installer return code [%s]). Please check the logs."
% (self.plugin_name, str(return_code)))
def test_anonymous_proxy_no_connection(cfg, http_proxy, fresh_log_file):
env = dict(os.environ)
env["http_proxy"] = http_proxy.anonymous_url
assert process.run_subprocess_with_logging(it.esrally_command_line_for(cfg, "list tracks"), env=env) == 0
assert_log_line_present(fresh_log_file, f"Connecting via proxy URL [{http_proxy.anonymous_url}] to the Internet")
# unauthenticated proxy access is prevented
assert_log_line_present(fresh_log_file, "No Internet connection detected")
def pull_ts(src_dir, ts):
fetch(src_dir)
clean_src = io.escape_path(src_dir)
revision = process.run_subprocess_with_output(
"git -C {0} rev-list -n 1 --before=\"{1}\" --date=iso8601 origin/master".format(clean_src, ts))[0].strip()
if process.run_subprocess_with_logging("git -C {0} checkout {1}".format(clean_src, revision)):
raise exceptions.SupplyError("Could not checkout source tree for timestamped revision [%s]" % ts)
def _prepare_env(self, car, node_name, t):
env = {}
env.update(os.environ)
java_home = self.cfg.opts("runtime", "java8.home")
# Unix specific!:
self._set_env(env, "PATH", "%s/bin" % java_home, separator=":")
# Don't merge here!
env["JAVA_HOME"] = java_home
# we just blindly trust telemetry here...
for k, v in t.instrument_candidate_env(car, node_name).items():
self._set_env(env, k, v)
# probe if this JVM supports +ExitOnOutOfMemoryError
if process.run_subprocess_with_logging("%s/bin/java -XX:+ExitOnOutOfMemoryError -version" % java_home):
logger.info("JVM supports +ExitOnOutOfMemoryError. Setting this option to detect out of memory errors during the benchmark.")
java_opts = "-XX:+ExitOnOutOfMemoryError "
else:
logger.info("JVM does not support +ExitOnOutOfMemoryError. Cannot detect out of memory errors. Please consider a JDK upgrade.")
java_opts = ""
if car.heap:
java_opts += "-Xms%s -Xmx%s " % (car.heap, car.heap)
if car.java_opts:
java_opts += car.java_opts
if len(java_opts) > 0:
self._set_env(env, "ES_JAVA_OPTS", java_opts)
self._set_env(env, "ES_GC_OPTS", car.gc_opts)
logger.info("ENV: %s" % str(env))
return env
def test_authenticated_proxy_user_can_connect(cfg, http_proxy, fresh_log_file):
env = dict(os.environ)
env["http_proxy"] = http_proxy.authenticated_url
assert process.run_subprocess_with_logging(it.esrally_command_line_for(cfg, "list tracks"), env=env) == 0
assert_log_line_present(fresh_log_file, f"Connecting via proxy URL [{http_proxy.authenticated_url}] to the Internet")
# authenticated proxy access is allowed
assert_log_line_present(fresh_log_file, "Detected a working Internet connection")
def run_docker_compose_up(test_command):
env_variables = os.environ.copy()
env_variables["TEST_COMMAND"] = test_command
env_variables['RALLY_VERSION'] = version.__version__
return process.run_subprocess_with_logging(f"docker-compose -f {it.ROOT_DIR}/docker/docker-compose-tests.yml up "
f"--abort-on-container-exit", env=env_variables)
def supports_option(java_home, option):
"""
Detects support for a specific option (or combination of options) for the JVM version available in java_home.
:param java_home: The JAVA_HOME to use for probing.
:param option: The JVM option or combination of JVM options (separated by spaces) to check.
:return: True iff the provided ``option`` is supported on this JVM.
"""
return process.exit_status_as_bool(lambda: process.run_subprocess_with_logging("%s/bin/java %s -version" % (java_home, option)))
def supports_option(java_home, option):
"""
Detects support for a specific option (or combination of options) for the JVM version available in java_home.
:param java_home: The JAVA_HOME to use for probing.
:param option: The JVM option or combination of JVM options (separated by spaces) to check.
:return: True iff the provided ``option`` is supported on this JVM.
"""
return process.exit_status_as_bool(lambda: process.run_subprocess_with_logging("%s/bin/java %s -version" % (java_home, option)))
def probe(src, *args, **kwargs):
# Probe for -C
if not process.exit_status_as_bool(lambda: process.run_subprocess_with_logging("git -C %s --version" % src, level=logging.DEBUG)):
version = process.run_subprocess_with_output("git --version")
if version:
version = str(version).strip()
else:
version = "Unknown"
raise exceptions.SystemSetupError("Your git version is [%s] but Rally requires at least git 1.9. Please update git." % version)
return f(src, *args, **kwargs)
def probe(src, *args, **kwargs):
# Probe for -C
if not process.exit_status_as_bool(lambda: process.run_subprocess_with_logging(
"git -C {} --version".format(src), level=logging.DEBUG), quiet=True):
version = process.run_subprocess_with_output("git --version")
if version:
version = str(version).strip()
else:
version = "Unknown"
raise exceptions.SystemSetupError("Your git version is [%s] but Rally requires at least git 1.9. Please update git." % version)
return f(src, *args, **kwargs)
def _start_process(self, binary_path):
compose_cmd = _get_docker_compose_cmd(binary_path, "up -d")
ret = process.run_subprocess_with_logging(compose_cmd)
if ret != 0:
msg = "Docker daemon startup failed with exit code[{}]".format(ret)
logging.error(msg)
raise exceptions.LaunchError(msg)
container_id = _get_container_id(binary_path)
_wait_for_healthy_running_container(container_id)
def build_docker_image():
rally_version = version.__version__
env_variables = os.environ.copy()
env_variables['RALLY_VERSION'] = rally_version
env_variables['RALLY_LICENSE'] = get_license()
command = f"docker build -t elastic/rally:{rally_version} --build-arg RALLY_VERSION --build-arg RALLY_LICENSE " \
f"-f {ROOT_DIR}/docker/Dockerfiles/Dockerfile-dev {ROOT_DIR}"
if process.run_subprocess_with_logging(command, env=env_variables) != 0:
raise AssertionError("It was not possible to build the docker image from Dockerfile-dev")
def probe(src, *args, **kwargs):
# Probe for -C
if not process.run_subprocess_with_logging("git -C %s --version" % src,
level=logging.DEBUG):
version = process.run_subprocess_with_output("git --version")
if version:
version = str(version).strip()
else:
version = "Unknown"
raise exceptions.SystemSetupError(
"Your git version is [%s] but Rally requires at least git 1.9. Please update git."
% version)
return f(src, *args, **kwargs)
def _start_process(binary_path, env):
if os.geteuid() == 0:
raise exceptions.LaunchError("Cannot launch Elasticsearch as root. Please run Rally as a non-root user.")
os.chdir(binary_path)
cmd = ["bin/elasticsearch"]
cmd.extend(["-d", "-p", "pid"])
ret = process.run_subprocess_with_logging(command_line=" ".join(cmd), env=env)
if ret != 0:
msg = "Daemon startup failed with exit code[{}]".format(ret)
logging.error(msg)
raise exceptions.LaunchError(msg)
return wait_for_pidfile("./pid")
def create_keystore(install_root, keystore_binary, env):
logger = logging.getLogger(LOGGER_NAME)
keystore_create_command = "{keystore} -s create".format(
keystore=keystore_binary)
return_code = process.run_subprocess_with_logging(keystore_create_command,
env=env)
if return_code != 0:
logger.error("%s has exited with code [%d]", keystore_create_command,
return_code)
raise exceptions.SystemSetupError(
"Could not initialize a keystore. Please see the log for details.")
def install_certificates(config_names, variables, **kwargs):
if "security" not in config_names:
return False
node_name = variables["node_name"]
node_ip = variables["node_ip"]
install_root = variables["install_root_path"]
x_pack_config_path = os.path.join(install_root, "config", "x-pack")
logger.info("Installing x-pack certificates for node [%s]." % node_name)
# 0. Create instances.yml for the current ES node.
instances_yml = os.path.join(tempfile.mkdtemp(), "instances.yml")
with open(instances_yml, "w") as f:
f.write(
instances_yml_template.format(node_name=node_name,
node_ip=node_ip))
# 1. Create certificate if needed. We will prebundle the CA with Rally and generate instance certificates based on this CA.
cert_gen = os.path.join(install_root, "bin", "x-pack", "certgen")
cert_bundle = os.path.join(install_root, "config", "x-pack",
"node-cert.zip")
# ./bin/x-pack/certgen
# -cert=/Users/daniel/.rally/benchmarks/distributions/elasticsearch-5.5.0/config/x-pack/ca/ca.crt
# -key=/Users/daniel/.rally/benchmarks/distributions/elasticsearch-5.5.0/config/x-pack/ca/ca.key
# -in=/Users/daniel/.rally/benchmarks/distributions/elasticsearch-5.5.0/config/instances.yml
# -out=/Users/daniel/.rally/benchmarks/distributions/elasticsearch-5.5.0/config/x-pack/node-cert.zip
return_code = process.run_subprocess_with_logging(
'{cert_gen} -cert="{config_path}/ca/ca.crt" -key="{config_path}/ca/ca.key" -in="{instances_yml}" -out="{cert_bundle}"'
.format(cert_gen=cert_gen,
config_path=x_pack_config_path,
instances_yml=instances_yml,
cert_bundle=cert_bundle))
if return_code != 0:
logger.error("certgen has exited with code [%d]" % return_code)
raise exceptions.SystemSetupError(
"Could not create x-pack certificate bundle for node [%s]. Please see the log for details."
% node_name)
# 2. Unzip /Users/daniel/.rally/benchmarks/distributions/elasticsearch-5.5.0/config/x-pack/node-cert.zip
io.decompress(cert_bundle, x_pack_config_path)
# Success
return True
def _do_download_via_s3(self, url, data_set_path, size_in_bytes):
tmp_data_set_path = data_set_path + ".tmp"
s3cmd = "s3cmd -v get %s %s" % (url, tmp_data_set_path)
try:
success = process.run_subprocess_with_logging(s3cmd)
# Exit code for s3cmd does not seem to be reliable so we also check the file size although this is rather fragile...
if not success or (size_in_bytes is not None and os.path.getsize(tmp_data_set_path) != size_in_bytes):
# cleanup probably corrupt data file...
if os.path.isfile(tmp_data_set_path):
os.remove(tmp_data_set_path)
raise RuntimeError("Could not get benchmark data from S3: '%s'. Is s3cmd installed and set up properly?" % s3cmd)
except:
logger.info("Removing temp file %s" % tmp_data_set_path)
os.remove(tmp_data_set_path)
raise
else:
os.rename(tmp_data_set_path, data_set_path)
def unzip(zip_name, target_directory):
"""
Decompresses the provided archive to the target directory. The following file extensions are supported:
* zip: Relies that the 'unzip' tool is available on the path
* bz2: Can be uncompressed using standard library facilities, so no external tool is required.
* gz: Can be uncompressed using standard library facilities, so no external tool is required.
* tar: Can be uncompressed using standard library facilities, so no external tool is required.
* tar.gz Can be uncompressed using standard library facilities, so no external tool is required.
* tgz Can be uncompressed using standard library facilities, so no external tool is required.
* tar.bz2 Can be uncompressed using standard library facilities, so no external tool is required.
Did not implement LZMA because LZMAFile is not thread-safe.
The decompression method is chosen based on the file extension.
:param zip_name: The full path name to the file that should be decompressed.
:param target_directory: The directory to which files should be decompressed. May or may not exist prior to calling
this function.
"""
filename, extension = splitext(zip_name)
if extension == ".zip":
if not process.run_subprocess_with_logging(
"unzip %s -d %s" % (zip_name, target_directory)):
raise RuntimeError("Could not unzip %s to %s" %
(zip_name, target_directory))
elif extension == ".bz2":
_do_unzip(target_directory, filename, bz2.BZ2File(zip_name, "rb"))
elif extension == ".gz":
_do_unzip(target_directory, filename, gzip.GzipFile(zip_name, "rb"))
elif extension == ".tar":
_do_unzip(target_directory, filename, tarfile.TarFile(zip_name, "rb"))
elif extension == ".tar.gz":
_do_unzip(target_directory, filename,
tarfile.TarFile(zip_name, "r:gz"))
elif extension == ".tgz":
_do_unzip(target_directory, filename,
tarfile.TarFile(zip_name, "r:gz"))
elif extension == ".tar.bz2":
_do_unzip(target_directory, filename,
tarfile.TarFile(zip_name, "r:bz2"))
else:
raise RuntimeError(
"Unsupported file extension '%s'. Cannot unzip '%s'" %
(extension, zip_name))
def download_via_s3(url, data_set_path, size_in_bytes):
tmp_data_set_path = data_set_path + ".tmp"
s3cmd = "s3cmd -v get %s %s" % (url, tmp_data_set_path)
try:
success = process.run_subprocess_with_logging(s3cmd)
# Exit code for s3cmd does not seem to be reliable so we also check the file size although this is rather fragile...
if not success or (size_in_bytes is not None and os.path.getsize(tmp_data_set_path) != size_in_bytes):
# cleanup probably corrupt data file...
if os.path.isfile(tmp_data_set_path):
os.remove(tmp_data_set_path)
raise exceptions.SystemSetupError(
"Could not get benchmark data from S3: '%s'. Is s3cmd installed and set up properly?" % s3cmd)
except:
os.remove(tmp_data_set_path)
raise
else:
os.rename(tmp_data_set_path, data_set_path)
def add_property_to_keystore(keystore_binary, client_name, property_name,
property_value, env):
logger = logging.getLogger(LOGGER_NAME)
p1 = subprocess.Popen(["echo", property_value], stdout=subprocess.PIPE)
keystore_command = "{keystore} --silent add --stdin azure.client.{client_name}.{key}".format(
keystore=keystore_binary, client_name=client_name, key=property_name)
return_code = process.run_subprocess_with_logging(keystore_command,
stdin=p1.stdout,
env=env)
if return_code != 0:
logger.error("%s has exited with code [%d]", keystore_command,
return_code)
raise exceptions.SystemSetupError(
"Could not add Azure keystore secure setting [{}]. Please see the log for details."
.format(property_name))
def install_certificates(config_names, variables, **kwargs):
if "x-pack-security" not in config_names:
return False
logger = logging.getLogger(LOGGER_NAME)
cert_binary = "elasticsearch-certutil"
node_name = variables["node_name"]
node_ip = variables["node_ip"]
install_root = variables["install_root_path"]
bundled_ca_path = os.path.join(os.path.dirname(__file__), "ca")
x_pack_config_path = os.path.join(install_root, "config", "x-pack")
logger.info("Installing certificates for node [%s].", node_name)
instances_yml = os.path.join(tempfile.mkdtemp(), "instances.yml")
with open(instances_yml, "w") as f:
f.write(
instances_yml_template.format(node_name=node_name,
node_ip=node_ip))
# Generate instance certificates based on a CA that is pre-bundled with Rally
certutil = resolve_binary(install_root, cert_binary)
cert_bundle = os.path.join(install_root, "node-cert.zip")
return_code = process.run_subprocess_with_logging(
'{certutil} cert --silent --in "{instances_yml}" --out="{cert_bundle}" --ca-cert="{ca_path}/ca.crt" '
'--ca-key="{ca_path}/ca.key" --pass ""'.format(
certutil=certutil,
ca_path=bundled_ca_path,
instances_yml=instances_yml,
cert_bundle=cert_bundle),
env=kwargs.get("env"))
if return_code != 0:
logger.error("%s has exited with code [%d]", cert_binary, return_code)
raise exceptions.SystemSetupError(
"Could not create certificate bundle for node [{}]. Please see the log for details."
.format(node_name))
io.decompress(cert_bundle, x_pack_config_path)
# Success
return True
def install(self, es_home_path, plugin_url=None):
installer_binary_path = os.path.join(es_home_path, "bin", "elasticsearch-plugin")
if plugin_url:
logger.info("Installing [%s] into [%s] from [%s]" % (self.plugin_name, es_home_path, plugin_url))
install_cmd = '%s install --batch "%s"' % (installer_binary_path, plugin_url)
else:
logger.info("Installing [%s] into [%s]" % (self.plugin_name, es_home_path))
install_cmd = '%s install --batch "%s"' % (installer_binary_path, self.plugin_name)
return_code = process.run_subprocess_with_logging(install_cmd)
# see: https://www.elastic.co/guide/en/elasticsearch/plugins/current/_other_command_line_parameters.html
if return_code == 0:
logger.info("Successfully installed [%s]." % self.plugin_name)
elif return_code == 64:
# most likely this is an unknown plugin
raise exceptions.SystemSetupError("Unknown plugin [%s]" % self.plugin_name)
elif return_code == 74:
raise exceptions.SupplyError("I/O error while trying to install [%s]" % self.plugin_name)
else:
raise exceptions.RallyError("Unknown error while trying to install [%s] (installer return code [%s]). Please check the logs." %
(self.plugin_name, str(return_code)))
def configure_keystore(config_names, variables, **kwargs):
logger = logging.getLogger(LOGGER_NAME)
keystore_params = ["gcs_client_name", "gcs_credentials_file"]
client_name = variables.get(keystore_params[0])
credentials_file = variables.get(keystore_params[1])
if not (credentials_file and client_name):
logger.warning(
"Skipping keystore configuration for repository-gcs as plugin-params %s were not supplied",
keystore_params)
return False
keystore_binary_filename = "elasticsearch-keystore"
install_root = variables["install_root_path"]
keystore_binary = resolve_binary(install_root, keystore_binary_filename)
env = kwargs.get("env")
if not os.path.isfile(resolve_keystore_config(install_root)):
create_keystore(install_root, keystore_binary, env)
keystore_command = "{keystore} --silent add-file gcs.client.{client_name}.credentials_file {credentials_file}".format(
keystore=keystore_binary,
client_name=client_name,
credentials_file=credentials_file)
return_code = process.run_subprocess_with_logging(keystore_command,
env=env)
if return_code != 0:
logger.error("%s has exited with code [%d]", keystore_command,
return_code)
raise exceptions.SystemSetupError(
"Could not add GCS keystore secure setting. Please see the log for details."
)
# Success
return True
def unzip(zip_name, target_directory):
"""
Decompresses the provided archive to the target directory. The following file extensions are supported:
* zip: Relies that the 'unzip' tool is available on the path
* bz2: Can be uncompressed using standard library facilities, so no external tool is required.
The decompression method is chosen based on the file extension.
:param zip_name: The full path name to the file that should be decompressed.
:param target_directory: The directory to which files should be decompressed. May or may not exist prior to calling this function.
"""
filename, extension = splitext(zip_name)
if extension == ".zip":
if not process.run_subprocess_with_logging("unzip %s -d %s" % (zip_name, target_directory)):
raise RuntimeError("Could not unzip %s to %s" % (zip_name, target_directory))
elif extension == ".bz2":
# We rather avoid external tools as much as possible to simplify Rally's setup, hence we use the library functions
target_file = os.path.join(target_directory, filename)
with open(target_file, "wb") as extracted, bz2.BZ2File(zip_name, "rb") as file:
for data in iter(lambda: file.read(100 * 1024), b''):
extracted.write(data)
else:
raise RuntimeError("Unsupported file extension '%s'. Cannot unzip '%s'" % (extension, zip_name))
def stop(self, cluster):
logger.info("Stopping Docker container")
docker_cfg_path = self._docker_cfg_path()
process.run_subprocess_with_logging("docker-compose down -v -f %s" % docker_cfg_path)
