def create(cfg, metrics_store, node_ip, node_http_port, all_node_ips, all_node_ids, sources=False, distribution=False,
external=False, docker=False):
race_root_path = paths.race_root(cfg)
node_ids = cfg.opts("provisioning", "node.ids", mandatory=False)
node_name_prefix = cfg.opts("provisioning", "node.name.prefix")
car, plugins = load_team(cfg, external)
if sources or distribution:
s = supplier.create(cfg, sources, distribution, car, plugins)
p = []
all_node_names = ["%s-%s" % (node_name_prefix, n) for n in all_node_ids]
for node_id in node_ids:
node_name = "%s-%s" % (node_name_prefix, node_id)
p.append(
provisioner.local(cfg, car, plugins, node_ip, node_http_port, all_node_ips,
all_node_names, race_root_path, node_name))
l = launcher.ProcessLauncher(cfg)
elif external:
raise exceptions.RallyAssertionError("Externally provisioned clusters should not need to be managed by Rally's mechanic")
elif docker:
if len(plugins) > 0:
raise exceptions.SystemSetupError("You cannot specify any plugins for Docker clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
s = lambda: None
p = []
for node_id in node_ids:
node_name = "%s-%s" % (node_name_prefix, node_id)
p.append(provisioner.docker(cfg, car, node_ip, node_http_port, race_root_path, node_name))
l = launcher.DockerLauncher(cfg)
else:
# It is a programmer error (and not a user error) if this function is called with wrong parameters
raise RuntimeError("One of sources, distribution, docker or external must be True")
return Mechanic(cfg, metrics_store, s, p, l)
def test_starts_container_successfully(self, run_subprocess_with_output,
run_subprocess_with_logging):
run_subprocess_with_logging.return_value = 0
# Docker container id (from docker-compose ps), Docker container id (from docker ps --filter ...)
run_subprocess_with_output.side_effect = [["de604d0d"], ["de604d0d"]]
cfg = config.Config()
docker = launcher.DockerLauncher(cfg)
node_config = NodeConfiguration(build_type="docker",
car_env={},
car_runtime_jdks="12,11",
ip="127.0.0.1",
node_name="testnode",
node_root_path="/tmp",
binary_path="/bin",
data_paths="/tmp")
nodes = docker.start([node_config])
self.assertEqual(1, len(nodes))
node = nodes[0]
self.assertEqual(0, node.pid)
self.assertEqual("/bin", node.binary_path)
self.assertEqual("127.0.0.1", node.host_name)
self.assertEqual("testnode", node.node_name)
self.assertIsNotNone(node.telemetry)
run_subprocess_with_logging.assert_called_once_with(
"docker-compose -f /bin/docker-compose.yml up -d")
run_subprocess_with_output.assert_has_calls([
mock.call("docker-compose -f /bin/docker-compose.yml ps -q"),
mock.call(
'docker ps -a --filter "id=de604d0d" --filter "status=running" --filter "health=healthy" -q'
)
])
def test_container_not_started(self, run_subprocess_with_output,
run_subprocess_with_logging, sleep):
run_subprocess_with_logging.return_value = 0
# Docker container id (from docker-compose ps), but NO Docker container id (from docker ps --filter...) twice
run_subprocess_with_output.side_effect = [["de604d0d"], [], []]
cfg = config.Config()
# ensure we only check the status two times
stop_watch = DockerLauncherTests.IterationBasedStopWatch(
max_iterations=2)
docker = launcher.DockerLauncher(
cfg, clock=DockerLauncherTests.TestClock(stop_watch=stop_watch))
node_config = NodeConfiguration(build_type="docker",
car_env={},
car_runtime_jdks="12,11",
ip="127.0.0.1",
node_name="testnode",
node_root_path="/tmp",
binary_path="/bin",
data_paths="/tmp")
with self.assertRaisesRegex(
exceptions.LaunchError,
"No healthy running container after 600 seconds!"):
docker.start([node_config])
def create(cfg,
metrics_store,
single_machine=True,
sources=False,
build=False,
distribution=False,
external=False,
docker=False):
challenge_root_path = paths.race_root(cfg)
install_dir = "%s/install" % challenge_root_path
log_dir = "%s/logs" % challenge_root_path
io.ensure_dir(log_dir)
if sources:
try:
src_dir = cfg.opts("source", "local.src.dir")
except config.ConfigError:
logger.exception("Cannot determine source directory")
raise exceptions.SystemSetupError(
"You cannot benchmark Elasticsearch from sources. Are you missing Gradle 2.13? Please install"
" all prerequisites and reconfigure Rally with %s configure" %
PROGRAM_NAME)
remote_url = cfg.opts("source", "remote.repo.url")
revision = cfg.opts("mechanic", "source.revision")
gradle = cfg.opts("build", "gradle.bin")
java_home = cfg.opts("runtime", "java8.home")
s = lambda: supplier.from_sources(remote_url, src_dir, revision,
gradle, java_home, log_dir, build)
p = provisioner.local_provisioner(cfg, install_dir, single_machine)
l = launcher.InProcessLauncher(cfg, metrics_store, challenge_root_path,
log_dir)
elif distribution:
version = cfg.opts("mechanic", "distribution.version")
repo_name = cfg.opts("mechanic", "distribution.repository")
distributions_root = "%s/%s" % (cfg.opts(
"node", "root.dir"), cfg.opts("source", "distribution.dir"))
s = lambda: supplier.from_distribution(version=version,
repo_name=repo_name,
distributions_root=
distributions_root)
p = provisioner.local_provisioner(cfg, install_dir, single_machine)
l = launcher.InProcessLauncher(cfg, metrics_store, challenge_root_path,
log_dir)
elif external:
s = lambda: None
p = provisioner.no_op_provisioner(cfg)
l = launcher.ExternalLauncher(cfg, metrics_store)
elif docker:
s = lambda: None
p = provisioner.docker_provisioner(cfg, install_dir)
l = launcher.DockerLauncher(cfg, metrics_store)
else:
# It is a programmer error (and not a user error) if this function is called with wrong parameters
raise RuntimeError(
"One of sources, distribution, docker or external must be True")
return Mechanic(s, p, l)
def create(cfg,
metrics_store,
sources=False,
build=False,
distribution=False,
external=False,
docker=False):
if sources:
s = lambda: supplier.from_sources(cfg, build)
p = provisioner.local_provisioner(cfg)
l = launcher.InProcessLauncher(cfg, metrics_store)
elif distribution:
s = lambda: supplier.from_distribution(cfg)
p = provisioner.local_provisioner(cfg)
l = launcher.InProcessLauncher(cfg, metrics_store)
elif external:
s = lambda: None
p = provisioner.no_op_provisioner(cfg)
l = launcher.ExternalLauncher(cfg, metrics_store)
elif docker:
s = lambda: None
p = provisioner.no_op_provisioner(cfg)
l = launcher.DockerLauncher(cfg, metrics_store)
else:
# It is a programmer error (and not a user error) if this function is called with wrong parameters
raise RuntimeError(
"One of sources, distribution, docker or external must be True")
return Mechanic(cfg, s, p, l)
def create(cfg,
metrics_store,
all_node_ips,
cluster_settings=None,
sources=False,
build=False,
distribution=False,
external=False,
docker=False):
races_root = paths.races_root(cfg)
challenge_root_path = paths.race_root(cfg)
node_ids = cfg.opts("provisioning", "node.ids", mandatory=False)
car, plugins = load_team(cfg, external)
if sources or distribution:
s = supplier.create(cfg, sources, distribution, build,
challenge_root_path, car, plugins)
p = []
for node_id in node_ids:
p.append(
provisioner.local_provisioner(cfg, car, plugins,
cluster_settings, all_node_ips,
challenge_root_path, node_id))
l = launcher.InProcessLauncher(cfg, metrics_store, races_root)
elif external:
if cluster_settings:
logging.getLogger(__name__).warning(
"Cannot apply challenge-specific cluster settings [%s] for an externally provisioned cluster. Please ensure that the cluster "
"settings are present or the benchmark may fail or behave unexpectedly."
% cluster_settings)
if len(plugins) > 0:
raise exceptions.SystemSetupError(
"You cannot specify any plugins for externally provisioned clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
s = lambda: None
p = [provisioner.no_op_provisioner()]
l = launcher.ExternalLauncher(cfg, metrics_store)
elif docker:
if len(plugins) > 0:
raise exceptions.SystemSetupError(
"You cannot specify any plugins for Docker clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
s = lambda: None
p = []
for node_id in node_ids:
p.append(
provisioner.docker_provisioner(cfg, car, cluster_settings,
challenge_root_path, node_id))
l = launcher.DockerLauncher(cfg, metrics_store)
else:
# It is a programmer error (and not a user error) if this function is called with wrong parameters
raise RuntimeError(
"One of sources, distribution, docker or external must be True")
return Mechanic(s, p, l)
def test_stops_container_successfully(self, run_subprocess_with_logging, add_metadata_for_node):
cfg = config.Config()
metrics_store = None
docker = launcher.DockerLauncher(cfg)
nodes = [cluster.Node(0, "/bin", "127.0.0.1", "testnode", telemetry.Telemetry())]
docker.stop(nodes, metrics_store=metrics_store)
add_metadata_for_node.assert_called_once_with(metrics_store, "testnode", "127.0.0.1")
run_subprocess_with_logging.assert_called_once_with("docker-compose -f /bin/docker-compose.yml down")
def stop(cfg):
root_path = paths.install_root(cfg)
node_config = provisioner.load_node_configuration(root_path)
if node_config.build_type == "tar":
node_launcher = launcher.ProcessLauncher(cfg)
elif node_config.build_type == "docker":
node_launcher = launcher.DockerLauncher(cfg)
else:
raise exceptions.SystemSetupError("Unknown build type [{}]".format(
node_config.build_type))
nodes, race_id = _load_node_file(root_path)
cls = metrics.metrics_store_class(cfg)
metrics_store = cls(cfg)
race_store = metrics.race_store(cfg)
try:
current_race = race_store.find_by_race_id(race_id)
except exceptions.NotFound:
logging.getLogger(__name__).info(
"Could not find race [%s] most likely because an in-memory metrics store is "
"used across multiple machines. Use an Elasticsearch metrics store to persist "
"results.", race_id)
# we are assuming here that we use an Elasticsearch metrics store... . If we use a file race store (across
# multiple machines) we will not be able to retrieve a race. In that case we open our in-memory metrics store
# with settings derived from startup parameters (because we can't store system metrics persistently anyway).
current_race = metrics.create_race(cfg, track=None, challenge=None)
metrics_store.open(race_id=current_race.race_id,
race_timestamp=current_race.race_timestamp,
track_name=current_race.track_name,
challenge_name=current_race.challenge_name)
node_launcher.stop(nodes, metrics_store)
_delete_node_file(root_path)
metrics_store.flush(refresh=True)
for node in nodes:
results = metrics.calculate_system_results(metrics_store,
node.node_name)
current_race.add_results(results)
metrics.results_store(cfg).store_results(current_race)
metrics_store.close()
# TODO: Do we need to expose this as a separate command as well?
provisioner.cleanup(preserve=cfg.opts("mechanic", "preserve.install"),
install_dir=node_config.binary_path,
data_paths=node_config.data_paths)
def stop(cfg):
root_path = paths.install_root(cfg)
node_config = provisioner.load_node_configuration(root_path)
if node_config.build_type == "tar":
node_launcher = launcher.ProcessLauncher(cfg)
elif node_config.build_type == "docker":
node_launcher = launcher.DockerLauncher(cfg)
else:
raise exceptions.SystemSetupError("Unknown build type [{}]".format(
node_config.build_type))
nodes, race_id = _load_node_file(root_path)
cls = metrics.metrics_store_class(cfg)
metrics_store = cls(cfg)
race_store = metrics.race_store(cfg)
try:
current_race = race_store.find_by_race_id(race_id)
metrics_store.open(race_id=current_race.race_id,
race_timestamp=current_race.race_timestamp,
track_name=current_race.track_name,
challenge_name=current_race.challenge_name)
except exceptions.NotFound:
logging.getLogger(__name__).info(
"Could not find race [%s] and will thus not persist system metrics.",
race_id)
# Don't persist system metrics if we can't retrieve the race as we cannot derive the required meta-data.
current_race = None
metrics_store = None
node_launcher.stop(nodes, metrics_store)
_delete_node_file(root_path)
if current_race:
metrics_store.flush(refresh=True)
for node in nodes:
results = metrics.calculate_system_results(metrics_store,
node.node_name)
current_race.add_results(results)
metrics.results_store(cfg).store_results(current_race)
metrics_store.close()
# TODO: Do we need to expose this as a separate command as well?
provisioner.cleanup(preserve=cfg.opts("mechanic", "preserve.install"),
install_dir=node_config.binary_path,
data_paths=node_config.data_paths)
def start(cfg):
root_path = paths.install_root(cfg)
race_id = cfg.opts("system", "race.id")
# avoid double-launching - we expect that the node file is absent
with contextlib.suppress(FileNotFoundError):
_load_node_file(root_path)
install_id = cfg.opts("system", "install.id")
raise exceptions.SystemSetupError("A node with this installation id is already running. Please stop it first "
"with {} stop --installation-id={}".format(PROGRAM_NAME, install_id))
node_config = provisioner.load_node_configuration(root_path)
if node_config.build_type == "tar":
node_launcher = launcher.ProcessLauncher(cfg)
elif node_config.build_type == "docker":
node_launcher = launcher.DockerLauncher(cfg)
else:
raise exceptions.SystemSetupError("Unknown build type [{}]".format(node_config.build_type))
nodes = node_launcher.start([node_config])
_store_node_file(root_path, (nodes, race_id))
def create(cfg,
metrics_store,
all_node_ips,
cluster_settings=None,
sources=False,
build=False,
distribution=False,
external=False,
docker=False):
races_root = paths.races_root(cfg)
challenge_root_path = paths.race_root(cfg)
node_ids = cfg.opts("provisioning", "node.ids", mandatory=False)
repo = team.team_repo(cfg)
# externally provisioned clusters do not support cars / plugins
if external:
car = None
plugins = []
else:
car = team.load_car(repo, cfg.opts("mechanic", "car.names"))
plugins = team.load_plugins(repo, cfg.opts("mechanic", "car.plugins"))
if sources:
try:
src_dir = cfg.opts("node", "src.root.dir")
except config.ConfigError:
logger.exception("Cannot determine source directory")
raise exceptions.SystemSetupError(
"You cannot benchmark Elasticsearch from sources. Did you install Gradle? Please install"
" all prerequisites and reconfigure Rally with %s configure" %
PROGRAM_NAME)
remote_url = cfg.opts("source", "remote.repo.url")
revision = cfg.opts("mechanic", "source.revision")
gradle = cfg.opts("build", "gradle.bin")
java_home = cfg.opts("runtime", "java.home")
src_config = cfg.all_opts("source")
s = lambda: supplier.from_sources(
remote_url, src_dir, revision, gradle, java_home,
challenge_root_path, plugins, src_config, build)
p = []
for node_id in node_ids:
p.append(
provisioner.local_provisioner(cfg, car, plugins,
cluster_settings, all_node_ips,
challenge_root_path, node_id))
l = launcher.InProcessLauncher(cfg, metrics_store, races_root)
elif distribution:
version = cfg.opts("mechanic", "distribution.version")
repo_name = cfg.opts("mechanic", "distribution.repository")
distributions_root = "%s/%s" % (cfg.opts(
"node", "root.dir"), cfg.opts("source", "distribution.dir"))
distribution_cfg = cfg.all_opts("distributions")
s = lambda: supplier.from_distribution(
version=version,
repo_name=repo_name,
distribution_config=distribution_cfg,
distributions_root=distributions_root,
plugins=plugins)
p = []
for node_id in node_ids:
p.append(
provisioner.local_provisioner(cfg, car, plugins,
cluster_settings, all_node_ips,
challenge_root_path, node_id))
l = launcher.InProcessLauncher(cfg, metrics_store, races_root)
elif external:
if cluster_settings:
logger.warning(
"Cannot apply challenge-specific cluster settings [%s] for an externally provisioned cluster. Please ensure "
"that the cluster settings are present or the benchmark may fail or behave unexpectedly."
% cluster_settings)
if len(plugins) > 0:
raise exceptions.SystemSetupError(
"You cannot specify any plugins for externally provisioned clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
s = lambda: None
p = [provisioner.no_op_provisioner()]
l = launcher.ExternalLauncher(cfg, metrics_store)
elif docker:
if len(plugins) > 0:
raise exceptions.SystemSetupError(
"You cannot specify any plugins for Docker clusters. Please remove "
"\"--elasticsearch-plugins\" and try again.")
s = lambda: None
p = []
for node_id in node_ids:
p.append(
provisioner.docker_provisioner(cfg, car, cluster_settings,
challenge_root_path, node_id))
l = launcher.DockerLauncher(cfg, metrics_store)
else:
# It is a programmer error (and not a user error) if this function is called with wrong parameters
raise RuntimeError(
"One of sources, distribution, docker or external must be True")
return Mechanic(s, p, l)