def test_prepare_default_data_paths(self, mock_rm, mock_ensure_dir, mock_decompress):
installer = provisioner.ElasticsearchInstaller(car=team.Car(names="defaults",
root_path=None,
config_paths="/tmp"),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
http_port=9200,
node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest")
installer.install("/data/builds/distributions")
self.assertEqual(installer.es_home_path, "/install/elasticsearch-5.0.0-SNAPSHOT")
self.assertEqual({
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"data_paths": ["/install/elasticsearch-5.0.0-SNAPSHOT/data"],
"log_path": HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server",
"heap_dump_path": HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump",
"node_ip": "10.17.22.23",
"network_host": "10.17.22.23",
"http_port": "9200-9300",
"transport_port": "9300-9400",
"all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]",
"all_node_names": "[\"rally-node-0\",\"rally-node-1\"]",
"minimum_master_nodes": 2,
"install_root_path": "/install/elasticsearch-5.0.0-SNAPSHOT"
}, installer.variables)
self.assertEqual(installer.data_paths, ["/install/elasticsearch-5.0.0-SNAPSHOT/data"])
def test_env_options_order(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "keep.running", False)
cfg.add(config.Scope.application, "system", "env.name", "test")
ms = get_metrics_store(cfg)
proc_launcher = launcher.ProcessLauncher(cfg,
ms,
races_root_dir="/home")
default_car = team.Car(names="default-car",
root_path=None,
config_paths=["/tmp/rally-config"])
node_telemetry = [
telemetry.FlightRecorder(telemetry_params={},
log_root="/tmp/telemetry",
java_major_version=8)
]
t = telemetry.Telemetry(["jfr"], devices=node_telemetry)
env = proc_launcher._prepare_env(car=default_car,
node_name="node0",
java_home="/java_home",
t=t)
self.assertEqual("/java_home/bin" + os.pathsep + os.environ["PATH"],
env["PATH"])
self.assertEqual(
"-XX:+ExitOnOutOfMemoryError -XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints "
"-XX:+UnlockCommercialFeatures -XX:+FlightRecorder "
"-XX:FlightRecorderOptions=disk=true,maxage=0s,maxsize=0,dumponexit=true,dumponexitpath=/tmp/telemetry/default-car-node0.jfr "
"-XX:StartFlightRecording=defaultrecording=true",
env["ES_JAVA_OPTS"])
def test_invokes_hook_no_java_home(self):
installer = provisioner.ElasticsearchInstaller(
car=team.Car(names="defaults",
root_path="/tmp",
config_paths="/tmp/templates",
variables={"data_paths": "/tmp/some/data-path-dir"}),
java_home=None,
node_name="rally-node-0",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
http_port=9200,
node_root_dir="~/.rally/benchmarks/races/unittest",
hook_handler_class=NoopHookHandler)
self.assertEqual(0, len(installer.hook_handler.hook_calls))
installer.invoke_install_hook(team.BootstrapPhase.post_install,
{"foo": "bar"})
self.assertEqual(1, len(installer.hook_handler.hook_calls))
self.assertEqual(
{"foo": "bar"},
installer.hook_handler.hook_calls["post_install"]["variables"])
self.assertEqual(
{"env": {}},
installer.hook_handler.hook_calls["post_install"]["kwargs"])
def test_invokes_hook_with_java_home(self):
installer = provisioner.ElasticsearchInstaller(
car=team.Car(
names="defaults",
root_path="/tmp",
config_paths="/tmp/templates",
variables={
"data_paths": "/tmp/some/data-path-dir",
},
),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
http_port=9200,
node_root_dir="~/.rally/benchmarks/races/unittest",
hook_handler_class=NoopHookHandler,
)
assert len(installer.hook_handler.hook_calls) == 0
installer.invoke_install_hook(team.BootstrapPhase.post_install,
{"foo": "bar"})
assert len(installer.hook_handler.hook_calls) == 1
assert installer.hook_handler.hook_calls["post_install"][
"variables"] == {
"foo": "bar"
}
assert installer.hook_handler.hook_calls["post_install"]["kwargs"] == {
"env": {
"JAVA_HOME": "/usr/local/javas/java8"
}
}
def test_prepare_user_provided_data_path(self, mock_rm, mock_ensure_dir,
mock_decompress):
installer = provisioner.ElasticsearchInstaller(
car=team.Car(name="defaults",
config_paths="/tmp",
variables={"data_paths": "/tmp/some/data-path-dir"}),
node_name="rally-node-0",
all_node_ips=["10.17.22.22", "10.17.22.23"],
ip="10.17.22.23",
http_port=9200,
node_root_dir="~/.rally/benchmarks/races/unittest")
installer.install("/data/builds/distributions")
self.assertEqual(installer.es_home_path,
"/install/elasticsearch-5.0.0-SNAPSHOT")
self.assertEqual(
{
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"data_paths": ["/tmp/some/data-path-dir"],
"log_path": "~/.rally/benchmarks/races/unittest/logs/server",
"node_ip": "10.17.22.23",
"network_host": "10.17.22.23",
"http_port": "9200-9300",
"transport_port": "9300-9400",
"all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]",
"minimum_master_nodes": 2,
"node_count_per_host": 1,
"install_root_path": "/install/elasticsearch-5.0.0-SNAPSHOT"
}, installer.variables)
self.assertEqual(installer.data_paths, ["/tmp/some/data-path-dir"])
def test_provisioning_with_variables(self, uuid4):
uuid4.return_value = "86f42ae0-5840-4b5b-918d-41e7907cb644"
node_root_dir = tempfile.gettempdir()
log_dir = os.path.join(node_root_dir, "logs", "server")
heap_dump_dir = os.path.join(node_root_dir, "heapdump")
data_dir = os.path.join(node_root_dir, "data",
"86f42ae0-5840-4b5b-918d-41e7907cb644")
rally_root = os.path.normpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
os.pardir, os.pardir, "esrally"))
c = team.Car("unit-test-car",
None,
"/tmp",
variables={
"docker_image":
"docker.elastic.co/elasticsearch/elasticsearch",
"docker_mem_limit": "256m",
"docker_cpu_count": 2
})
docker = provisioner.DockerProvisioner(car=c,
node_name="rally-node-0",
ip="10.17.22.33",
http_port=39200,
node_root_dir=node_root_dir,
distribution_version="6.3.0",
rally_root=rally_root)
docker_cfg = docker._render_template_from_file(
docker.docker_vars(mounts={}))
self.assertEqual(
"""version: '2.2'
services:
elasticsearch1:
cap_add:
- IPC_LOCK
image: "docker.elastic.co/elasticsearch/elasticsearch:6.3.0"
labels:
io.rally.description: "elasticsearch-rally"
cpu_count: 2
mem_limit: 256m
ports:
- 39200:39200
- 9300
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- %s:/usr/share/elasticsearch/data
- %s:/var/log/elasticsearch
- %s:/usr/share/elasticsearch/heapdump
healthcheck:
test: nc -z 127.0.0.1 39200
interval: 5s
timeout: 2s
retries: 10""" % (data_dir, log_dir, heap_dump_dir), docker_cfg)
def test_create_suppliers_for_es_only_config(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "distribution.version",
"6.0.0")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision",
"current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "runtime", "java10.home",
"/usr/local/bin/java10/")
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
car = team.Car("default", root_path=None, config_paths=[])
composite_supplier = supplier.create(cfg,
sources=False,
distribution=True,
build=False,
challenge_root_path="/",
car=car)
self.assertEqual(1, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0],
supplier.ElasticsearchDistributionSupplier)
def test_create_suppliers_for_es_distribution_plugin_source_build(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current")
cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release")
cfg.add(config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
cfg.add(config.Scope.application, "distributions", "release.cache", True)
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src")
cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch")
cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin")
car = team.Car("default", root_path=None, config_paths=[], variables={"build.jdk": "10"})
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
# --revision="community-plugin:effab" --distribution-version="6.0.0"
composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", car=car, plugins=[
core_plugin,
external_plugin
])
self.assertEqual(3, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier)
self.assertIsInstance(composite_supplier.suppliers[1], supplier.PluginDistributionSupplier)
self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin)
self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier)
self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin)
self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_prepare_without_plugins(self, mock_rm, mock_ensure_dir,
mock_decompress):
apply_config_calls = []
def null_apply_config(source_root_path, target_root_path, config_vars):
apply_config_calls.append(
(source_root_path, target_root_path, config_vars))
installer = provisioner.ElasticsearchInstaller(
car=team.Car(
name="unit-test-car",
config_paths=["~/.rally/benchmarks/teams/default/my-car"],
variables={"heap": "4g"}),
node_name="rally-node-0",
node_root_dir="~/.rally/benchmarks/races/unittest",
all_node_ips=["10.17.22.22", "10.17.22.23"],
ip="10.17.22.23",
http_port=9200)
p = provisioner.BareProvisioner(
cluster_settings={"indices.query.bool.max_clause_count": 50000},
es_installer=installer,
plugin_installers=[],
preserve=True,
apply_config=null_apply_config)
node_config = p.prepare(
{"elasticsearch": "/opt/elasticsearch-5.0.0.tar.gz"})
self.assertEqual(installer.car, node_config.car)
self.assertEqual("/opt/elasticsearch-5.0.0", node_config.binary_path)
self.assertEqual(["/opt/elasticsearch-5.0.0/data"],
node_config.data_paths)
self.assertEqual(1, len(apply_config_calls))
source_root_path, target_root_path, config_vars = apply_config_calls[0]
self.assertEqual("~/.rally/benchmarks/teams/default/my-car",
source_root_path)
self.assertEqual("/opt/elasticsearch-5.0.0", target_root_path)
self.assertEqual(
{
"cluster_settings": {
"indices.query.bool.max_clause_count": 50000,
},
"heap": "4g",
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"data_paths": ["/opt/elasticsearch-5.0.0/data"],
"log_path": "~/.rally/benchmarks/races/unittest/logs/server",
"heap_dump_path":
"~/.rally/benchmarks/races/unittest/heapdump",
"node_ip": "10.17.22.23",
"network_host": "10.17.22.23",
"http_port": "9200-9300",
"transport_port": "9300-9400",
"all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]",
"minimum_master_nodes": 2,
"node_count_per_host": 1,
"install_root_path": "/opt/elasticsearch-5.0.0"
}, config_vars)
def test_no_build(self):
car = team.Car("default", root_path=None, config_paths=[], variables={
"clean_command": "./gradlew clean",
"build_command": "./gradlew assemble"
})
es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=None)
es.prepare()
def test_create_suppliers_for_es_missing_distribution_plugin_source_skip(
self):
cfg = config.Config()
# no distribution version!
# cfg.add(config.Scope.application, "mechanic", "distribution.version", "")
# default value from command line
cfg.add(config.Scope.application, "mechanic", "source.revision",
"community-plugin:current")
cfg.add(config.Scope.application, "mechanic",
"distribution.repository", "release")
cfg.add(
config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
)
cfg.add(config.Scope.application, "distributions", "release.cache",
True)
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin",
core_plugin=False)
car = team.Car("default", root_path=None, config_paths=[])
# --from-sources-skip-build --revision="community-plugin:current" (distribution version is missing!)
with self.assertRaises(exceptions.SystemSetupError) as ctx:
supplier.create(cfg,
sources=True,
distribution=False,
build=False,
car=car,
plugins=[core_plugin, external_plugin])
self.assertRegex(ctx.exception.args[0],
r"Could not determine version..*")
def test_merges_options_set_by_different_devices(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "telemetry.devices",
"jfr")
cfg.add(config.Scope.application, "system", "challenge.root.dir",
"challenge-root")
cfg.add(config.Scope.application, "benchmarks", "metrics.log.dir",
"telemetry")
devices = [
MockTelemetryDevice({"ES_JAVA_OPTS": "-Xms256M"}),
MockTelemetryDevice({"ES_JAVA_OPTS": "-Xmx512M"}),
MockTelemetryDevice({"ES_NET_HOST": "127.0.0.1"})
]
t = telemetry.Telemetry(enabled_devices=None, devices=devices)
default_car = team.Car(name="default-car",
config_paths=["/tmp/rally-config"])
opts = t.instrument_candidate_env(default_car, "default-node")
self.assertTrue(opts)
self.assertEqual(len(opts), 2)
self.assertEqual("-Xms256M -Xmx512M", opts["ES_JAVA_OPTS"])
self.assertEqual("127.0.0.1", opts["ES_NET_HOST"])
def test_create_suppliers_for_es_and_plugin_source_build(self):
cfg = config.Config()
cfg.add(config.Scope.application, "mechanic", "source.revision", "elasticsearch:abc,community-plugin:current")
cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release")
cfg.add(config.Scope.application, "distributions", "release.url",
"https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz")
cfg.add(config.Scope.application, "distributions", "release.cache", True)
cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally")
cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src")
cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch")
cfg.add(config.Scope.application, "source", "remote.repo.url", "https://github.com/elastic/elasticsearch.git")
cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.subdir", "elasticsearch-extra/community-plugin")
car = team.Car("default", root_path=None, config_paths=[], variables={
"clean_command": "./gradlew clean",
"build_command": "./gradlew assemble",
"build.jdk": "11"
})
core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
# --revision="elasticsearch:abc,community-plugin:effab"
composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=True, challenge_root_path="/", car=car, plugins=[
core_plugin,
external_plugin
])
self.assertEqual(3, len(composite_supplier.suppliers))
self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchSourceSupplier)
self.assertIsInstance(composite_supplier.suppliers[1], supplier.CorePluginSourceSupplier)
self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin)
self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier)
self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin)
self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_raises_error_on_missing_car_variable(self):
car = team.Car(
"default",
root_path=None,
config_paths=[],
variables={
"clean_command": "./gradlew clean",
# system.build_command is not defined
},
)
renderer = supplier.TemplateRenderer(version="abc")
builder = mock.create_autospec(supplier.Builder)
es = supplier.ElasticsearchSourceSupplier(revision="abc",
es_src_dir="/src",
remote_url="",
car=car,
builder=builder,
template_renderer=renderer)
with pytest.raises(
exceptions.SystemSetupError,
match='Car "default" requires config key "system.build_command"'
):
es.prepare()
assert builder.build.call_count == 0
def test_add_elasticsearch_binary(self):
car = team.Car("default", root_path=None, config_paths=[], variables={
"clean_command": "./gradlew clean",
"build_command": "./gradlew assemble",
"artifact_path_pattern": "distribution/archives/tar/build/distributions/*.tar.gz"
})
es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=None)
binaries = {}
es.add(binaries=binaries)
self.assertEqual(binaries, {"elasticsearch": "elasticsearch.tar.gz"})
def test_build(self):
car = team.Car("default", root_path=None, config_paths=[], variables={
"clean_command": "./gradlew clean",
"build_command": "./gradlew assemble"
})
builder = mock.create_autospec(supplier.Builder)
es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=builder)
es.prepare()
builder.build.assert_called_once_with(["./gradlew clean", "./gradlew assemble"])
def test_raises_error_on_missing_car_variable(self):
car = team.Car("default", root_path=None, config_paths=[], variables={
"clean_command": "./gradlew clean",
# build_command is not defined
})
builder = mock.create_autospec(supplier.Builder)
es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=builder)
with self.assertRaisesRegex(exceptions.SystemSetupError,
"Car \"default\" requires config key \"build_command\"."):
es.prepare()
self.assertEqual(0, builder.build.call_count)
def test_cleanup_nothing_on_preserve(self, mock_path_exists, mock_rm):
mock_path_exists.return_value = False
installer = provisioner.ElasticsearchInstaller(car=team.Car("defaults", None, "/tmp"),
node_name="rally-node-0",
all_node_ips={"127.0.0.1"},
ip="127.0.0.1",
http_port=9200,
node_root_dir="~/.rally/benchmarks/races/unittest")
installer.cleanup(preserve=True)
self.assertEqual(0, mock_path_exists.call_count)
self.assertEqual(0, mock_rm.call_count)
def test_cleanup_nothing_on_preserve(self, mock_path_exists, mock_rm):
mock_path_exists.return_value = False
installer = provisioner.ElasticsearchInstaller(car=team.Car("defaults", "/tmp"),
node_name="rally-node-0",
all_node_ips={"127.0.0.1"},
ip="127.0.0.1",
http_port=9200,
node_root_dir="~/.rally/benchmarks/races/unittest",
data_root_paths=["/tmp/some-data-path"])
installer.cleanup(preserve=True)
mock_path_exists.assert_not_called()
mock_rm.assert_not_called()
def test_cleanup(self, mock_path_exists, mock_rm):
mock_path_exists.return_value = True
installer = provisioner.ElasticsearchInstaller(car=team.Car("defaults", "/tmp"),
node_name="rally-node-0",
all_node_ips={"127.0.0.1"},
ip="127.0.0.1",
http_port=9200,
node_root_dir="~/.rally/benchmarks/races/unittest",
data_root_paths=["/tmp/some-data-path"])
installer.data_paths = ["/tmp/some/data-path-dir"]
installer.cleanup(preserve=True)
expected_dir_calls = [mock.call("/tmp/some/data-path-dir"), mock.call("/rally-root/track/challenge/es-bin")]
mock_path_exists.mock_calls = expected_dir_calls
mock_rm.mock_calls = expected_dir_calls
def test_cleanup(self, mock_path_exists, mock_rm):
mock_path_exists.return_value = True
installer = provisioner.ElasticsearchInstaller(car=team.Car(names="defaults",
root_path=None,
config_paths="/tmp",
variables={"data_paths": "/tmp/some/data-path-dir"}),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
all_node_ips={"127.0.0.1"},
ip="127.0.0.1",
http_port=9200,
node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest")
installer.cleanup(preserve=True)
expected_dir_calls = [mock.call("/tmp/some/data-path-dir"), mock.call("/rally-root/track/challenge/es-bin")]
mock_path_exists.mock_calls = expected_dir_calls
mock_rm.mock_calls = expected_dir_calls
def test_prepare_user_provided_data_path(self, mock_rm, mock_ensure_dir,
mock_decompress):
installer = provisioner.ElasticsearchInstaller(
car=team.Car(
names="defaults",
root_path=None,
config_paths="/tmp",
variables={
"data_paths": "/tmp/some/data-path-dir",
},
),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
http_port=9200,
node_root_dir="~/.rally/benchmarks/races/unittest",
)
installer.install("/data/builds/distributions")
assert installer.es_home_path == "/install/elasticsearch-5.0.0-SNAPSHOT"
assert installer.variables == {
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"data_paths": ["/tmp/some/data-path-dir"],
"log_path": "~/.rally/benchmarks/races/unittest/logs/server",
"heap_dump_path": "~/.rally/benchmarks/races/unittest/heapdump",
"node_ip": "10.17.22.23",
"network_host": "10.17.22.23",
"http_port": "9200",
"transport_port": "9300",
"all_node_ips": '["10.17.22.22","10.17.22.23"]',
"all_node_names": '["rally-node-0","rally-node-1"]',
"minimum_master_nodes": 2,
"install_root_path": "/install/elasticsearch-5.0.0-SNAPSHOT",
}
assert installer.data_paths == ["/tmp/some/data-path-dir"]
def test_add_elasticsearch_binary(self):
car = team.Car(
"default",
root_path=None,
config_paths=[],
variables={
"clean_command":
"./gradlew clean",
"system.build_command":
"./gradlew assemble",
"system.artifact_path_pattern":
"distribution/archives/tar/build/distributions/*.tar.gz",
},
)
renderer = supplier.TemplateRenderer(version="abc")
es = supplier.ElasticsearchSourceSupplier(revision="abc",
es_src_dir="/src",
remote_url="",
car=car,
builder=None,
template_renderer=renderer)
binaries = {}
es.add(binaries=binaries)
assert binaries == {"elasticsearch": "elasticsearch.tar.gz"}
def test_provisioning(self, uuid4, total_memory):
total_memory.return_value = convert.gb_to_bytes(64)
uuid4.return_value = "9dbc682e-d32a-4669-8fbe-56fb77120dd4"
node_root_dir = tempfile.gettempdir()
log_dir = os.path.join(node_root_dir, "logs", "server")
data_dir = "%s/data/9dbc682e-d32a-4669-8fbe-56fb77120dd4" % node_root_dir
rally_root = os.path.normpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
"../../esrally"))
c = team.Car("unit-test-car",
"/tmp",
variables={"xpack.security.enabled": False})
docker = provisioner.DockerProvisioner(
car=c,
node_name="rally-node-0",
cluster_settings={"indices.query.bool.max_clause_count": 5000},
ip="10.17.22.33",
http_port=39200,
node_root_dir=node_root_dir,
distribution_version="5.0.0",
rally_root=rally_root,
preserve=False)
self.assertEqual(
{
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"data_paths": ["/usr/share/elasticsearch/data"],
"log_path": "/var/log/elasticsearch",
"network_host": "0.0.0.0",
"http_port": "39200-39300",
"transport_port": "39300-39400",
"node_count_per_host": 1,
"cluster_settings": {
"xpack.security.enabled": "false",
"xpack.ml.enabled": "false",
"xpack.monitoring.enabled": "false",
"xpack.watcher.enabled": "false",
"indices.query.bool.max_clause_count": 5000
},
"xpack.security.enabled": False
}, docker.config_vars)
self.assertEqual(
{
"es_data_dir": data_dir,
"es_log_dir": log_dir,
"es_version": "5.0.0",
"http_port": 39200,
"mounts": {}
}, docker.docker_vars(mounts={}))
docker_cfg = docker._render_template_from_file(
docker.docker_vars(mounts={}))
self.assertEqual(
"""version: '2'
services:
elasticsearch1:
cap_add:
- IPC_LOCK
image: "docker.elastic.co/elasticsearch/elasticsearch:5.0.0"
ports:
- 39200:39200
- 9300
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- %s:/usr/share/elasticsearch/data
- %s:/var/log/elasticsearch""" % (data_dir, log_dir), docker_cfg)
def test_provisioning_with_defaults(self, uuid4):
uuid4.return_value = "9dbc682e-d32a-4669-8fbe-56fb77120dd4"
node_root_dir = tempfile.gettempdir()
log_dir = os.path.join(node_root_dir, "logs", "server")
heap_dump_dir = os.path.join(node_root_dir, "heapdump")
data_dir = os.path.join(node_root_dir, "data",
"9dbc682e-d32a-4669-8fbe-56fb77120dd4")
rally_root = os.path.normpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
os.pardir, os.pardir, "esrally"))
c = team.Car("unit-test-car",
None,
"/tmp",
variables={
"docker_image":
"docker.elastic.co/elasticsearch/elasticsearch-oss"
})
docker = provisioner.DockerProvisioner(
car=c,
node_name="rally-node-0",
cluster_settings={"indices.query.bool.max_clause_count": 5000},
ip="10.17.22.33",
http_port=39200,
node_root_dir=node_root_dir,
distribution_version="6.3.0",
rally_root=rally_root)
self.assertDictEqual(
{
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"install_root_path": "/usr/share/elasticsearch",
"data_paths": ["/usr/share/elasticsearch/data"],
"log_path": "/var/log/elasticsearch",
"heap_dump_path": "/usr/share/elasticsearch/heapdump",
"discovery_type": "single-node",
"network_host": "0.0.0.0",
"http_port": "39200",
"transport_port": "39300",
"cluster_settings": {
"indices.query.bool.max_clause_count": 5000
},
"docker_image":
"docker.elastic.co/elasticsearch/elasticsearch-oss"
}, docker.config_vars)
self.assertDictEqual(
{
"es_data_dir": data_dir,
"es_log_dir": log_dir,
"es_heap_dump_dir": heap_dump_dir,
"es_version": "6.3.0",
"docker_image":
"docker.elastic.co/elasticsearch/elasticsearch-oss",
"http_port": 39200,
"mounts": {}
}, docker.docker_vars(mounts={}))
docker_cfg = docker._render_template_from_file(
docker.docker_vars(mounts={}))
self.assertEqual(
"""version: '2.2'
services:
elasticsearch1:
cap_add:
- IPC_LOCK
image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.0"
labels:
io.rally.description: "elasticsearch-rally"
ports:
- 39200:39200
- 9300
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- %s:/usr/share/elasticsearch/data
- %s:/var/log/elasticsearch
- %s:/usr/share/elasticsearch/heapdump
healthcheck:
test: nc -z 127.0.0.1 39200
interval: 5s
timeout: 2s
retries: 10""" % (data_dir, log_dir, heap_dump_dir), docker_cfg)
def test_provisioning_with_defaults(self, uuid4):
uuid4.return_value = "9dbc682e-d32a-4669-8fbe-56fb77120dd4"
node_ip = "10.17.22.33"
node_root_dir = tempfile.gettempdir()
log_dir = os.path.join(node_root_dir, "logs", "server")
heap_dump_dir = os.path.join(node_root_dir, "heapdump")
data_dir = os.path.join(node_root_dir, "data",
"9dbc682e-d32a-4669-8fbe-56fb77120dd4")
rally_root = os.path.normpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
os.pardir, os.pardir, "esrally"))
c = team.Car(
"unit-test-car",
None,
"/tmp",
variables={
"docker_image":
"docker.elastic.co/elasticsearch/elasticsearch-oss",
},
)
docker = provisioner.DockerProvisioner(
car=c,
node_name="rally-node-0",
ip=node_ip,
http_port=39200,
node_root_dir=node_root_dir,
distribution_version="6.3.0",
rally_root=rally_root,
)
assert docker.config_vars == {
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"install_root_path": "/usr/share/elasticsearch",
"data_paths": ["/usr/share/elasticsearch/data"],
"log_path": "/var/log/elasticsearch",
"heap_dump_path": "/usr/share/elasticsearch/heapdump",
"discovery_type": "single-node",
"network_host": "0.0.0.0",
"http_port": "39200",
"transport_port": "39300",
"cluster_settings": {},
"docker_image":
"docker.elastic.co/elasticsearch/elasticsearch-oss",
}
assert docker.docker_vars(mounts={}) == {
"es_data_dir": data_dir,
"es_log_dir": log_dir,
"es_heap_dump_dir": heap_dump_dir,
"es_version": "6.3.0",
"docker_image":
"docker.elastic.co/elasticsearch/elasticsearch-oss",
"http_port": 39200,
"node_ip": node_ip,
"mounts": {},
}
docker_cfg = docker._render_template_from_file(
docker.docker_vars(mounts={}))
assert docker_cfg == (f'''version: '2.2'
services:
elasticsearch1:
cap_add:
- IPC_LOCK
image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.0"
labels:
io.rally.description: "elasticsearch-rally"
ports:
- 39200:39200
- 9300
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- {data_dir}:/usr/share/elasticsearch/data
- {log_dir}:/var/log/elasticsearch
- {heap_dump_dir}:/usr/share/elasticsearch/heapdump
healthcheck:
test: nc -z 127.0.0.1 39200
interval: 5s
timeout: 2s
retries: 10
networks:
- rally-es
networks:
rally-es:
driver_opts:
com.docker.network.bridge.host_binding_ipv4: "{node_ip}"''')
def test_prepare_distribution_ge_63_with_plugins(self, mock_rm,
mock_ensure_dir,
mock_install,
mock_decompress):
"""
Test that plugin.mandatory is set to the meta plugin name (e.g. `x-pack`) and not
the specific plugin name (e.g. `x-pack-security`) for Elasticsearch >=6.3.0
See: https://github.com/elastic/elasticsearch/pull/28710
"""
apply_config_calls = []
def null_apply_config(source_root_path, target_root_path, config_vars):
apply_config_calls.append(
(source_root_path, target_root_path, config_vars))
installer = provisioner.ElasticsearchInstaller(
car=team.Car(names="unit-test-car",
root_path=None,
config_paths=[
HOME_DIR +
"/.rally/benchmarks/teams/default/my-car"
],
variables={
"heap": "4g",
"runtime.jdk": "8",
"runtime.jdk.bundled": "true"
}),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
http_port=9200)
p = provisioner.BareProvisioner(
cluster_settings={"indices.query.bool.max_clause_count": 50000},
es_installer=installer,
plugin_installers=[
provisioner.PluginInstaller(
BareProvisionerTests.MockRallyTeamXPackPlugin(),
java_home="/usr/local/javas/java8",
hook_handler_class=BareProvisionerTests.NoopHookHandler)
],
distribution_version="6.3.0",
apply_config=null_apply_config)
node_config = p.prepare(
{"elasticsearch": "/opt/elasticsearch-6.3.0.tar.gz"})
self.assertEqual({}, node_config.car_env)
self.assertEqual("8", node_config.car_runtime_jdks)
self.assertEqual("/opt/elasticsearch-6.3.0", node_config.binary_path)
self.assertEqual(["/opt/elasticsearch-6.3.0/data"],
node_config.data_paths)
self.assertEqual(1, len(apply_config_calls))
source_root_path, target_root_path, config_vars = apply_config_calls[0]
self.assertEqual(HOME_DIR + "/.rally/benchmarks/teams/default/my-car",
source_root_path)
self.assertEqual("/opt/elasticsearch-6.3.0", target_root_path)
self.maxDiff = None
self.assertEqual(
{
"cluster_settings": {
"indices.query.bool.max_clause_count": 50000,
"plugin.mandatory": ["x-pack"]
},
"heap": "4g",
"runtime.jdk": "8",
"runtime.jdk.bundled": "true",
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"data_paths": ["/opt/elasticsearch-6.3.0/data"],
"log_path":
HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server",
"heap_dump_path":
HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump",
"node_ip": "10.17.22.23",
"network_host": "10.17.22.23",
"http_port": "9200",
"transport_port": "9300",
"all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]",
"all_node_names": "[\"rally-node-0\",\"rally-node-1\"]",
"minimum_master_nodes": 2,
"install_root_path": "/opt/elasticsearch-6.3.0",
"plugin_name": "x-pack-security",
"xpack_security_enabled": True
}, config_vars)
def test_prepare_without_plugins(self, mock_rm, mock_ensure_dir,
mock_decompress):
apply_config_calls = []
def null_apply_config(source_root_path, target_root_path, config_vars):
apply_config_calls.append(
(source_root_path, target_root_path, config_vars))
installer = provisioner.ElasticsearchInstaller(
car=team.Car(names="unit-test-car",
root_path=None,
config_paths=[
HOME_DIR +
"/.rally/benchmarks/teams/default/my-car"
],
variables={
"heap": "4g",
"runtime.jdk": "8",
"runtime.jdk.bundled": "true"
}),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
http_port=9200)
p = provisioner.BareProvisioner(es_installer=installer,
plugin_installers=[],
apply_config=null_apply_config)
node_config = p.prepare(
{"elasticsearch": "/opt/elasticsearch-5.0.0.tar.gz"})
self.assertEqual("8", node_config.car_runtime_jdks)
self.assertEqual("/opt/elasticsearch-5.0.0", node_config.binary_path)
self.assertEqual(["/opt/elasticsearch-5.0.0/data"],
node_config.data_paths)
self.assertEqual(1, len(apply_config_calls))
source_root_path, target_root_path, config_vars = apply_config_calls[0]
self.assertEqual(HOME_DIR + "/.rally/benchmarks/teams/default/my-car",
source_root_path)
self.assertEqual("/opt/elasticsearch-5.0.0", target_root_path)
self.assertEqual(
{
"cluster_settings": {},
"heap": "4g",
"runtime.jdk": "8",
"runtime.jdk.bundled": "true",
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"data_paths": ["/opt/elasticsearch-5.0.0/data"],
"log_path":
HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server",
"heap_dump_path":
HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump",
"node_ip": "10.17.22.23",
"network_host": "10.17.22.23",
"http_port": "9200",
"transport_port": "9300",
"all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]",
"all_node_names": "[\"rally-node-0\",\"rally-node-1\"]",
"minimum_master_nodes": 2,
"install_root_path": "/opt/elasticsearch-5.0.0"
}, config_vars)
def test_prepare_distribution_lt_63_with_plugins(self, mock_rm,
mock_ensure_dir,
mock_install,
mock_decompress):
"""
Test that plugin.mandatory is set to the specific plugin name (e.g. `x-pack-security`) and not
the meta plugin name (e.g. `x-pack`) for Elasticsearch <6.3
See: https://github.com/elastic/elasticsearch/pull/28710
"""
apply_config_calls = []
def null_apply_config(source_root_path, target_root_path, config_vars):
apply_config_calls.append(
(source_root_path, target_root_path, config_vars))
installer = provisioner.ElasticsearchInstaller(
car=team.Car(
names="unit-test-car",
root_path=None,
config_paths=[
HOME_DIR + "/.rally/benchmarks/teams/default/my-car"
],
variables={
"heap": "4g",
"runtime.jdk": "8",
"runtime.jdk.bundled": "true",
},
),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
http_port=9200,
)
p = provisioner.BareProvisioner(
es_installer=installer,
plugin_installers=[
provisioner.PluginInstaller(
self.MockRallyTeamXPackPlugin(),
java_home="/usr/local/javas/java8",
hook_handler_class=self.NoopHookHandler,
)
],
distribution_version="6.2.3",
apply_config=null_apply_config,
)
node_config = p.prepare(
{"elasticsearch": "/opt/elasticsearch-5.0.0.tar.gz"})
assert node_config.car_runtime_jdks == "8"
assert node_config.binary_path == "/opt/elasticsearch-5.0.0"
assert node_config.data_paths == ["/opt/elasticsearch-5.0.0/data"]
assert len(apply_config_calls) == 1
source_root_path, target_root_path, config_vars = apply_config_calls[0]
assert source_root_path == HOME_DIR + "/.rally/benchmarks/teams/default/my-car"
assert target_root_path == "/opt/elasticsearch-5.0.0"
self.maxDiff = None
assert config_vars == {
"cluster_settings": {
"plugin.mandatory": ["x-pack-security"]
},
"heap": "4g",
"runtime.jdk": "8",
"runtime.jdk.bundled": "true",
"cluster_name": "rally-benchmark",
"node_name": "rally-node-0",
"data_paths": ["/opt/elasticsearch-5.0.0/data"],
"log_path":
HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server",
"heap_dump_path":
HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump",
"node_ip": "10.17.22.23",
"network_host": "10.17.22.23",
"http_port": "9200",
"transport_port": "9300",
"all_node_ips": '["10.17.22.22","10.17.22.23"]',
"all_node_names": '["rally-node-0","rally-node-1"]',
"minimum_master_nodes": 2,
"install_root_path": "/opt/elasticsearch-5.0.0",
"plugin_name": "x-pack-security",
"xpack_security_enabled": True,
}
