def test_prepare_default_data_paths(self, mock_rm, mock_ensure_dir, mock_decompress): installer = provisioner.ElasticsearchInstaller(car=team.Car(names="defaults", root_path=None, config_paths="/tmp"), java_home="/usr/local/javas/java8", node_name="rally-node-0", all_node_ips=["10.17.22.22", "10.17.22.23"], all_node_names=["rally-node-0", "rally-node-1"], ip="10.17.22.23", http_port=9200, node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest") installer.install("/data/builds/distributions") self.assertEqual(installer.es_home_path, "/install/elasticsearch-5.0.0-SNAPSHOT") self.assertEqual({ "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/install/elasticsearch-5.0.0-SNAPSHOT/data"], "log_path": HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server", "heap_dump_path": HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump", "node_ip": "10.17.22.23", "network_host": "10.17.22.23", "http_port": "9200-9300", "transport_port": "9300-9400", "all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]", "all_node_names": "[\"rally-node-0\",\"rally-node-1\"]", "minimum_master_nodes": 2, "install_root_path": "/install/elasticsearch-5.0.0-SNAPSHOT" }, installer.variables) self.assertEqual(installer.data_paths, ["/install/elasticsearch-5.0.0-SNAPSHOT/data"])
def test_env_options_order(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "keep.running", False) cfg.add(config.Scope.application, "system", "env.name", "test") ms = get_metrics_store(cfg) proc_launcher = launcher.ProcessLauncher(cfg, ms, races_root_dir="/home") default_car = team.Car(names="default-car", root_path=None, config_paths=["/tmp/rally-config"]) node_telemetry = [ telemetry.FlightRecorder(telemetry_params={}, log_root="/tmp/telemetry", java_major_version=8) ] t = telemetry.Telemetry(["jfr"], devices=node_telemetry) env = proc_launcher._prepare_env(car=default_car, node_name="node0", java_home="/java_home", t=t) self.assertEqual("/java_home/bin" + os.pathsep + os.environ["PATH"], env["PATH"]) self.assertEqual( "-XX:+ExitOnOutOfMemoryError -XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints " "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder " "-XX:FlightRecorderOptions=disk=true,maxage=0s,maxsize=0,dumponexit=true,dumponexitpath=/tmp/telemetry/default-car-node0.jfr " "-XX:StartFlightRecording=defaultrecording=true", env["ES_JAVA_OPTS"])
def test_invokes_hook_no_java_home(self): installer = provisioner.ElasticsearchInstaller( car=team.Car(names="defaults", root_path="/tmp", config_paths="/tmp/templates", variables={"data_paths": "/tmp/some/data-path-dir"}), java_home=None, node_name="rally-node-0", all_node_ips=["10.17.22.22", "10.17.22.23"], all_node_names=["rally-node-0", "rally-node-1"], ip="10.17.22.23", http_port=9200, node_root_dir="~/.rally/benchmarks/races/unittest", hook_handler_class=NoopHookHandler) self.assertEqual(0, len(installer.hook_handler.hook_calls)) installer.invoke_install_hook(team.BootstrapPhase.post_install, {"foo": "bar"}) self.assertEqual(1, len(installer.hook_handler.hook_calls)) self.assertEqual( {"foo": "bar"}, installer.hook_handler.hook_calls["post_install"]["variables"]) self.assertEqual( {"env": {}}, installer.hook_handler.hook_calls["post_install"]["kwargs"])
def test_invokes_hook_with_java_home(self): installer = provisioner.ElasticsearchInstaller( car=team.Car( names="defaults", root_path="/tmp", config_paths="/tmp/templates", variables={ "data_paths": "/tmp/some/data-path-dir", }, ), java_home="/usr/local/javas/java8", node_name="rally-node-0", all_node_ips=["10.17.22.22", "10.17.22.23"], all_node_names=["rally-node-0", "rally-node-1"], ip="10.17.22.23", http_port=9200, node_root_dir="~/.rally/benchmarks/races/unittest", hook_handler_class=NoopHookHandler, ) assert len(installer.hook_handler.hook_calls) == 0 installer.invoke_install_hook(team.BootstrapPhase.post_install, {"foo": "bar"}) assert len(installer.hook_handler.hook_calls) == 1 assert installer.hook_handler.hook_calls["post_install"][ "variables"] == { "foo": "bar" } assert installer.hook_handler.hook_calls["post_install"]["kwargs"] == { "env": { "JAVA_HOME": "/usr/local/javas/java8" } }
def test_prepare_user_provided_data_path(self, mock_rm, mock_ensure_dir, mock_decompress): installer = provisioner.ElasticsearchInstaller( car=team.Car(name="defaults", config_paths="/tmp", variables={"data_paths": "/tmp/some/data-path-dir"}), node_name="rally-node-0", all_node_ips=["10.17.22.22", "10.17.22.23"], ip="10.17.22.23", http_port=9200, node_root_dir="~/.rally/benchmarks/races/unittest") installer.install("/data/builds/distributions") self.assertEqual(installer.es_home_path, "/install/elasticsearch-5.0.0-SNAPSHOT") self.assertEqual( { "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/tmp/some/data-path-dir"], "log_path": "~/.rally/benchmarks/races/unittest/logs/server", "node_ip": "10.17.22.23", "network_host": "10.17.22.23", "http_port": "9200-9300", "transport_port": "9300-9400", "all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]", "minimum_master_nodes": 2, "node_count_per_host": 1, "install_root_path": "/install/elasticsearch-5.0.0-SNAPSHOT" }, installer.variables) self.assertEqual(installer.data_paths, ["/tmp/some/data-path-dir"])
def test_provisioning_with_variables(self, uuid4): uuid4.return_value = "86f42ae0-5840-4b5b-918d-41e7907cb644" node_root_dir = tempfile.gettempdir() log_dir = os.path.join(node_root_dir, "logs", "server") heap_dump_dir = os.path.join(node_root_dir, "heapdump") data_dir = os.path.join(node_root_dir, "data", "86f42ae0-5840-4b5b-918d-41e7907cb644") rally_root = os.path.normpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "esrally")) c = team.Car("unit-test-car", None, "/tmp", variables={ "docker_image": "docker.elastic.co/elasticsearch/elasticsearch", "docker_mem_limit": "256m", "docker_cpu_count": 2 }) docker = provisioner.DockerProvisioner(car=c, node_name="rally-node-0", ip="10.17.22.33", http_port=39200, node_root_dir=node_root_dir, distribution_version="6.3.0", rally_root=rally_root) docker_cfg = docker._render_template_from_file( docker.docker_vars(mounts={})) self.assertEqual( """version: '2.2' services: elasticsearch1: cap_add: - IPC_LOCK image: "docker.elastic.co/elasticsearch/elasticsearch:6.3.0" labels: io.rally.description: "elasticsearch-rally" cpu_count: 2 mem_limit: 256m ports: - 39200:39200 - 9300 ulimits: memlock: soft: -1 hard: -1 volumes: - %s:/usr/share/elasticsearch/data - %s:/var/log/elasticsearch - %s:/usr/share/elasticsearch/heapdump healthcheck: test: nc -z 127.0.0.1 39200 interval: 5s timeout: 2s retries: 10""" % (data_dir, log_dir, heap_dump_dir), docker_cfg)
def test_create_suppliers_for_es_only_config(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0") # default value from command line cfg.add(config.Scope.application, "mechanic", "source.revision", "current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add( config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" ) cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") car = team.Car("default", root_path=None, config_paths=[]) composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", car=car) self.assertEqual(1, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier)
def test_create_suppliers_for_es_distribution_plugin_source_build(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "distribution.version", "6.0.0") # default value from command line cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin") car = team.Car("default", root_path=None, config_paths=[], variables={"build.jdk": "10"}) core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --revision="community-plugin:effab" --distribution-version="6.0.0" composite_supplier = supplier.create(cfg, sources=False, distribution=True, build=False, challenge_root_path="/", car=car, plugins=[ core_plugin, external_plugin ]) self.assertEqual(3, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchDistributionSupplier) self.assertIsInstance(composite_supplier.suppliers[1], supplier.PluginDistributionSupplier) self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin) self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier) self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin) self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_prepare_without_plugins(self, mock_rm, mock_ensure_dir, mock_decompress): apply_config_calls = [] def null_apply_config(source_root_path, target_root_path, config_vars): apply_config_calls.append( (source_root_path, target_root_path, config_vars)) installer = provisioner.ElasticsearchInstaller( car=team.Car( name="unit-test-car", config_paths=["~/.rally/benchmarks/teams/default/my-car"], variables={"heap": "4g"}), node_name="rally-node-0", node_root_dir="~/.rally/benchmarks/races/unittest", all_node_ips=["10.17.22.22", "10.17.22.23"], ip="10.17.22.23", http_port=9200) p = provisioner.BareProvisioner( cluster_settings={"indices.query.bool.max_clause_count": 50000}, es_installer=installer, plugin_installers=[], preserve=True, apply_config=null_apply_config) node_config = p.prepare( {"elasticsearch": "/opt/elasticsearch-5.0.0.tar.gz"}) self.assertEqual(installer.car, node_config.car) self.assertEqual("/opt/elasticsearch-5.0.0", node_config.binary_path) self.assertEqual(["/opt/elasticsearch-5.0.0/data"], node_config.data_paths) self.assertEqual(1, len(apply_config_calls)) source_root_path, target_root_path, config_vars = apply_config_calls[0] self.assertEqual("~/.rally/benchmarks/teams/default/my-car", source_root_path) self.assertEqual("/opt/elasticsearch-5.0.0", target_root_path) self.assertEqual( { "cluster_settings": { "indices.query.bool.max_clause_count": 50000, }, "heap": "4g", "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/opt/elasticsearch-5.0.0/data"], "log_path": "~/.rally/benchmarks/races/unittest/logs/server", "heap_dump_path": "~/.rally/benchmarks/races/unittest/heapdump", "node_ip": "10.17.22.23", "network_host": "10.17.22.23", "http_port": "9200-9300", "transport_port": "9300-9400", "all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]", "minimum_master_nodes": 2, "node_count_per_host": 1, "install_root_path": "/opt/elasticsearch-5.0.0" }, config_vars)
def test_no_build(self): car = team.Car("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "build_command": "./gradlew assemble" }) es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=None) es.prepare()
def test_create_suppliers_for_es_missing_distribution_plugin_source_skip( self): cfg = config.Config() # no distribution version! # cfg.add(config.Scope.application, "mechanic", "distribution.version", "") # default value from command line cfg.add(config.Scope.application, "mechanic", "source.revision", "community-plugin:current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add( config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" ) cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) car = team.Car("default", root_path=None, config_paths=[]) # --from-sources-skip-build --revision="community-plugin:current" (distribution version is missing!) with self.assertRaises(exceptions.SystemSetupError) as ctx: supplier.create(cfg, sources=True, distribution=False, build=False, car=car, plugins=[core_plugin, external_plugin]) self.assertRegex(ctx.exception.args[0], r"Could not determine version..*")
def test_merges_options_set_by_different_devices(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "telemetry.devices", "jfr") cfg.add(config.Scope.application, "system", "challenge.root.dir", "challenge-root") cfg.add(config.Scope.application, "benchmarks", "metrics.log.dir", "telemetry") devices = [ MockTelemetryDevice({"ES_JAVA_OPTS": "-Xms256M"}), MockTelemetryDevice({"ES_JAVA_OPTS": "-Xmx512M"}), MockTelemetryDevice({"ES_NET_HOST": "127.0.0.1"}) ] t = telemetry.Telemetry(enabled_devices=None, devices=devices) default_car = team.Car(name="default-car", config_paths=["/tmp/rally-config"]) opts = t.instrument_candidate_env(default_car, "default-node") self.assertTrue(opts) self.assertEqual(len(opts), 2) self.assertEqual("-Xms256M -Xmx512M", opts["ES_JAVA_OPTS"]) self.assertEqual("127.0.0.1", opts["ES_NET_HOST"])
def test_create_suppliers_for_es_and_plugin_source_build(self): cfg = config.Config() cfg.add(config.Scope.application, "mechanic", "source.revision", "elasticsearch:abc,community-plugin:current") cfg.add(config.Scope.application, "mechanic", "distribution.repository", "release") cfg.add(config.Scope.application, "distributions", "release.url", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz") cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") cfg.add(config.Scope.application, "source", "remote.repo.url", "https://github.com/elastic/elasticsearch.git") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.subdir", "elasticsearch-extra/community-plugin") car = team.Car("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "build_command": "./gradlew assemble", "build.jdk": "11" }) core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True) external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False) # --revision="elasticsearch:abc,community-plugin:effab" composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=True, challenge_root_path="/", car=car, plugins=[ core_plugin, external_plugin ]) self.assertEqual(3, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.ElasticsearchSourceSupplier) self.assertIsInstance(composite_supplier.suppliers[1], supplier.CorePluginSourceSupplier) self.assertEqual(core_plugin, composite_supplier.suppliers[1].plugin) self.assertIsInstance(composite_supplier.suppliers[2], supplier.ExternalPluginSourceSupplier) self.assertEqual(external_plugin, composite_supplier.suppliers[2].plugin) self.assertIsNotNone(composite_supplier.suppliers[2].builder)
def test_raises_error_on_missing_car_variable(self): car = team.Car( "default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", # system.build_command is not defined }, ) renderer = supplier.TemplateRenderer(version="abc") builder = mock.create_autospec(supplier.Builder) es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=builder, template_renderer=renderer) with pytest.raises( exceptions.SystemSetupError, match='Car "default" requires config key "system.build_command"' ): es.prepare() assert builder.build.call_count == 0
def test_add_elasticsearch_binary(self): car = team.Car("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "build_command": "./gradlew assemble", "artifact_path_pattern": "distribution/archives/tar/build/distributions/*.tar.gz" }) es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=None) binaries = {} es.add(binaries=binaries) self.assertEqual(binaries, {"elasticsearch": "elasticsearch.tar.gz"})
def test_build(self): car = team.Car("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "build_command": "./gradlew assemble" }) builder = mock.create_autospec(supplier.Builder) es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=builder) es.prepare() builder.build.assert_called_once_with(["./gradlew clean", "./gradlew assemble"])
def test_raises_error_on_missing_car_variable(self): car = team.Car("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", # build_command is not defined }) builder = mock.create_autospec(supplier.Builder) es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=builder) with self.assertRaisesRegex(exceptions.SystemSetupError, "Car \"default\" requires config key \"build_command\"."): es.prepare() self.assertEqual(0, builder.build.call_count)
def test_cleanup_nothing_on_preserve(self, mock_path_exists, mock_rm): mock_path_exists.return_value = False installer = provisioner.ElasticsearchInstaller(car=team.Car("defaults", None, "/tmp"), node_name="rally-node-0", all_node_ips={"127.0.0.1"}, ip="127.0.0.1", http_port=9200, node_root_dir="~/.rally/benchmarks/races/unittest") installer.cleanup(preserve=True) self.assertEqual(0, mock_path_exists.call_count) self.assertEqual(0, mock_rm.call_count)
def test_cleanup_nothing_on_preserve(self, mock_path_exists, mock_rm): mock_path_exists.return_value = False installer = provisioner.ElasticsearchInstaller(car=team.Car("defaults", "/tmp"), node_name="rally-node-0", all_node_ips={"127.0.0.1"}, ip="127.0.0.1", http_port=9200, node_root_dir="~/.rally/benchmarks/races/unittest", data_root_paths=["/tmp/some-data-path"]) installer.cleanup(preserve=True) mock_path_exists.assert_not_called() mock_rm.assert_not_called()
def test_cleanup(self, mock_path_exists, mock_rm): mock_path_exists.return_value = True installer = provisioner.ElasticsearchInstaller(car=team.Car("defaults", "/tmp"), node_name="rally-node-0", all_node_ips={"127.0.0.1"}, ip="127.0.0.1", http_port=9200, node_root_dir="~/.rally/benchmarks/races/unittest", data_root_paths=["/tmp/some-data-path"]) installer.data_paths = ["/tmp/some/data-path-dir"] installer.cleanup(preserve=True) expected_dir_calls = [mock.call("/tmp/some/data-path-dir"), mock.call("/rally-root/track/challenge/es-bin")] mock_path_exists.mock_calls = expected_dir_calls mock_rm.mock_calls = expected_dir_calls
def test_cleanup(self, mock_path_exists, mock_rm): mock_path_exists.return_value = True installer = provisioner.ElasticsearchInstaller(car=team.Car(names="defaults", root_path=None, config_paths="/tmp", variables={"data_paths": "/tmp/some/data-path-dir"}), java_home="/usr/local/javas/java8", node_name="rally-node-0", all_node_ips={"127.0.0.1"}, ip="127.0.0.1", http_port=9200, node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest") installer.cleanup(preserve=True) expected_dir_calls = [mock.call("/tmp/some/data-path-dir"), mock.call("/rally-root/track/challenge/es-bin")] mock_path_exists.mock_calls = expected_dir_calls mock_rm.mock_calls = expected_dir_calls
def test_prepare_user_provided_data_path(self, mock_rm, mock_ensure_dir, mock_decompress): installer = provisioner.ElasticsearchInstaller( car=team.Car( names="defaults", root_path=None, config_paths="/tmp", variables={ "data_paths": "/tmp/some/data-path-dir", }, ), java_home="/usr/local/javas/java8", node_name="rally-node-0", all_node_ips=["10.17.22.22", "10.17.22.23"], all_node_names=["rally-node-0", "rally-node-1"], ip="10.17.22.23", http_port=9200, node_root_dir="~/.rally/benchmarks/races/unittest", ) installer.install("/data/builds/distributions") assert installer.es_home_path == "/install/elasticsearch-5.0.0-SNAPSHOT" assert installer.variables == { "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/tmp/some/data-path-dir"], "log_path": "~/.rally/benchmarks/races/unittest/logs/server", "heap_dump_path": "~/.rally/benchmarks/races/unittest/heapdump", "node_ip": "10.17.22.23", "network_host": "10.17.22.23", "http_port": "9200", "transport_port": "9300", "all_node_ips": '["10.17.22.22","10.17.22.23"]', "all_node_names": '["rally-node-0","rally-node-1"]', "minimum_master_nodes": 2, "install_root_path": "/install/elasticsearch-5.0.0-SNAPSHOT", } assert installer.data_paths == ["/tmp/some/data-path-dir"]
def test_add_elasticsearch_binary(self): car = team.Car( "default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "system.build_command": "./gradlew assemble", "system.artifact_path_pattern": "distribution/archives/tar/build/distributions/*.tar.gz", }, ) renderer = supplier.TemplateRenderer(version="abc") es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", car=car, builder=None, template_renderer=renderer) binaries = {} es.add(binaries=binaries) assert binaries == {"elasticsearch": "elasticsearch.tar.gz"}
def test_provisioning(self, uuid4, total_memory): total_memory.return_value = convert.gb_to_bytes(64) uuid4.return_value = "9dbc682e-d32a-4669-8fbe-56fb77120dd4" node_root_dir = tempfile.gettempdir() log_dir = os.path.join(node_root_dir, "logs", "server") data_dir = "%s/data/9dbc682e-d32a-4669-8fbe-56fb77120dd4" % node_root_dir rally_root = os.path.normpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../esrally")) c = team.Car("unit-test-car", "/tmp", variables={"xpack.security.enabled": False}) docker = provisioner.DockerProvisioner( car=c, node_name="rally-node-0", cluster_settings={"indices.query.bool.max_clause_count": 5000}, ip="10.17.22.33", http_port=39200, node_root_dir=node_root_dir, distribution_version="5.0.0", rally_root=rally_root, preserve=False) self.assertEqual( { "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/usr/share/elasticsearch/data"], "log_path": "/var/log/elasticsearch", "network_host": "0.0.0.0", "http_port": "39200-39300", "transport_port": "39300-39400", "node_count_per_host": 1, "cluster_settings": { "xpack.security.enabled": "false", "xpack.ml.enabled": "false", "xpack.monitoring.enabled": "false", "xpack.watcher.enabled": "false", "indices.query.bool.max_clause_count": 5000 }, "xpack.security.enabled": False }, docker.config_vars) self.assertEqual( { "es_data_dir": data_dir, "es_log_dir": log_dir, "es_version": "5.0.0", "http_port": 39200, "mounts": {} }, docker.docker_vars(mounts={})) docker_cfg = docker._render_template_from_file( docker.docker_vars(mounts={})) self.assertEqual( """version: '2' services: elasticsearch1: cap_add: - IPC_LOCK image: "docker.elastic.co/elasticsearch/elasticsearch:5.0.0" ports: - 39200:39200 - 9300 ulimits: memlock: soft: -1 hard: -1 volumes: - %s:/usr/share/elasticsearch/data - %s:/var/log/elasticsearch""" % (data_dir, log_dir), docker_cfg)
def test_provisioning_with_defaults(self, uuid4): uuid4.return_value = "9dbc682e-d32a-4669-8fbe-56fb77120dd4" node_root_dir = tempfile.gettempdir() log_dir = os.path.join(node_root_dir, "logs", "server") heap_dump_dir = os.path.join(node_root_dir, "heapdump") data_dir = os.path.join(node_root_dir, "data", "9dbc682e-d32a-4669-8fbe-56fb77120dd4") rally_root = os.path.normpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "esrally")) c = team.Car("unit-test-car", None, "/tmp", variables={ "docker_image": "docker.elastic.co/elasticsearch/elasticsearch-oss" }) docker = provisioner.DockerProvisioner( car=c, node_name="rally-node-0", cluster_settings={"indices.query.bool.max_clause_count": 5000}, ip="10.17.22.33", http_port=39200, node_root_dir=node_root_dir, distribution_version="6.3.0", rally_root=rally_root) self.assertDictEqual( { "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "install_root_path": "/usr/share/elasticsearch", "data_paths": ["/usr/share/elasticsearch/data"], "log_path": "/var/log/elasticsearch", "heap_dump_path": "/usr/share/elasticsearch/heapdump", "discovery_type": "single-node", "network_host": "0.0.0.0", "http_port": "39200", "transport_port": "39300", "cluster_settings": { "indices.query.bool.max_clause_count": 5000 }, "docker_image": "docker.elastic.co/elasticsearch/elasticsearch-oss" }, docker.config_vars) self.assertDictEqual( { "es_data_dir": data_dir, "es_log_dir": log_dir, "es_heap_dump_dir": heap_dump_dir, "es_version": "6.3.0", "docker_image": "docker.elastic.co/elasticsearch/elasticsearch-oss", "http_port": 39200, "mounts": {} }, docker.docker_vars(mounts={})) docker_cfg = docker._render_template_from_file( docker.docker_vars(mounts={})) self.assertEqual( """version: '2.2' services: elasticsearch1: cap_add: - IPC_LOCK image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.0" labels: io.rally.description: "elasticsearch-rally" ports: - 39200:39200 - 9300 ulimits: memlock: soft: -1 hard: -1 volumes: - %s:/usr/share/elasticsearch/data - %s:/var/log/elasticsearch - %s:/usr/share/elasticsearch/heapdump healthcheck: test: nc -z 127.0.0.1 39200 interval: 5s timeout: 2s retries: 10""" % (data_dir, log_dir, heap_dump_dir), docker_cfg)
def test_provisioning_with_defaults(self, uuid4): uuid4.return_value = "9dbc682e-d32a-4669-8fbe-56fb77120dd4" node_ip = "10.17.22.33" node_root_dir = tempfile.gettempdir() log_dir = os.path.join(node_root_dir, "logs", "server") heap_dump_dir = os.path.join(node_root_dir, "heapdump") data_dir = os.path.join(node_root_dir, "data", "9dbc682e-d32a-4669-8fbe-56fb77120dd4") rally_root = os.path.normpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "esrally")) c = team.Car( "unit-test-car", None, "/tmp", variables={ "docker_image": "docker.elastic.co/elasticsearch/elasticsearch-oss", }, ) docker = provisioner.DockerProvisioner( car=c, node_name="rally-node-0", ip=node_ip, http_port=39200, node_root_dir=node_root_dir, distribution_version="6.3.0", rally_root=rally_root, ) assert docker.config_vars == { "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "install_root_path": "/usr/share/elasticsearch", "data_paths": ["/usr/share/elasticsearch/data"], "log_path": "/var/log/elasticsearch", "heap_dump_path": "/usr/share/elasticsearch/heapdump", "discovery_type": "single-node", "network_host": "0.0.0.0", "http_port": "39200", "transport_port": "39300", "cluster_settings": {}, "docker_image": "docker.elastic.co/elasticsearch/elasticsearch-oss", } assert docker.docker_vars(mounts={}) == { "es_data_dir": data_dir, "es_log_dir": log_dir, "es_heap_dump_dir": heap_dump_dir, "es_version": "6.3.0", "docker_image": "docker.elastic.co/elasticsearch/elasticsearch-oss", "http_port": 39200, "node_ip": node_ip, "mounts": {}, } docker_cfg = docker._render_template_from_file( docker.docker_vars(mounts={})) assert docker_cfg == (f'''version: '2.2' services: elasticsearch1: cap_add: - IPC_LOCK image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.3.0" labels: io.rally.description: "elasticsearch-rally" ports: - 39200:39200 - 9300 ulimits: memlock: soft: -1 hard: -1 volumes: - {data_dir}:/usr/share/elasticsearch/data - {log_dir}:/var/log/elasticsearch - {heap_dump_dir}:/usr/share/elasticsearch/heapdump healthcheck: test: nc -z 127.0.0.1 39200 interval: 5s timeout: 2s retries: 10 networks: - rally-es networks: rally-es: driver_opts: com.docker.network.bridge.host_binding_ipv4: "{node_ip}"''')
def test_prepare_distribution_ge_63_with_plugins(self, mock_rm, mock_ensure_dir, mock_install, mock_decompress): """ Test that plugin.mandatory is set to the meta plugin name (e.g. `x-pack`) and not the specific plugin name (e.g. `x-pack-security`) for Elasticsearch >=6.3.0 See: https://github.com/elastic/elasticsearch/pull/28710 """ apply_config_calls = [] def null_apply_config(source_root_path, target_root_path, config_vars): apply_config_calls.append( (source_root_path, target_root_path, config_vars)) installer = provisioner.ElasticsearchInstaller( car=team.Car(names="unit-test-car", root_path=None, config_paths=[ HOME_DIR + "/.rally/benchmarks/teams/default/my-car" ], variables={ "heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true" }), java_home="/usr/local/javas/java8", node_name="rally-node-0", node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest", all_node_ips=["10.17.22.22", "10.17.22.23"], all_node_names=["rally-node-0", "rally-node-1"], ip="10.17.22.23", http_port=9200) p = provisioner.BareProvisioner( cluster_settings={"indices.query.bool.max_clause_count": 50000}, es_installer=installer, plugin_installers=[ provisioner.PluginInstaller( BareProvisionerTests.MockRallyTeamXPackPlugin(), java_home="/usr/local/javas/java8", hook_handler_class=BareProvisionerTests.NoopHookHandler) ], distribution_version="6.3.0", apply_config=null_apply_config) node_config = p.prepare( {"elasticsearch": "/opt/elasticsearch-6.3.0.tar.gz"}) self.assertEqual({}, node_config.car_env) self.assertEqual("8", node_config.car_runtime_jdks) self.assertEqual("/opt/elasticsearch-6.3.0", node_config.binary_path) self.assertEqual(["/opt/elasticsearch-6.3.0/data"], node_config.data_paths) self.assertEqual(1, len(apply_config_calls)) source_root_path, target_root_path, config_vars = apply_config_calls[0] self.assertEqual(HOME_DIR + "/.rally/benchmarks/teams/default/my-car", source_root_path) self.assertEqual("/opt/elasticsearch-6.3.0", target_root_path) self.maxDiff = None self.assertEqual( { "cluster_settings": { "indices.query.bool.max_clause_count": 50000, "plugin.mandatory": ["x-pack"] }, "heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true", "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/opt/elasticsearch-6.3.0/data"], "log_path": HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server", "heap_dump_path": HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump", "node_ip": "10.17.22.23", "network_host": "10.17.22.23", "http_port": "9200", "transport_port": "9300", "all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]", "all_node_names": "[\"rally-node-0\",\"rally-node-1\"]", "minimum_master_nodes": 2, "install_root_path": "/opt/elasticsearch-6.3.0", "plugin_name": "x-pack-security", "xpack_security_enabled": True }, config_vars)
def test_prepare_without_plugins(self, mock_rm, mock_ensure_dir, mock_decompress): apply_config_calls = [] def null_apply_config(source_root_path, target_root_path, config_vars): apply_config_calls.append( (source_root_path, target_root_path, config_vars)) installer = provisioner.ElasticsearchInstaller( car=team.Car(names="unit-test-car", root_path=None, config_paths=[ HOME_DIR + "/.rally/benchmarks/teams/default/my-car" ], variables={ "heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true" }), java_home="/usr/local/javas/java8", node_name="rally-node-0", node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest", all_node_ips=["10.17.22.22", "10.17.22.23"], all_node_names=["rally-node-0", "rally-node-1"], ip="10.17.22.23", http_port=9200) p = provisioner.BareProvisioner(es_installer=installer, plugin_installers=[], apply_config=null_apply_config) node_config = p.prepare( {"elasticsearch": "/opt/elasticsearch-5.0.0.tar.gz"}) self.assertEqual("8", node_config.car_runtime_jdks) self.assertEqual("/opt/elasticsearch-5.0.0", node_config.binary_path) self.assertEqual(["/opt/elasticsearch-5.0.0/data"], node_config.data_paths) self.assertEqual(1, len(apply_config_calls)) source_root_path, target_root_path, config_vars = apply_config_calls[0] self.assertEqual(HOME_DIR + "/.rally/benchmarks/teams/default/my-car", source_root_path) self.assertEqual("/opt/elasticsearch-5.0.0", target_root_path) self.assertEqual( { "cluster_settings": {}, "heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true", "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/opt/elasticsearch-5.0.0/data"], "log_path": HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server", "heap_dump_path": HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump", "node_ip": "10.17.22.23", "network_host": "10.17.22.23", "http_port": "9200", "transport_port": "9300", "all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]", "all_node_names": "[\"rally-node-0\",\"rally-node-1\"]", "minimum_master_nodes": 2, "install_root_path": "/opt/elasticsearch-5.0.0" }, config_vars)
def test_prepare_distribution_lt_63_with_plugins(self, mock_rm, mock_ensure_dir, mock_install, mock_decompress): """ Test that plugin.mandatory is set to the specific plugin name (e.g. `x-pack-security`) and not the meta plugin name (e.g. `x-pack`) for Elasticsearch <6.3 See: https://github.com/elastic/elasticsearch/pull/28710 """ apply_config_calls = [] def null_apply_config(source_root_path, target_root_path, config_vars): apply_config_calls.append( (source_root_path, target_root_path, config_vars)) installer = provisioner.ElasticsearchInstaller( car=team.Car( names="unit-test-car", root_path=None, config_paths=[ HOME_DIR + "/.rally/benchmarks/teams/default/my-car" ], variables={ "heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true", }, ), java_home="/usr/local/javas/java8", node_name="rally-node-0", node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest", all_node_ips=["10.17.22.22", "10.17.22.23"], all_node_names=["rally-node-0", "rally-node-1"], ip="10.17.22.23", http_port=9200, ) p = provisioner.BareProvisioner( es_installer=installer, plugin_installers=[ provisioner.PluginInstaller( self.MockRallyTeamXPackPlugin(), java_home="/usr/local/javas/java8", hook_handler_class=self.NoopHookHandler, ) ], distribution_version="6.2.3", apply_config=null_apply_config, ) node_config = p.prepare( {"elasticsearch": "/opt/elasticsearch-5.0.0.tar.gz"}) assert node_config.car_runtime_jdks == "8" assert node_config.binary_path == "/opt/elasticsearch-5.0.0" assert node_config.data_paths == ["/opt/elasticsearch-5.0.0/data"] assert len(apply_config_calls) == 1 source_root_path, target_root_path, config_vars = apply_config_calls[0] assert source_root_path == HOME_DIR + "/.rally/benchmarks/teams/default/my-car" assert target_root_path == "/opt/elasticsearch-5.0.0" self.maxDiff = None assert config_vars == { "cluster_settings": { "plugin.mandatory": ["x-pack-security"] }, "heap": "4g", "runtime.jdk": "8", "runtime.jdk.bundled": "true", "cluster_name": "rally-benchmark", "node_name": "rally-node-0", "data_paths": ["/opt/elasticsearch-5.0.0/data"], "log_path": HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server", "heap_dump_path": HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump", "node_ip": "10.17.22.23", "network_host": "10.17.22.23", "http_port": "9200", "transport_port": "9300", "all_node_ips": '["10.17.22.22","10.17.22.23"]', "all_node_names": '["rally-node-0","rally-node-1"]', "minimum_master_nodes": 2, "install_root_path": "/opt/elasticsearch-5.0.0", "plugin_name": "x-pack-security", "xpack_security_enabled": True, }