def start(self, node_configurations=None): hosts = self.cfg.opts("client", "hosts").default client_options = self.cfg.opts("client", "options").default es = self.client_factory(hosts, client_options).create() # cannot enable custom telemetry devices here t = telemetry.Telemetry(devices=[ # This is needed to actually populate the nodes telemetry.ClusterMetaDataInfo(es), # will gather node specific meta-data for all nodes telemetry.ExternalEnvironmentInfo(es, self.metrics_store), ]) # We create a pseudo-cluster here to get information about all nodes. # cluster nodes will be populated by the external environment info telemetry device. We cannot know this upfront. c = cluster.Cluster(hosts, [], t) user_defined_version = self.cfg.opts("mechanic", "distribution.version", mandatory=False) distribution_version = es.info()["version"]["number"] if not user_defined_version or user_defined_version.strip() == "": self.logger.info("Distribution version was not specified by user. Rally-determined version is [%s]", distribution_version) self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version) elif user_defined_version != distribution_version: console.warn( "Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster." % (user_defined_version, distribution_version), logger=self.logger) t.attach_to_cluster(c) return c.nodes
def start(self, car=None, binary=None, data_paths=None): console.println(ExternalLauncher.BOGUS_RESULTS_WARNING) hosts = self.cfg.opts("client", "hosts") client_options = self.cfg.opts("client", "options") es = self.client_factory(hosts, client_options).create() # cannot enable custom telemetry devices here t = telemetry.Telemetry(devices=[ telemetry.ClusterMetaDataInfo(es), telemetry.ExternalEnvironmentInfo(es, self.metrics_store), telemetry.NodeStats(es, self.metrics_store), telemetry.IndexStats(es, self.metrics_store) ]) # cluster nodes will be populated by the external environment info telemetry device. We cannot know this upfront. c = cluster.Cluster(hosts, [], t) user_defined_version = self.cfg.opts("mechanic", "distribution.version", mandatory=False) distribution_version = es.info()["version"]["number"] if not user_defined_version or user_defined_version.strip() == "": logger.info("Distribution version was not specified by user. Rally-determined version is [%s]" % distribution_version) self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version) elif user_defined_version != distribution_version: console.warn( "Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster." % (user_defined_version, distribution_version), logger=logger) t.attach_to_cluster(c) return c
def test_stores_cluster_level_metrics_on_attach(self, metrics_store_add_meta_info): nodes_stats = { "nodes": { "FCFjozkeTiOpN-SI88YEcg": { "name": "rally0", "host": "127.0.0.1" } } } nodes_info = { "nodes": { "FCFjozkeTiOpN-SI88YEcg": { "name": "rally0", "host": "127.0.0.1", "attributes": { "az": "us_east1" }, "os": { "name": "Mac OS X", "version": "10.11.4", "available_processors": 8 }, "jvm": { "version": "1.8.0_74", "vm_vendor": "Oracle Corporation" } } } } cluster_info = { "version": { "build_hash": "253032b", "number": "5.0.0" } } client = Client(nodes=SubClient(stats=nodes_stats, info=nodes_info), info=cluster_info) metrics_store = metrics.EsMetricsStore(self.cfg) env_device = telemetry.ExternalEnvironmentInfo(client, metrics_store) t = telemetry.Telemetry(devices=[env_device]) t.attach_to_cluster(cluster.Cluster([], [], t)) calls = [ mock.call(metrics.MetaInfoScope.cluster, None, "source_revision", "253032b"), mock.call(metrics.MetaInfoScope.cluster, None, "distribution_version", "5.0.0"), mock.call(metrics.MetaInfoScope.node, "rally0", "node_name", "rally0"), mock.call(metrics.MetaInfoScope.node, "rally0", "host_name", "127.0.0.1"), mock.call(metrics.MetaInfoScope.node, "rally0", "os_name", "Mac OS X"), mock.call(metrics.MetaInfoScope.node, "rally0", "os_version", "10.11.4"), mock.call(metrics.MetaInfoScope.node, "rally0", "cpu_logical_cores", 8), mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_vendor", "Oracle Corporation"), mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_version", "1.8.0_74"), mock.call(metrics.MetaInfoScope.node, "rally0", "attribute_az", "us_east1"), mock.call(metrics.MetaInfoScope.cluster, None, "attribute_az", "us_east1") ] metrics_store_add_meta_info.assert_has_calls(calls)
def test_fallback_when_host_not_available(self, metrics_store_add_meta_info): nodes_stats = { "nodes": { "FCFjozkeTiOpN-SI88YEcg": { "name": "rally0", } } } nodes_info = { "nodes": { "FCFjozkeTiOpN-SI88YEcg": { "name": "rally0", "os": { "name": "Mac OS X", "version": "10.11.4", "available_processors": 8 }, "jvm": { "version": "1.8.0_74", "vm_vendor": "Oracle Corporation" } } } } cluster_info = {"version": {"build_hash": "abc123"}} client = Client(cluster=SubClient(nodes_stats), nodes=SubClient(nodes_info), info=cluster_info) metrics_store = metrics.EsMetricsStore(self.cfg) env_device = telemetry.ExternalEnvironmentInfo(self.cfg, client, metrics_store) t = telemetry.Telemetry(self.cfg, devices=[env_device]) t.attach_to_cluster(cluster.Cluster([], t)) calls = [ mock.call(metrics.MetaInfoScope.cluster, None, "source_revision", "abc123"), mock.call(metrics.MetaInfoScope.node, "rally0", "node_name", "rally0"), mock.call(metrics.MetaInfoScope.node, "rally0", "host_name", "unknown"), mock.call(metrics.MetaInfoScope.node, "rally0", "os_name", "Mac OS X"), mock.call(metrics.MetaInfoScope.node, "rally0", "os_version", "10.11.4"), mock.call(metrics.MetaInfoScope.node, "rally0", "cpu_logical_cores", 8), mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_vendor", "Oracle Corporation"), mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_version", "1.8.0_74") ] metrics_store_add_meta_info.assert_has_calls(calls)
def start(self, car=None): console.println(ExternalLauncher.BOGUS_RESULTS_WARNING) hosts = self.cfg.opts("launcher", "external.target.hosts") client_options = self.cfg.opts("launcher", "client.options") # unified client config self.cfg.add(config.Scope.benchmark, "client", "hosts", hosts) self.cfg.add(config.Scope.benchmark, "client", "options", client_options) es = self.client_factory(hosts, client_options).create() t = telemetry.Telemetry( self.cfg, devices=[ telemetry.ExternalEnvironmentInfo(self.cfg, es, self.metrics_store), telemetry.NodeStats(self.cfg, es, self.metrics_store), telemetry.IndexStats(self.cfg, es, self.metrics_store) ]) c = cluster.Cluster([], t) user_defined_version = self.cfg.opts("source", "distribution.version", mandatory=False) distribution_version = es.info()["version"]["number"] if not user_defined_version or user_defined_version.strip() == "": logger.info( "Distribution version was not specified by user. Rally-determined version is [%s]" % distribution_version) self.cfg.add(config.Scope.benchmark, "source", "distribution.version", distribution_version) elif user_defined_version != distribution_version: console.println( "Warning: Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster." % (user_defined_version, distribution_version), logger=logger.warn) t.attach_to_cluster(c) return c
def test_stores_all_node_metrics_on_attach(self, metrics_store_add_meta_info): nodes_stats = { "nodes": { "FCFjozkeTiOpN-SI88YEcg": { "name": "rally0", "host": "127.0.0.1" } } } nodes_info = { "nodes": { "FCFjozkeTiOpN-SI88YEcg": { "name": "rally0", "host": "127.0.0.1", "attributes": { "az": "us_east1" }, "os": { "name": "Mac OS X", "version": "10.11.4", "available_processors": 8 }, "jvm": { "version": "1.8.0_74", "vm_vendor": "Oracle Corporation" }, "plugins": [{ "name": "ingest-geoip", "version": "5.0.0", "description": "Ingest processor that uses looksup geo data ...", "classname": "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin", "has_native_controller": False }] } } } cluster_info = { "version": { "build_hash": "253032b", "number": "5.0.0" } } client = Client(nodes=SubClient(stats=nodes_stats, info=nodes_info), info=cluster_info) metrics_store = metrics.EsMetricsStore(self.cfg) env_device = telemetry.ExternalEnvironmentInfo(client, metrics_store) t = telemetry.Telemetry(devices=[env_device]) t.attach_to_cluster(cluster.Cluster([], [], t)) calls = [ mock.call(metrics.MetaInfoScope.node, "rally0", "node_name", "rally0"), mock.call(metrics.MetaInfoScope.node, "rally0", "host_name", "127.0.0.1"), mock.call(metrics.MetaInfoScope.node, "rally0", "os_name", "Mac OS X"), mock.call(metrics.MetaInfoScope.node, "rally0", "os_version", "10.11.4"), mock.call(metrics.MetaInfoScope.node, "rally0", "cpu_logical_cores", 8), mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_vendor", "Oracle Corporation"), mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_version", "1.8.0_74"), mock.call(metrics.MetaInfoScope.node, "rally0", "plugins", ["ingest-geoip"]), # these are automatically pushed up to cluster level (additionally) if all nodes match mock.call(metrics.MetaInfoScope.cluster, None, "plugins", ["ingest-geoip"]), mock.call(metrics.MetaInfoScope.node, "rally0", "attribute_az", "us_east1"), mock.call(metrics.MetaInfoScope.cluster, None, "attribute_az", "us_east1"), ] metrics_store_add_meta_info.assert_has_calls(calls)