def start(self, node_configurations=None):
hosts = self.cfg.opts("client", "hosts").default
client_options = self.cfg.opts("client", "options").default
es = self.client_factory(hosts, client_options).create()
# cannot enable custom telemetry devices here
t = telemetry.Telemetry(devices=[
# This is needed to actually populate the nodes
telemetry.ClusterMetaDataInfo(es),
# will gather node specific meta-data for all nodes
telemetry.ExternalEnvironmentInfo(es, self.metrics_store),
])
# We create a pseudo-cluster here to get information about all nodes.
# cluster nodes will be populated by the external environment info telemetry device. We cannot know this upfront.
c = cluster.Cluster(hosts, [], t)
user_defined_version = self.cfg.opts("mechanic", "distribution.version", mandatory=False)
distribution_version = es.info()["version"]["number"]
if not user_defined_version or user_defined_version.strip() == "":
self.logger.info("Distribution version was not specified by user. Rally-determined version is [%s]", distribution_version)
self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version)
elif user_defined_version != distribution_version:
console.warn(
"Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster." %
(user_defined_version, distribution_version), logger=self.logger)
t.attach_to_cluster(c)
return c.nodes
def start(self, car=None, binary=None, data_paths=None):
console.println(ExternalLauncher.BOGUS_RESULTS_WARNING)
hosts = self.cfg.opts("client", "hosts")
client_options = self.cfg.opts("client", "options")
es = self.client_factory(hosts, client_options).create()
# cannot enable custom telemetry devices here
t = telemetry.Telemetry(devices=[
telemetry.ClusterMetaDataInfo(es),
telemetry.ExternalEnvironmentInfo(es, self.metrics_store),
telemetry.NodeStats(es, self.metrics_store),
telemetry.IndexStats(es, self.metrics_store)
])
# cluster nodes will be populated by the external environment info telemetry device. We cannot know this upfront.
c = cluster.Cluster(hosts, [], t)
user_defined_version = self.cfg.opts("mechanic", "distribution.version", mandatory=False)
distribution_version = es.info()["version"]["number"]
if not user_defined_version or user_defined_version.strip() == "":
logger.info("Distribution version was not specified by user. Rally-determined version is [%s]" % distribution_version)
self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version)
elif user_defined_version != distribution_version:
console.warn(
"Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster." %
(user_defined_version, distribution_version), logger=logger)
t.attach_to_cluster(c)
return c
def test_stores_cluster_level_metrics_on_attach(self, metrics_store_add_meta_info):
nodes_stats = {
"nodes": {
"FCFjozkeTiOpN-SI88YEcg": {
"name": "rally0",
"host": "127.0.0.1"
}
}
}
nodes_info = {
"nodes": {
"FCFjozkeTiOpN-SI88YEcg": {
"name": "rally0",
"host": "127.0.0.1",
"attributes": {
"az": "us_east1"
},
"os": {
"name": "Mac OS X",
"version": "10.11.4",
"available_processors": 8
},
"jvm": {
"version": "1.8.0_74",
"vm_vendor": "Oracle Corporation"
}
}
}
}
cluster_info = {
"version":
{
"build_hash": "253032b",
"number": "5.0.0"
}
}
client = Client(nodes=SubClient(stats=nodes_stats, info=nodes_info), info=cluster_info)
metrics_store = metrics.EsMetricsStore(self.cfg)
env_device = telemetry.ExternalEnvironmentInfo(client, metrics_store)
t = telemetry.Telemetry(devices=[env_device])
t.attach_to_cluster(cluster.Cluster([], [], t))
calls = [
mock.call(metrics.MetaInfoScope.cluster, None, "source_revision", "253032b"),
mock.call(metrics.MetaInfoScope.cluster, None, "distribution_version", "5.0.0"),
mock.call(metrics.MetaInfoScope.node, "rally0", "node_name", "rally0"),
mock.call(metrics.MetaInfoScope.node, "rally0", "host_name", "127.0.0.1"),
mock.call(metrics.MetaInfoScope.node, "rally0", "os_name", "Mac OS X"),
mock.call(metrics.MetaInfoScope.node, "rally0", "os_version", "10.11.4"),
mock.call(metrics.MetaInfoScope.node, "rally0", "cpu_logical_cores", 8),
mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_vendor", "Oracle Corporation"),
mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_version", "1.8.0_74"),
mock.call(metrics.MetaInfoScope.node, "rally0", "attribute_az", "us_east1"),
mock.call(metrics.MetaInfoScope.cluster, None, "attribute_az", "us_east1")
]
metrics_store_add_meta_info.assert_has_calls(calls)
def test_fallback_when_host_not_available(self,
metrics_store_add_meta_info):
nodes_stats = {
"nodes": {
"FCFjozkeTiOpN-SI88YEcg": {
"name": "rally0",
}
}
}
nodes_info = {
"nodes": {
"FCFjozkeTiOpN-SI88YEcg": {
"name": "rally0",
"os": {
"name": "Mac OS X",
"version": "10.11.4",
"available_processors": 8
},
"jvm": {
"version": "1.8.0_74",
"vm_vendor": "Oracle Corporation"
}
}
}
}
cluster_info = {"version": {"build_hash": "abc123"}}
client = Client(cluster=SubClient(nodes_stats),
nodes=SubClient(nodes_info),
info=cluster_info)
metrics_store = metrics.EsMetricsStore(self.cfg)
env_device = telemetry.ExternalEnvironmentInfo(self.cfg, client,
metrics_store)
t = telemetry.Telemetry(self.cfg, devices=[env_device])
t.attach_to_cluster(cluster.Cluster([], t))
calls = [
mock.call(metrics.MetaInfoScope.cluster, None, "source_revision",
"abc123"),
mock.call(metrics.MetaInfoScope.node, "rally0", "node_name",
"rally0"),
mock.call(metrics.MetaInfoScope.node, "rally0", "host_name",
"unknown"),
mock.call(metrics.MetaInfoScope.node, "rally0", "os_name",
"Mac OS X"),
mock.call(metrics.MetaInfoScope.node, "rally0", "os_version",
"10.11.4"),
mock.call(metrics.MetaInfoScope.node, "rally0",
"cpu_logical_cores", 8),
mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_vendor",
"Oracle Corporation"),
mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_version",
"1.8.0_74")
]
metrics_store_add_meta_info.assert_has_calls(calls)
def start(self, car=None):
console.println(ExternalLauncher.BOGUS_RESULTS_WARNING)
hosts = self.cfg.opts("launcher", "external.target.hosts")
client_options = self.cfg.opts("launcher", "client.options")
# unified client config
self.cfg.add(config.Scope.benchmark, "client", "hosts", hosts)
self.cfg.add(config.Scope.benchmark, "client", "options",
client_options)
es = self.client_factory(hosts, client_options).create()
t = telemetry.Telemetry(
self.cfg,
devices=[
telemetry.ExternalEnvironmentInfo(self.cfg, es,
self.metrics_store),
telemetry.NodeStats(self.cfg, es, self.metrics_store),
telemetry.IndexStats(self.cfg, es, self.metrics_store)
])
c = cluster.Cluster([], t)
user_defined_version = self.cfg.opts("source",
"distribution.version",
mandatory=False)
distribution_version = es.info()["version"]["number"]
if not user_defined_version or user_defined_version.strip() == "":
logger.info(
"Distribution version was not specified by user. Rally-determined version is [%s]"
% distribution_version)
self.cfg.add(config.Scope.benchmark, "source",
"distribution.version", distribution_version)
elif user_defined_version != distribution_version:
console.println(
"Warning: Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster."
% (user_defined_version, distribution_version),
logger=logger.warn)
t.attach_to_cluster(c)
return c
def test_stores_all_node_metrics_on_attach(self,
metrics_store_add_meta_info):
nodes_stats = {
"nodes": {
"FCFjozkeTiOpN-SI88YEcg": {
"name": "rally0",
"host": "127.0.0.1"
}
}
}
nodes_info = {
"nodes": {
"FCFjozkeTiOpN-SI88YEcg": {
"name":
"rally0",
"host":
"127.0.0.1",
"attributes": {
"az": "us_east1"
},
"os": {
"name": "Mac OS X",
"version": "10.11.4",
"available_processors": 8
},
"jvm": {
"version": "1.8.0_74",
"vm_vendor": "Oracle Corporation"
},
"plugins": [{
"name": "ingest-geoip",
"version": "5.0.0",
"description":
"Ingest processor that uses looksup geo data ...",
"classname":
"org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
"has_native_controller": False
}]
}
}
}
cluster_info = {
"version": {
"build_hash": "253032b",
"number": "5.0.0"
}
}
client = Client(nodes=SubClient(stats=nodes_stats, info=nodes_info),
info=cluster_info)
metrics_store = metrics.EsMetricsStore(self.cfg)
env_device = telemetry.ExternalEnvironmentInfo(client, metrics_store)
t = telemetry.Telemetry(devices=[env_device])
t.attach_to_cluster(cluster.Cluster([], [], t))
calls = [
mock.call(metrics.MetaInfoScope.node, "rally0", "node_name",
"rally0"),
mock.call(metrics.MetaInfoScope.node, "rally0", "host_name",
"127.0.0.1"),
mock.call(metrics.MetaInfoScope.node, "rally0", "os_name",
"Mac OS X"),
mock.call(metrics.MetaInfoScope.node, "rally0", "os_version",
"10.11.4"),
mock.call(metrics.MetaInfoScope.node, "rally0",
"cpu_logical_cores", 8),
mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_vendor",
"Oracle Corporation"),
mock.call(metrics.MetaInfoScope.node, "rally0", "jvm_version",
"1.8.0_74"),
mock.call(metrics.MetaInfoScope.node, "rally0", "plugins",
["ingest-geoip"]),
# these are automatically pushed up to cluster level (additionally) if all nodes match
mock.call(metrics.MetaInfoScope.cluster, None, "plugins",
["ingest-geoip"]),
mock.call(metrics.MetaInfoScope.node, "rally0", "attribute_az",
"us_east1"),
mock.call(metrics.MetaInfoScope.cluster, None, "attribute_az",
"us_east1"),
]
metrics_store_add_meta_info.assert_has_calls(calls)
