def test_hosts(ansible_playbook, workload_stop_nodes, managed_cluster):
"""
Check that Grafana panel *Hosts* is showing correct values.
"""
if managed_cluster["short_name"]:
cluster_identifier = managed_cluster["short_name"]
else:
cluster_identifier = managed_cluster["integration_id"]
grafana = grafanaapi.GrafanaApi()
graphite = graphiteapi.GraphiteApi()
hosts_panel = grafana.get_panel("Hosts",
row_title="At-a-glance",
dashboard="cluster-dashboard")
"""
:step:
Send **GET** request to ``GRAPHITE/render?target=[target]&format=json``
where [target] is part of uri obtained from previous GRAFANA call.
There should be target for statuses of a hosts.
Compare number of hosts from Graphite with value retrieved from
``workload_stop_nodes`` fixture.
:result:
JSON structure containing data related to hosts status is similar
to values set by ``workload_stop_nodes`` fixture in given time.
"""
# get graphite target pointing at data containing numbers of hosts
targets = grafana.get_panel_chart_targets(hosts_panel, cluster_identifier)
targets_used = (targets[0][0], targets[1][0], targets[2][0])
targets_expected = ('nodes_count.total', 'nodes_count.up',
'nodes_count.down')
for idx, target in enumerate(targets_used):
pytest.check(
target.endswith(targets_expected[idx]),
"There is used target that ends with `{}`".format(
targets_expected[idx]))
# make sure that all data in graphite are saved
time.sleep(3)
# check value *Total* of hosts
graphite.compare_data_mean(
workload_stop_nodes["result"], (targets_used[0], ),
workload_stop_nodes["start"],
workload_stop_nodes["end"],
divergence=1,
issue="https://bugzilla.redhat.com/show_bug.cgi?id=1687333")
# check value *Up* of hosts
graphite.compare_data_mean(
0.0, (targets_used[1], ),
workload_stop_nodes["start"],
workload_stop_nodes["end"],
divergence=1,
issue="https://bugzilla.redhat.com/show_bug.cgi?id=1687333")
# check value *Down* of hosts
graphite.compare_data_mean(
workload_stop_nodes["result"], (targets_used[2], ),
workload_stop_nodes["start"],
workload_stop_nodes["end"],
divergence=1,
issue="https://bugzilla.redhat.com/show_bug.cgi?id=1687333")
def test_memory_available(
ansible_playbook,
workload_memory_utilization,
managed_cluster):
"""
Check that Grafana panel *Memory Available* is showing correct values.
"""
# TODO(fbalak): get this number dynamically
# number of samples from graphite target per minute
if managed_cluster["short_name"]:
cluster_identifier = managed_cluster["short_name"]
else:
cluster_identifier = managed_cluster["integration_id"]
grafana = grafanaapi.GrafanaApi()
graphite = graphiteapi.GraphiteApi()
memory_panel = grafana.get_panel(
"Memory Available",
row_title="At-a-Glance",
dashboard="host-dashboard")
"""
:step:
Send **GET** request to ``GRAPHITE/render?target=[target]&format=json``
where [target] is part of uri obtained from previous GRAFANA call.
There should be target for memory utilization of a host.
Compare number of hosts from Graphite with value retrieved from
``workload_memory_utilization`` fixture.
:result:
JSON structure containing data related to memory utilization is similar
to values set by ``workload_memory_utilization`` fixture in given time.
"""
# get graphite target pointing at data containing number of host
targets = grafana.get_panel_chart_targets(memory_panel, cluster_identifier)
LOGGER.debug(targets)
targets_used = (targets[0][0], targets[0][1])
for key, target_expected in enumerate((
"memory.memory-cached",
"memory.memory-free")):
pytest.check(
targets_used[key].endswith(target_expected),
"There is used target that ends with `{}`".format(target_expected))
# make sure that all data in graphite are saved
time.sleep(2)
expected_available_mem = (
1 - workload_memory_utilization["result"]/100) * int(
workload_memory_utilization['metadata']['total_memory'])
graphite.compare_data_mean(
expected_available_mem,
targets_used,
workload_memory_utilization["start"],
workload_memory_utilization["end"],
divergence=0.15*expected_available_mem)
def test_capacity_available(ansible_playbook, workload_capacity_utilization,
managed_cluster, gluster_volume):
"""
Check that Grafana panel *Capacity Available* is showing
correct values.
"""
if managed_cluster["short_name"]:
cluster_identifier = managed_cluster["short_name"]
else:
cluster_identifier = managed_cluster["integration_id"]
grafana = grafanaapi.GrafanaApi()
graphite = graphiteapi.GraphiteApi()
capacity_panel = grafana.get_panel("Capacity Available",
row_title="Capacity",
dashboard="volume-dashboard")
"""
:step:
Send **GET** request to ``GRAPHITE/render?target=[target]&format=json``
where [target] is part of uri obtained from previous GRAFANA call.
There should be target for capacity utilization of a host.
Compare number of hosts from Graphite with value retrieved from
``workload_capacity_utilization`` fixture.
:result:
JSON structure containing data related to capacity utilization is similar
to values set by ``workload_capacity_utilization`` fixture in given time.
"""
# get graphite target pointing at data containing number of host
targets = grafana.get_panel_chart_targets(
capacity_panel, cluster_identifier,
workload_capacity_utilization["metadata"]["volume_name"])
targets_used = (targets[0][0], targets[0][1])
for key, target_expected in enumerate(
("usable_capacity", "used_capacity")):
pytest.check(
targets_used[key].endswith(target_expected),
"There is used target that ends with `{}`".format(target_expected))
# make sure that all data in graphite are saved
time.sleep(2)
expected_available = workload_capacity_utilization["metadata"][
"total_capacity"] * (1 -
workload_capacity_utilization["result"] * 0.01)
divergence = workload_capacity_utilization["metadata"][
"total_capacity"] * 0.05
graphite.compare_data_mean(expected_available,
targets_used,
workload_capacity_utilization["start"],
workload_capacity_utilization["end"],
divergence=divergence,
operation='diff')
def test_cpu_utilization(
ansible_playbook, workload_cpu_utilization, managed_cluster):
"""
Check that Grafana panel *CPU Utilization* is showing correct values.
"""
# TODO(fbalak): get this number dynamically
# number of samples from graphite target per minute
if managed_cluster["short_name"]:
cluster_identifier = managed_cluster["short_name"]
else:
cluster_identifier = managed_cluster["integration_id"]
grafana = grafanaapi.GrafanaApi()
graphite = graphiteapi.GraphiteApi()
cpu_panel = grafana.get_panel(
"CPU Utilization",
row_title="At-a-Glance",
dashboard="host-dashboard")
"""
:step:
Send **GET** request to ``GRAPHITE/render?target=[target]&format=json``
where [target] is part of uri obtained from previous GRAFANA call.
There should be target for CPU utilization of a host.
Compare number of hosts from Graphite with value retrieved from
``workload_cpu_utilization`` fixture.
:result:
JSON structure containing data related to CPU utilization is similar
to values set by ``workload_cpu_utilization`` fixture in given time.
"""
# get graphite target pointing at data containing number of host
targets = grafana.get_panel_chart_targets(cpu_panel, cluster_identifier)
pytest.check(
[t.split(".")[-1] for t in targets[-1]] == ["percent-user", "percent-system"],
"The panel CPU Utilization is composed of user and system parts")
targets_used = (targets[-1][0],)
# make sure that all data in graphite are saved
time.sleep(2)
graphite.compare_data_mean(
workload_cpu_utilization["result"],
targets_used,
workload_cpu_utilization["start"],
workload_cpu_utilization["end"])
def test_swap_utilization(
ansible_playbook, workload_swap_utilization, managed_cluster):
"""
Check that Grafana panel *Swap Utilization* is showing correct values.
"""
if managed_cluster["short_name"]:
cluster_identifier = managed_cluster["short_name"]
else:
cluster_identifier = managed_cluster["integration_id"]
grafana = grafanaapi.GrafanaApi()
graphite = graphiteapi.GraphiteApi()
swap_panel = grafana.get_panel(
"Swap Utilization",
row_title="At-a-Glance",
dashboard="host-dashboard")
"""
:step:
Send **GET** request to ``GRAPHITE/render?target=[target]&format=json``
where [target] is part of uri obtained from previous GRAFANA call.
There should be target for swap utilization of a host.
Compare number of hosts from Graphite with value retrieved from
``workload_swap_utilization`` fixture.
:result:
JSON structure containing data related to swap utilization is similar
to values set by ``workload_swap_utilization`` fixture in given time.
"""
# get graphite target pointing at data containing number of host
targets = grafana.get_panel_chart_targets(swap_panel, cluster_identifier)
target_used = targets[0][0]
target_expected = 'swap.percent-used'
pytest.check(
target_used.endswith(target_expected),
"There is used target that ends with `{}`".format(target_expected))
# make sure that all data in graphite are saved
time.sleep(2)
graphite.compare_data_mean(
workload_swap_utilization["result"],
(target_used,),
workload_swap_utilization["start"],
workload_swap_utilization["end"],
divergence=15)
def test_cluster_unmanage_valid(valid_session_credentials, cluster_reuse,
valid_trusted_pool_reuse):
"""@pylatest api/gluster.cluster_unmanage
.. test_step:: 1
Check that tested cluster is correctly managed by Tendrl.
.. test_result:: 1
There is in Tendrl ``"is_managed":"yes"`` for cluster with id [cluster_id].
Graphite contains data related to health of tested cluster.
"""
tendrl_api = glusterapi.TendrlApiGluster(auth=valid_session_credentials)
graphite_api = graphiteapi.GraphiteApi()
cluster_id = cluster_reuse["cluster_id"]
pytest.check(cluster_id is not None,
"Cluster id is: {}".format(cluster_id))
pytest.check(
cluster_reuse["is_managed"] == "yes",
"is_managed: {}\nThere should be ``yes``.".format(
cluster_reuse["is_managed"]))
# graphite target uses short name if it is set
if cluster_reuse["short_name"]:
cluster_target_id = cluster_reuse["short_name"]
else:
cluster_target_id = cluster_reuse["cluster_id"]
# it takes 15 minutes to refresh data Host status panel
for i in range(31):
cluster_health = graphite_api.get_datapoints(
target="tendrl.clusters.{}.status".format(cluster_target_id))
if cluster_health:
break
else:
time.sleep(30)
pytest.check(
cluster_health, """graphite health of cluster {}: {}
There should be related data.""".format(cluster_id, cluster_health))
"""@pylatest api/gluster.cluster_unmanage
.. test_step:: 2
Send POST request to Tendrl API ``APIURL/clusters/:cluster_id/unmanage``.
.. test_result:: 2
Server should return response in JSON format:
{
"job_id": job_id
}
Return code should be **202**
with data ``{"message": "Accepted"}``.
"""
job_id = tendrl_api.unmanage_cluster(cluster_id)["job_id"]
tendrl_api.wait_for_job_status(job_id)
"""@pylatest api/gluster.cluster_unmanage
.. test_step:: 3
Check that tested cluster is correctly managed by Tendrl.
.. test_result:: 3
There is in Tendrl ``"is_managed": "no"`` for cluster with id [cluster_id].
Graphite contains no data related to health of tested cluster.
"""
# TODO(fbalak) remove this workaround when BZ 1589321 is resolved
for i in range(15):
cluster_list = tendrl_api.get_cluster_list()
if len(cluster_list) > 0:
break
else:
time.sleep(10)
assert cluster_list
for cluster in cluster_list:
if cluster["cluster_id"] == cluster_id:
unmanaged_cluster = cluster
break
pytest.check(
unmanaged_cluster["is_managed"] == "no",
"is_managed: {}\nThere should be ``no``.".format(
unmanaged_cluster["is_managed"]))
cluster_health = graphite_api.get_datapoints(
target="tendrl.clusters.{}.status".format(cluster_target_id))
pytest.check(
cluster_health == [], """graphite health of cluster {}: `{}`
There should be `[]`.""".format(cluster_id, cluster_health))
"""@pylatest api/gluster.cluster_unmanage
.. test_step:: 4
Reimport cluster and check that tested cluster is correctly managed by Tendrl.
.. test_result:: 4
There is ``"is_managed": "yes"`` in Tendrl for cluster with id [cluster_id].
"""
job_id = tendrl_api.import_cluster(cluster_id)["job_id"]
tendrl_api.wait_for_job_status(job_id)
for cluster in tendrl_api.get_cluster_list():
if cluster["cluster_id"] == cluster_id:
managed_cluster = cluster
break
pytest.check(
managed_cluster["is_managed"] == "yes",
"is_managed: {}\nThere should be ``yes``.".format(
managed_cluster["is_managed"]))
def test_status(cluster_reuse):
"""@pylatest grafana/status
API-grafana: hosts
*******************
.. test_metadata:: author [email protected]
Description
===========
Check that Grafana panel *Hosts* is showing correct values.
"""
if cluster_reuse["short_name"]:
cluster_identifier = cluster_reuse["short_name"]
else:
cluster_identifier = cluster_reuse["integration_id"]
gluster = GlusterCommon()
states = gluster.get_cluster_hosts_connection_states(
pytest.config.getini("usm_cluster_member"))
grafana = grafanaapi.GrafanaApi()
graphite = graphiteapi.GraphiteApi()
"""@pylatest grafana/hosts
.. test_step:: 1
Send **GET** request to:
``GRAFANA/dashboards/db/cluster-dashboard``.
.. test_result:: 1
JSON structure containing data related to layout is returned.
"""
layout = grafana.get_dashboard("cluster-dashboard")
assert len(layout) > 0
"""
structure of grafana data:
{
...
"dashboard":
"rows":
[
...
{
...
"displayName": ...,
"panels":
[
...
{
...
targets:
[
...
{
"refid": ...,
"target": ...,
...
}
...
]
...
}
...
]
...
}
...
]
"""
dashboard_rows = layout["dashboard"]["rows"]
assert len(dashboard_rows) > 0
# first row contains some links links and navigation
assert len(dashboard_rows[0]) > 0
# second row contains At-a-glance panels
assert len(dashboard_rows[1]) > 0
# third row contains Top Consumers panels
assert len(dashboard_rows[2]) > 0
# fourth row contains Status panels
assert len(dashboard_rows[3]) > 0
panels = dashboard_rows[1]["panels"]
panel = [
panel for panel in panels
if "displayName" in panel and panel["displayName"] == "Hosts"
]
"""@pylatest grafana/hosts
.. test_step:: 2
Send **GET** request to ``GRAPHITE/render?target=[target]&format=json``
where [target] is part of uri obtained from previous GRAFANA call.
There should be target for Total number of hosts.
Compare number of hosts from Graphite with value retrieved from gluster
command.
.. test_result:: 2
JSON structure containing data relatedo Total host count with last
value coresponding with output of gluster command.
"""
# get graphite target pointing at data containing number of host
target = panel[0]["targets"][0]["target"]
target = target.replace("$cluster_id", cluster_identifier)
LOGGER.debug("Total hosts target: {}".format(target))
g_total = graphite.get_datapoints(target)[-1][0]
pytest.check(
g_total == len(states),
"Number of total hosts in graphite ({}) should be {}".format(
g_total, len(states)))
"""@pylatest grafana/hosts
.. test_step:: 3
Send **GET** request to ``GRAPHITE/render?target=[target]&format=json``
where [target] is part of uri obtained from previous GRAFANA call.
There should be target for number of hosts that are Up.
Compare number of hosts from Graphite with value retrieved from gluster
command.
.. test_result:: 3
JSON structure containing data related to Up host count with last
value coresponding with output of gluster command.
"""
# get graphite target pointing at data containing number of host that are
# up
target = panel[0]["targets"][1]["target"]
target = target.replace("$cluster_id", cluster_identifier)
LOGGER.debug("Up hosts target: {}".format(target))
g_up = graphite.get_datapoints(target)[-1][0]
real_up = []
for host in states.keys():
if states[host]:
real_up.append(host)
pytest.check(
g_up == len(real_up),
"Number of hosts that are up in graphite ({}) should be {}".format(
g_up, len(real_up)))
"""@pylatest grafana/hosts
.. test_step:: 4
Send **GET** request to ``GRAPHITE/render?target=[target]&format=json``
where [target] is part of uri obtained from previous GRAFANA call.
There should be target for number of hosts that are Down.
Compare number of hosts from Graphite with value retrieved from gluster
command.
.. test_result:: 4
JSON structure containing data related to Down host count with last
value coresponding with output of gluster command.
"""
# get graphite target pointing at data containing number of host that are
# down
target = panel[0]["targets"][2]["target"]
target = target.replace("$cluster_id", cluster_identifier)
LOGGER.debug("Down hosts target: {}".format(target))
g_down = graphite.get_datapoints(target)[-1][0]
real_down = []
for host in states.keys():
if not states[host]:
real_down.append(host)
pytest.check(
g_down == len(real_down),
"Number of hosts that are down in graphite ({}) should be {}".format(
g_down, len(real_down)))