def test_monitoring_enabled():
"""
OCS Monitoring is enabled after OCS installation (which is why this test
has a post deployment marker) by asking for values of one ceph and one
noobaa related metrics.
"""
prometheus = PrometheusAPI()
# ask for values of ceph_pool_stored metric
logger.info("Checking that ceph data are provided in OCS monitoring")
result = prometheus.query('ceph_pool_stored')
msg = "check that we actually received some values for a ceph query"
assert len(result) > 0, msg
for metric in result:
_, value = metric['value']
assert_msg = "number of bytes in a pool isn't a positive integer or zero"
assert int(value) >= 0, assert_msg
# additional check that values makes at least some sense
logger.info(
"Checking that size of ceph_pool_stored result matches number of pools")
ct_pod = pod.get_ceph_tools_pod()
ceph_pools = ct_pod.exec_ceph_cmd("ceph osd pool ls")
assert len(result) == len(ceph_pools)
# again for a noobaa metric
logger.info("Checking that MCG/NooBaa data are provided in OCS monitoring")
result = prometheus.query('NooBaa_bucket_status')
msg = "check that we actually received some values for a MCG/NooBaa query"
assert len(result) > 0, msg
for metric in result:
_, value = metric['value']
assert int(value) >= 0, "bucket status isn't a positive integer or zero"
def test_ceph_rbd_metrics_available():
"""
Ceph RBD metrics should be provided via OCP Prometheus as well.
See also: https://ceph.com/rbd/new-in-nautilus-rbd-performance-monitoring/
"""
# this is not a full list, but it is enough to check whether we have
# rbd metrics available via OCS monitoring
list_of_metrics = [
"ceph_rbd_write_ops",
"ceph_rbd_read_ops",
"ceph_rbd_write_bytes",
"ceph_rbd_read_bytes",
"ceph_rbd_write_latency_sum",
"ceph_rbd_write_latency_count"]
prometheus = PrometheusAPI()
list_of_metrics_without_results = []
for metric in list_of_metrics:
result = prometheus.query(metric)
# check that we actually received some values
if len(result) == 0:
logger.error(f"failed to get results for {metric}")
list_of_metrics_without_results.append(metric)
msg = (
"OCS Monitoring should provide some value(s) for tested rbd metrics, "
"so that the list of metrics without results is empty.")
assert list_of_metrics_without_results == [], msg
def test_monitoring_enabled():
"""
OCS Monitoring is enabled after OCS installation (which is why this test
has a post deployment marker) by asking for values of one ceph and one
noobaa related metrics.
"""
prometheus = PrometheusAPI()
if (
storagecluster_independent_check()
and float(config.ENV_DATA["ocs_version"]) < 4.6
):
logger.info(
f"Skipping ceph metrics because it is not enabled for external "
f"mode for OCS {float(config.ENV_DATA['ocs_version'])}"
)
else:
# ask for values of ceph_pool_stored metric
logger.info("Checking that ceph data are provided in OCS monitoring")
result = prometheus.query("ceph_pool_stored")
msg = "check that we actually received some values for a ceph query"
assert len(result) > 0, msg
for metric in result:
_, value = metric["value"]
assert_msg = "number of bytes in a pool isn't a positive integer or zero"
assert int(value) >= 0, assert_msg
# additional check that values makes at least some sense
logger.info(
"Checking that size of ceph_pool_stored result matches number of pools"
)
ct_pod = pod.get_ceph_tools_pod()
ceph_pools = ct_pod.exec_ceph_cmd("ceph osd pool ls")
assert len(result) == len(ceph_pools)
# again for a noobaa metric
logger.info("Checking that MCG/NooBaa data are provided in OCS monitoring")
result = prometheus.query("NooBaa_bucket_status")
msg = "check that we actually received some values for a MCG/NooBaa query"
assert len(result) > 0, msg
for metric in result:
_, value = metric["value"]
assert int(value) >= 0, "bucket status isn't a positive integer or zero"
class ClusterLoad:
"""
A class for cluster load functionalities
"""
def __init__(
self, project_factory=None, pvc_factory=None, sa_factory=None,
pod_factory=None, target_percentage=None
):
"""
Initializer for ClusterLoad
Args:
project_factory (function): A call to project_factory function
pvc_factory (function): A call to pvc_factory function
sa_factory (function): A call to service_account_factory function
pod_factory (function): A call to pod_factory function
target_percentage (float): The percentage of cluster load that is
required. The value should be greater than 0.1 and smaller than 0.95
"""
self.prometheus_api = PrometheusAPI()
self.pvc_factory = pvc_factory
self.sa_factory = sa_factory
self.pod_factory = pod_factory
self.target_percentage = target_percentage
self.cluster_limit = None
self.dc_objs = list()
self.pvc_objs = list()
self.previous_iops = None
self.current_iops = None
self.rate = None
self.pvc_size = int(get_osd_pods_memory_sum() * 0.5)
self.sleep_time = 45
self.target_pods_number = None
if project_factory:
project_name = f"{defaults.BG_LOAD_NAMESPACE}-{uuid4().hex[:5]}"
self.project = project_factory(project_name=project_name)
def increase_load(self, rate, wait=True):
"""
Create a PVC, a service account and a DeploymentConfig of FIO pod
Args:
rate (str): FIO 'rate' value (e.g. '20M')
wait (bool): True for waiting for IO to kick in on the
newly created pod, False otherwise
"""
pvc_obj = self.pvc_factory(
interface=constants.CEPHBLOCKPOOL, project=self.project,
size=self.pvc_size, volume_mode=constants.VOLUME_MODE_BLOCK,
)
self.pvc_objs.append(pvc_obj)
service_account = self.sa_factory(pvc_obj.project)
# Set new arguments with the updated file size to be used for
# DeploymentConfig of FIO pod creation
fio_dc_data = templating.load_yaml(constants.FIO_DC_YAML)
args = fio_dc_data.get('spec').get('template').get(
'spec'
).get('containers')[0].get('args')
new_args = [
x for x in args if not x.startswith('--filesize=') and not x.startswith('--rate=')
]
io_file_size = f"{self.pvc_size * 1000 - 200}M"
new_args.append(f"--filesize={io_file_size}")
new_args.append(f"--rate={rate}")
dc_obj = self.pod_factory(
pvc=pvc_obj, pod_dict_path=constants.FIO_DC_YAML,
raw_block_pv=True, deployment_config=True,
service_account=service_account, command_args=new_args
)
self.dc_objs.append(dc_obj)
if wait:
logger.info(
f"Waiting {self.sleep_time} seconds for IO to kick-in on the newly "
f"created FIO pod {dc_obj.name}"
)
time.sleep(self.sleep_time)
def decrease_load(self, wait=True):
"""
Delete DeploymentConfig with its pods and the PVC. Then, wait for the
IO to be stopped
Args:
wait (bool): True for waiting for IO to drop after the deletion
of the FIO pod, False otherwise
"""
dc_name = self.dc_objs[-1].name
self.dc_objs[-1].delete()
self.dc_objs[-1].ocp.wait_for_delete(dc_name)
self.dc_objs.remove(self.dc_objs[-1])
self.pvc_objs[-1].delete()
self.pvc_objs[-1].ocp.wait_for_delete(self.pvc_objs[-1].name)
self.pvc_objs.remove(self.pvc_objs[-1])
if wait:
logger.info(
f"Waiting {self.sleep_time} seconds for IO to drop after "
f"the deletion of {dc_name}"
)
time.sleep(self.sleep_time)
def increase_load_and_print_data(self, rate, wait=True):
"""
Increase load and print data
Args:
rate (str): FIO 'rate' value (e.g. '20M')
wait (bool): True for waiting for IO to kick in on the
newly created pod, False otherwise
"""
self.increase_load(rate=rate, wait=wait)
self.previous_iops = self.current_iops
self.current_iops = self.calc_trim_metric_mean(metric=constants.IOPS_QUERY)
msg = f"Current: {self.current_iops:.2f} || Previous: {self.previous_iops:.2f}"
logger.info(f"IOPS:{wrap_msg(msg)}")
self.print_metrics()
def reach_cluster_load_percentage(self):
"""
Reach the cluster limit and then drop to the given target percentage.
The number of pods needed for the desired target percentage is determined by
creating pods one by one, while examining the cluster latency. Once the latency
is greater than 250 ms and it is growing exponentially, it means that
the cluster limit has been reached.
Then, dropping to the target percentage by deleting all pods and re-creating
ones with smaller value of FIO 'rate' param.
This leaves the number of pods needed running IO for cluster load to
be around the desired percentage.
"""
if not self.target_percentage:
logger.warning("The target percentage was not provided. Breaking")
return
if not 0.1 < self.target_percentage < 0.95:
logger.warning(
f"The target percentage is {self.target_percentage * 100}% which is "
"not within the accepted range. Therefore, IO will not be started"
)
return
low_diff_counter = 0
cluster_limit = None
latency_vals = list()
time_to_wait = 60 * 30
time_before = time.time()
self.current_iops = self.get_query(query=constants.IOPS_QUERY)
# Creating FIO DeploymentConfig pods one by one, with a large value of FIO
# 'rate' arg. This in order to determine the cluster limit faster.
# Once determined, these pods will be deleted. Then, new FIO DC pods will be
# created, with a smaller value of 'rate' param. This in order to be more
# accurate with reaching the target percentage
while True:
wait = False if len(self.dc_objs) <= 1 else True
self.increase_load_and_print_data(rate='250M', wait=wait)
if self.current_iops > self.previous_iops:
cluster_limit = self.current_iops
latency = self.calc_trim_metric_mean(metric=constants.LATENCY_QUERY) * 1000
latency_vals.append(latency)
logger.info(f"Latency values: {latency_vals}")
iops_diff = (self.current_iops / self.previous_iops * 100) - 100
low_diff_counter += 1 if -15 < iops_diff < 10 else 0
cluster_used_space = get_percent_used_capacity()
if len(latency_vals) > 1 and latency > 250:
# Checking for an exponential growth. In case the latest latency sample
# value is more than 128 times the first latency value sample, we can conclude
# that the cluster limit in terms of IOPS, has been reached.
# See https://blog.docbert.org/vdbench-curve/ for more details.
# In other cases, when the first latency sample value is greater than 3 ms,
# the multiplication factor we check according to, is lower, in order to
# determine the cluster load faster.
if latency > latency_vals[0] * 2 ** 7 or (
3 < latency_vals[0] < 50 and len(latency_vals) > 5
):
logger.info(
wrap_msg("The cluster limit was determined by latency growth")
)
break
# In case the latency is greater than 2 seconds,
# most chances the limit has been reached
elif latency > 2000:
logger.info(
wrap_msg(f"The limit was determined by the high latency - {latency} ms")
)
break
# For clusters that their nodes do not meet the minimum
# resource requirements, the cluster limit is being reached
# while the latency remains low. For that, the cluster limit
# needs to be determined by the following condition of IOPS
# diff between FIO pod creation iterations
elif low_diff_counter > 3:
logger.warning(
wrap_msg(
"Limit was determined by low IOPS diff between "
f"iterations - {iops_diff:.2f}%"
)
)
break
elif time.time() > time_before + time_to_wait:
logger.warning(
wrap_msg(
"Could not determine the cluster IOPS limit within"
f"the given {time_to_wait} seconds timeout. Breaking"
)
)
break
elif cluster_used_space > 60:
logger.warning(
wrap_msg(
f"Cluster used space is {cluster_used_space}%. Could "
"not reach the cluster IOPS limit before the "
"used spaced reached 60%. Breaking"
)
)
break
self.cluster_limit = cluster_limit
logger.info(wrap_msg(f"The cluster IOPS limit is {self.cluster_limit:.2f}"))
logger.info("Deleting all DC FIO pods that have large FIO rate")
while self.dc_objs:
self.decrease_load(wait=False)
target_iops = self.cluster_limit * self.target_percentage
range_map = RangeKeyDict(
{
(0, 500): (6, 0.82, 0.4),
(500, 1000): (8, 0.84, 0.45),
(1000, 1500): (10, 0.86, 0.5),
(1500, 2000): (12, 0.88, 0.55),
(2000, 2500): (14, 0.90, 0.6),
(2500, 3000): (16, 0.92, 0.65),
(3000, 3500): (18, 0.94, 0.7),
(3500, math.inf): (20, 0.96, 0.75),
}
)
self.rate = f'{range_map[target_iops][0]}M'
# Creating the first pod of small FIO 'rate' param, to speed up the process.
# In the meantime, the load will drop, following the deletion of the
# FIO pods with large FIO 'rate' param
logger.info("Creating FIO pods, one by one, until the target percentage is reached")
self.increase_load_and_print_data(rate=self.rate)
msg = (
f"The target load, in IOPS, is: {target_iops}, which is "
f"{self.target_percentage*100}% of the {self.cluster_limit} cluster limit"
)
logger.info(wrap_msg(msg))
while self.current_iops < target_iops * range_map[target_iops][1]:
wait = False if self.current_iops < target_iops * range_map[target_iops][2] else True
self.increase_load_and_print_data(rate=self.rate, wait=wait)
msg = f"The target load, of {self.target_percentage * 100}%, has been reached"
logger.info(wrap_msg(msg))
self.target_pods_number = len(self.dc_objs)
def get_query(self, query, mute_logs=False):
"""
Get query from Prometheus and parse it
Args:
query (str): Query to be done
mute_logs (bool): True for muting the logs, False otherwise
Returns:
float: the query result
"""
now = datetime.now
timestamp = datetime.timestamp
return float(
self.prometheus_api.query(
query, str(timestamp(now())), mute_logs=mute_logs
)[0]['value'][1]
)
def calc_trim_metric_mean(self, metric, samples=5, mute_logs=False):
"""
Get the trimmed mean of a given metric
Args:
metric (str): The metric to calculate the average result for
samples (int): The number of samples to take
mute_logs (bool): True for muting the logs, False otherwise
Returns:
float: The average result for the metric
"""
vals = list()
for i in range(samples):
vals.append(round(self.get_query(metric, mute_logs), 5))
if i == samples - 1:
break
time.sleep(5)
return round(get_trim_mean(vals), 5)
def print_metrics(self, mute_logs=False):
"""
Print metrics
Args:
mute_logs (bool): True for muting the Prometheus logs, False otherwise
"""
high_latency = 200
metrics = {
"throughput": self.get_query(constants.THROUGHPUT_QUERY, mute_logs=mute_logs) * (
constants.TP_CONVERSION.get(' B/s')
),
"latency": self.get_query(constants.LATENCY_QUERY, mute_logs=mute_logs) * 1000,
"iops": self.get_query(constants.IOPS_QUERY, mute_logs=mute_logs),
"used_space": self.get_query(constants.USED_SPACE_QUERY, mute_logs=mute_logs) / 1e+9
}
limit_msg = (
f" ({metrics.get('iops') / self.cluster_limit * 100:.2f}% of the "
f"{self.cluster_limit:.2f} limit)"
) if self.cluster_limit else ""
pods_msg = f" || Number of FIO pods: {len(self.dc_objs)}" if self.dc_objs else ""
msg = (
f"Throughput: {metrics.get('throughput'):.2f} MB/s || "
f"Latency: {metrics.get('latency'):.2f} ms || "
f"IOPS: {metrics.get('iops'):.2f}{limit_msg} || "
f"Used Space: {metrics.get('used_space'):.2f} GB{pods_msg}"
)
logger.info(f"Cluster utilization:{wrap_msg(msg)}")
if metrics.get('latency') > high_latency:
logger.warning(f"Cluster latency is higher than {high_latency} ms!")
def adjust_load_if_needed(self):
"""
Dynamically adjust the IO load based on the cluster latency.
In case the latency goes beyond 250 ms, start deleting FIO pods.
Once latency drops back below 100 ms, re-create the FIO pods
to make sure that cluster load is around the target percentage
"""
latency = self.calc_trim_metric_mean(
constants.LATENCY_QUERY, mute_logs=True
)
if latency > 0.25 and len(self.dc_objs) > 0:
msg = (
f"Latency is too high - {latency * 1000:.2f} ms."
" Dropping the background load. Once the latency drops back to "
"normal, the background load will be increased back"
)
logger.warning(wrap_msg(msg))
self.decrease_load(wait=False)
if latency < 0.1 and self.target_pods_number > len(self.dc_objs):
msg = (
f"Latency is back to normal - {latency * 1000:.2f} ms. "
f"Increasing back the load"
)
logger.info(wrap_msg(msg))
self.increase_load(rate=self.rate, wait=False)
def pause_load(self):
"""
Pause the cluster load
"""
logger.info(wrap_msg("Pausing the cluster load"))
while self.dc_objs:
self.decrease_load(wait=False)
def resume_load(self):
"""
Resume the cluster load
"""
logger.info(wrap_msg("Resuming the cluster load"))
while len(self.dc_objs) < self.target_pods_number:
self.increase_load(rate=self.rate, wait=False)
def test_ceph_metrics_available():
"""
Ceph metrics as listed in KNIP-634 should be provided via OCP Prometheus.
Ceph Object Gateway https://docs.ceph.com/docs/master/radosgw/ is
deployed on on-prem platforms only (such as VMWare - see BZ 1763150),
so this test case ignores failures for ceph_rgw_* and ceph_objecter_*
metrics when running on cloud platforms (such as AWS).
"""
# this list is taken from spreadsheet attached to KNIP-634
list_of_metrics = [
"ceph_bluestore_state_aio_wait_lat_sum",
"ceph_paxos_store_state_latency_sum",
"ceph_osd_op_out_bytes",
"ceph_pg_incomplete",
"ceph_bluestore_submit_lat_sum",
"ceph_paxos_commit",
"ceph_paxos_new_pn_latency_count",
"ceph_osd_op_r_process_latency_count",
"ceph_osd_flag_norebalance",
"ceph_bluestore_submit_lat_count",
"ceph_osd_in",
"ceph_bluestore_kv_final_lat_sum",
"ceph_paxos_collect_keys_sum",
"ceph_paxos_accept_timeout",
"ceph_paxos_begin_latency_count",
"ceph_bluefs_wal_total_bytes",
"ceph_osd_flag_nobackfill",
"ceph_paxos_refresh",
"ceph_bluestore_read_lat_count",
"ceph_pg_degraded",
"ceph_mon_num_sessions",
"ceph_objecter_op_rmw",
"ceph_bluefs_bytes_written_wal",
"ceph_mon_num_elections",
"ceph_rocksdb_compact",
"ceph_bluestore_kv_sync_lat_sum",
"ceph_osd_op_process_latency_count",
"ceph_osd_op_w_prepare_latency_count",
"ceph_pool_stored",
"ceph_objecter_op_active",
"ceph_pg_backfill_unfound",
"ceph_num_objects_degraded",
"ceph_osd_flag_nodeep_scrub",
"ceph_osd_apply_latency_ms",
"ceph_paxos_begin_latency_sum",
"ceph_osd_flag_noin",
"ceph_osd_op_r",
"ceph_osd_op_rw_prepare_latency_sum",
"ceph_paxos_new_pn",
"ceph_rgw_qlen",
"ceph_rgw_req",
"ceph_rocksdb_get_latency_count",
"ceph_pool_max_avail",
"ceph_pool_rd",
"ceph_rgw_cache_miss",
"ceph_paxos_commit_latency_count",
"ceph_bluestore_throttle_lat_count",
"ceph_paxos_lease_ack_timeout",
"ceph_bluestore_commit_lat_sum",
"ceph_paxos_collect_bytes_sum",
"ceph_cluster_total_used_raw_bytes",
"ceph_pg_stale",
"ceph_health_status",
"ceph_pool_wr_bytes",
"ceph_osd_op_rw_latency_count",
"ceph_paxos_collect_uncommitted",
"ceph_osd_op_rw_latency_sum",
"ceph_paxos_share_state",
"ceph_pool_stored_raw",
"ceph_osd_op_r_prepare_latency_sum",
"ceph_bluestore_kv_flush_lat_sum",
"ceph_osd_op_rw_process_latency_sum",
"ceph_osd_metadata",
"ceph_rocksdb_rocksdb_write_memtable_time_count",
"ceph_paxos_collect_latency_count",
"ceph_pg_undersized",
"ceph_osd_op_rw_prepare_latency_count",
"ceph_paxos_collect_latency_sum",
"ceph_rocksdb_rocksdb_write_delay_time_count",
"ceph_objecter_op_rmw",
"ceph_paxos_begin_bytes_sum",
"ceph_pg_recovering",
"ceph_pg_peering",
"ceph_osd_numpg",
"ceph_osd_flag_noout",
"ceph_pg_inconsistent",
"ceph_osd_stat_bytes",
"ceph_rocksdb_submit_sync_latency_sum",
"ceph_rocksdb_compact_queue_merge",
"ceph_paxos_collect_bytes_count",
"ceph_osd_op",
"ceph_paxos_commit_keys_sum",
"ceph_osd_op_rw_in_bytes",
"ceph_osd_op_rw_out_bytes",
"ceph_bluefs_bytes_written_sst",
"ceph_rgw_put",
"ceph_osd_op_rw_process_latency_count",
"ceph_rocksdb_compact_queue_len",
"ceph_pool_wr",
"ceph_bluestore_throttle_lat_sum",
"ceph_bluefs_slow_used_bytes",
"ceph_osd_op_r_latency_sum",
"ceph_bluestore_kv_flush_lat_count",
"ceph_rocksdb_compact_range",
"ceph_osd_op_latency_sum",
"ceph_mon_session_add",
"ceph_paxos_share_state_keys_count",
"ceph_num_objects_misplaced",
"ceph_paxos_collect",
"ceph_osd_op_w_in_bytes",
"ceph_osd_op_r_process_latency_sum",
"ceph_paxos_start_peon",
"ceph_cluster_total_bytes",
"ceph_mon_session_trim",
"ceph_pg_recovery_wait",
"ceph_rocksdb_get_latency_sum",
"ceph_rocksdb_submit_transaction_sync",
"ceph_osd_op_rw",
"ceph_paxos_store_state_keys_count",
"ceph_rocksdb_rocksdb_write_delay_time_sum",
"ceph_pool_objects",
"ceph_pg_backfill_wait",
"ceph_objecter_op_r",
"ceph_objecter_op_active",
"ceph_objecter_op_w",
"ceph_osd_recovery_ops",
"ceph_bluefs_logged_bytes",
"ceph_rocksdb_get",
"ceph_pool_metadata",
"ceph_bluefs_db_total_bytes",
"ceph_rgw_put_initial_lat_sum",
"ceph_pg_recovery_toofull",
"ceph_osd_op_w_latency_count",
"ceph_rgw_put_initial_lat_count",
"ceph_mon_metadata",
"ceph_bluestore_commit_lat_count",
"ceph_bluestore_state_aio_wait_lat_count",
"ceph_pg_unknown",
"ceph_paxos_begin_bytes_count",
"ceph_pg_recovery_unfound",
"ceph_pool_quota_bytes",
"ceph_pg_snaptrim_wait",
"ceph_paxos_start_leader",
"ceph_pg_creating",
"ceph_mon_election_call",
"ceph_rocksdb_rocksdb_write_pre_and_post_time_count",
"ceph_mon_session_rm",
"ceph_cluster_total_used_bytes",
"ceph_pg_active",
"ceph_paxos_store_state",
"ceph_pg_activating",
"ceph_paxos_store_state_bytes_count",
"ceph_osd_op_w_latency_sum",
"ceph_rgw_keystone_token_cache_hit",
"ceph_rocksdb_submit_latency_count",
"ceph_pool_dirty",
"ceph_paxos_commit_latency_sum",
"ceph_rocksdb_rocksdb_write_memtable_time_sum",
"ceph_rgw_metadata",
"ceph_paxos_share_state_bytes_sum",
"ceph_osd_op_process_latency_sum",
"ceph_paxos_begin_keys_sum",
"ceph_pg_snaptrim_error",
"ceph_rgw_qactive",
"ceph_pg_backfilling",
"ceph_rocksdb_rocksdb_write_pre_and_post_time_sum",
"ceph_bluefs_wal_used_bytes",
"ceph_pool_rd_bytes",
"ceph_pg_deep",
"ceph_rocksdb_rocksdb_write_wal_time_sum",
"ceph_osd_op_wip",
"ceph_pg_backfill_toofull",
"ceph_osd_flag_noup",
"ceph_rgw_get_initial_lat_sum",
"ceph_pg_scrubbing",
"ceph_num_objects_unfound",
"ceph_mon_quorum_status",
"ceph_paxos_lease_timeout",
"ceph_osd_op_r_out_bytes",
"ceph_paxos_begin_keys_count",
"ceph_bluestore_kv_sync_lat_count",
"ceph_osd_op_prepare_latency_count",
"ceph_bluefs_bytes_written_slow",
"ceph_rocksdb_submit_latency_sum",
"ceph_pg_repair",
"ceph_osd_op_r_latency_count",
"ceph_paxos_share_state_keys_sum",
"ceph_paxos_store_state_bytes_sum",
"ceph_osd_op_latency_count",
"ceph_paxos_commit_bytes_count",
"ceph_paxos_restart",
"ceph_rgw_get_initial_lat_count",
"ceph_pg_down",
"ceph_bluefs_slow_total_bytes",
"ceph_paxos_collect_timeout",
"ceph_pg_peered",
"ceph_osd_commit_latency_ms",
"ceph_osd_op_w_process_latency_sum",
"ceph_osd_weight",
"ceph_paxos_collect_keys_count",
"ceph_paxos_share_state_bytes_count",
"ceph_osd_op_w_prepare_latency_sum",
"ceph_bluestore_read_lat_sum",
"ceph_osd_flag_noscrub",
"ceph_osd_stat_bytes_used",
"ceph_osd_flag_norecover",
"ceph_pg_clean",
"ceph_paxos_begin",
"ceph_mon_election_win",
"ceph_osd_op_w_process_latency_count",
"ceph_rgw_get_b",
"ceph_rgw_failed_req",
"ceph_rocksdb_rocksdb_write_wal_time_count",
"ceph_rgw_keystone_token_cache_miss",
"ceph_disk_occupation",
"ceph_pg_snaptrim",
"ceph_paxos_store_state_keys_sum",
"ceph_osd_numpg_removing",
"ceph_pg_remapped",
"ceph_paxos_commit_keys_count",
"ceph_pg_forced_backfill",
"ceph_paxos_new_pn_latency_sum",
"ceph_osd_op_in_bytes",
"ceph_paxos_store_state_latency_count",
"ceph_paxos_refresh_latency_count",
"ceph_rgw_get",
"ceph_pg_total",
"ceph_osd_op_r_prepare_latency_count",
"ceph_rgw_cache_hit",
"ceph_objecter_op_w",
"ceph_rocksdb_submit_transaction",
"ceph_objecter_op_r",
"ceph_bluefs_num_files",
"ceph_osd_up",
"ceph_rgw_put_b",
"ceph_mon_election_lose",
"ceph_osd_op_prepare_latency_sum",
"ceph_bluefs_db_used_bytes",
"ceph_bluestore_kv_final_lat_count",
"ceph_pool_quota_objects",
"ceph_osd_flag_nodown",
"ceph_pg_forced_recovery",
"ceph_paxos_refresh_latency_sum",
"ceph_osd_recovery_bytes",
"ceph_osd_op_w",
"ceph_paxos_commit_bytes_sum",
"ceph_bluefs_log_bytes",
"ceph_rocksdb_submit_sync_latency_count",
"ceph_pool_num_bytes_recovered",
"ceph_pool_num_objects_recovered",
"ceph_pool_recovering_bytes_per_sec",
"ceph_pool_recovering_keys_per_sec",
"ceph_pool_recovering_objects_per_sec"]
current_platform = config.ENV_DATA['platform'].lower()
prometheus = PrometheusAPI()
list_of_metrics_without_results = []
for metric in list_of_metrics:
result = prometheus.query(metric)
# check that we actually received some values
if len(result) == 0:
# Ceph Object Gateway https://docs.ceph.com/docs/master/radosgw/ is
# deployed on on-prem platforms only, so we are going to ignore
# missing metrics from these components on such platforms.
is_rgw_metric = (
metric.startswith("ceph_rgw")
or metric.startswith("ceph_objecter"))
if current_platform in constants.CLOUD_PLATFORMS and is_rgw_metric:
msg = (
f"failed to get results for {metric}, "
f"but it is expected on {current_platform}")
logger.info(msg)
else:
logger.error(f"failed to get results for {metric}")
list_of_metrics_without_results.append(metric)
msg = (
"OCS Monitoring should provide some value(s) for all tested metrics, "
"so that the list of metrics without results is empty.")
assert list_of_metrics_without_results == [], msg
class ClusterLoad:
"""
A class for cluster load functionalities
"""
def __init__(self,
project_factory=None,
pvc_factory=None,
sa_factory=None,
pod_factory=None,
target_percentage=None):
"""
Initializer for ClusterLoad
Args:
pvc_factory (function): A call to pvc_factory function
sa_factory (function): A call to service_account_factory function
pod_factory (function): A call to pod_factory function
target_percentage (float): The percentage of cluster load that is
required. The value should be greater than 0 and smaller than 1
"""
self.prometheus_api = PrometheusAPI()
self.pvc_factory = pvc_factory
self.sa_factory = sa_factory
self.pod_factory = pod_factory
self.target_percentage = target_percentage
self.cluster_limit = None
self.dc_objs = list()
self.pvc_objs = list()
self.name_suffix = 1
self.pvc_size = int(get_osd_pods_memory_sum() * 0.5)
self.io_file_size = f"{self.pvc_size * 1000 - 200}M"
self.sleep_time = 35
if project_factory:
project_name = f"{defaults.BG_LOAD_NAMESPACE}-{uuid4().hex[:5]}"
self.project = project_factory(project_name=project_name)
def increase_load(self, rate=None, wait=True):
"""
Create a PVC, a service account and a DeploymentConfig of FIO pod
Args:
rate (str): FIO 'rate' value (e.g. '20M')
wait (bool): True for waiting for IO to kick in on the
newly created pod, False otherwise
"""
pvc_obj = self.pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=self.project,
size=self.pvc_size,
volume_mode=constants.VOLUME_MODE_BLOCK,
)
self.pvc_objs.append(pvc_obj)
service_account = self.sa_factory(pvc_obj.project)
# Set new arguments with the updated file size to be used for
# DeploymentConfig of FIO pod creation
fio_dc_data = templating.load_yaml(constants.FIO_DC_YAML)
args = fio_dc_data.get('spec').get('template').get('spec').get(
'containers')[0].get('args')
new_args = [
x for x in args
if not x.startswith('--filesize=') and not x.startswith('--rate=')
]
new_args.append(f"--filesize={self.io_file_size}")
new_args.append(f"--rate={rate}")
self.name_suffix += 1
dc_obj = self.pod_factory(pvc=pvc_obj,
pod_dict_path=constants.FIO_DC_YAML,
raw_block_pv=True,
deployment_config=True,
service_account=service_account,
command_args=new_args)
self.dc_objs.append(dc_obj)
if wait:
logger.info(
f"Waiting {self.sleep_time} seconds for IO to kick-in on the newly "
f"created FIO pod {dc_obj.name}")
time.sleep(self.sleep_time)
def decrease_load(self, wait=True):
"""
Delete DeploymentConfig with its pods and the PVC. Then, wait for the
IO to be stopped
Args:
wait (bool): True for waiting for IO to drop after the deletion
of the FIO pod, False otherwise
"""
dc_name = self.dc_objs[-1].name
self.dc_objs[-1].delete()
self.dc_objs[-1].ocp.wait_for_delete(dc_name)
self.dc_objs.remove(self.dc_objs[-1])
self.pvc_objs[-1].delete()
self.pvc_objs[-1].ocp.wait_for_delete(self.pvc_objs[-1].name)
self.pvc_objs.remove(self.pvc_objs[-1])
if wait:
logger.info(
f"Waiting {self.sleep_time} seconds for IO to drop after the deletion of {dc_name}"
)
time.sleep(self.sleep_time)
def reach_cluster_load_percentage(self):
"""
Reach the cluster limit and then drop to the given target percentage.
The number of pods needed for the desired target percentage is determined by
creating pods one by one, while examining the cluster latency. Once the latency
is greater than 200 ms and it is growing exponentially, it means that
the cluster limit has been reached.
Then, dropping to the target percentage by deleting all pods and re-creating
ones with smaller value of FIO 'rate' param.
This leaves the number of pods needed running IO for cluster load to
be around the desired percentage.
"""
if not 0.1 < self.target_percentage < 0.95:
logger.warning(
f"The target percentage is {self.target_percentage * 100}% which is "
f"not within the accepted range. Therefore, IO will not be started"
)
return
low_diff_counter = 0
limit_reached = False
cluster_limit = None
latency_vals = list()
time_to_wait = 60 * 30
time_before = time.time()
current_iops = self.get_query(query=constants.IOPS_QUERY)
msg = ("\n======================\nCurrent IOPS: {:.2f}"
"\nPrevious IOPS: {:.2f}\n======================")
# Creating FIO DeploymentConfig pods one by one, with a large value of FIO
# 'rate' arg. This in order to determine the cluster limit faster.
# Once determined, these pods will be deleted. Then, new FIO DC pods will be
# created, with a smaller value of 'rate' param. This in order to be more
# accurate with reaching the target percentage
rate = '250M'
while not limit_reached:
self.increase_load(rate=rate)
previous_iops = current_iops
current_iops = self.get_query(query=constants.IOPS_QUERY)
if current_iops > previous_iops:
cluster_limit = current_iops
logger.info(
msg.format(current_iops, previous_iops, len(self.dc_objs)))
self.print_metrics()
latency = self.calc_trim_metric_mean(
metric=constants.LATENCY_QUERY) * 1000
latency_vals.append(latency)
logger.info(f"Latency values: {latency_vals}")
if len(latency_vals) > 1 and latency > 250:
# Checking for an exponential growth
if latency > latency_vals[0] * 2**7:
logger.info("Latency exponential growth was detected")
limit_reached = True
# In case the latency is greater than 3 seconds,
# most chances the limit has been reached
if latency > 3000:
logger.info(f"Limit was determined by latency, which is "
f"higher than 3 seconds - {latency} ms")
limit_reached = True
# For clusters that their nodes do not meet the minimum
# resource requirements, the cluster limit is being reached
# while the latency remains low. For that, the cluster limit
# needs to be determined by the following condition of IOPS
# diff between FIO pod creation iterations
iops_diff = (current_iops / previous_iops * 100) - 100
low_diff_counter += 1 if -15 < iops_diff < 10 else 0
if low_diff_counter > 3:
logger.warning(
f"Limit was determined by low IOPS diff between "
f"iterations - {iops_diff:.2f}%")
limit_reached = True
if time.time() > time_before + time_to_wait:
logger.warning(
f"Could not determine the cluster IOPS limit within"
f"\nthe given {time_to_wait} seconds timeout. Breaking")
limit_reached = True
cluster_used_space = get_percent_used_capacity()
if cluster_used_space > 60:
logger.warning(
f"Cluster used space is {cluster_used_space}%. Could "
f"not reach the cluster IOPS limit before the "
f"used spaced reached 60%. Breaking")
limit_reached = True
self.cluster_limit = cluster_limit
logger.info(
f"\n===================================\nThe cluster IOPS limit "
f"is {self.cluster_limit:.2f}\n==================================="
)
logger.info(
f"Deleting all DC FIO pods that have FIO rate parameter of {rate}")
while self.dc_objs:
self.decrease_load(wait=False)
# Creating the first pod of small FIO 'rate' param, to speed up the process.
# In the meantime, the load will drop, following the deletion of the
# FIO pods with large FIO 'rate' param
rate = '15M'
logger.info(
f"Creating FIO pods with a rate parameter of {rate}, one by "
f"one, until the target percentage is reached")
self.increase_load(rate=rate)
target_iops = self.cluster_limit * self.target_percentage
current_iops = self.get_query(query=constants.IOPS_QUERY)
logger.info(f"Target IOPS: {target_iops}")
logger.info(f"Current IOPS: {current_iops}")
while current_iops < target_iops * 0.95:
wait = False if current_iops < target_iops / 2 else True
self.increase_load(rate=rate, wait=wait)
previous_iops = current_iops
current_iops = self.get_query(query=constants.IOPS_QUERY)
logger.info(
msg.format(current_iops, previous_iops, len(self.dc_objs)))
self.print_metrics()
logger.info(
f"\n========================================\n"
f"The target load, of {self.target_percentage * 100}%, has been reached"
f"\n==========================================")
def get_query(self, query):
"""
Get query from Prometheus and parse it
Args:
query (str): Query to be done
Returns:
float: the query result
"""
now = datetime.now
timestamp = datetime.timestamp
return float(
self.prometheus_api.query(query,
str(timestamp(now())))[0]['value'][1])
def calc_trim_metric_mean(self, metric=constants.LATENCY_QUERY, samples=5):
"""
Get the trimmed mean of a given metric
Args:
metric (str): The metric to calculate the average result for
samples (int): The number of samples to take
Returns:
float: The average result for the metric
"""
vals = list()
for i in range(samples):
vals.append(round(self.get_query(metric), 5))
if i == samples - 1:
break
time.sleep(5)
return round(get_trim_mean(vals), 5)
def get_metrics(self):
"""
Get different cluster load and utilization metrics
"""
return {
"throughput":
self.get_query(constants.THROUGHPUT_QUERY) *
(constants.TP_CONVERSION.get(' B/s')),
"latency":
self.get_query(constants.LATENCY_QUERY) * 1000,
"iops":
self.get_query(constants.IOPS_QUERY),
"used_space":
self.get_query(constants.USED_SPACE_QUERY) / 1e+9
}
def print_metrics(self):
"""
Print metrics
"""
high_latency = 500
metrics = self.get_metrics()
limit_msg = ""
pods_msg = ""
if self.cluster_limit:
limit_msg = (
f"({metrics.get('iops') / self.cluster_limit * 100:.2f}% of the "
f"{self.cluster_limit:.1f} limit)\n")
if self.dc_objs:
pods_msg = (f"\nNumber of pods running FIO: {len(self.dc_objs)}")
logger.info(
f"\n===============================\n"
f"Cluster throughput: {metrics.get('throughput'):.2f} MB/s\n"
f"Cluster latency: {metrics.get('latency'):.2f} ms\n"
f"Cluster IOPS: {metrics.get('iops'):.2f}\n{limit_msg}"
f"Cluster used space: {metrics.get('used_space'):.2f} GB{pods_msg}"
f"\n===============================")
if metrics.get('latency') > high_latency:
logger.warning(
f"Cluster latency is higher than {high_latency} ms!")
