def test_monitoring_enabled(): """ OCS Monitoring is enabled after OCS installation (which is why this test has a post deployment marker) by asking for values of one ceph and one noobaa related metrics. """ prometheus = PrometheusAPI() # ask for values of ceph_pool_stored metric logger.info("Checking that ceph data are provided in OCS monitoring") result = prometheus.query('ceph_pool_stored') msg = "check that we actually received some values for a ceph query" assert len(result) > 0, msg for metric in result: _, value = metric['value'] assert_msg = "number of bytes in a pool isn't a positive integer or zero" assert int(value) >= 0, assert_msg # additional check that values makes at least some sense logger.info( "Checking that size of ceph_pool_stored result matches number of pools") ct_pod = pod.get_ceph_tools_pod() ceph_pools = ct_pod.exec_ceph_cmd("ceph osd pool ls") assert len(result) == len(ceph_pools) # again for a noobaa metric logger.info("Checking that MCG/NooBaa data are provided in OCS monitoring") result = prometheus.query('NooBaa_bucket_status') msg = "check that we actually received some values for a MCG/NooBaa query" assert len(result) > 0, msg for metric in result: _, value = metric['value'] assert int(value) >= 0, "bucket status isn't a positive integer or zero"
def test_ceph_rbd_metrics_available(): """ Ceph RBD metrics should be provided via OCP Prometheus as well. See also: https://ceph.com/rbd/new-in-nautilus-rbd-performance-monitoring/ """ # this is not a full list, but it is enough to check whether we have # rbd metrics available via OCS monitoring list_of_metrics = [ "ceph_rbd_write_ops", "ceph_rbd_read_ops", "ceph_rbd_write_bytes", "ceph_rbd_read_bytes", "ceph_rbd_write_latency_sum", "ceph_rbd_write_latency_count"] prometheus = PrometheusAPI() list_of_metrics_without_results = [] for metric in list_of_metrics: result = prometheus.query(metric) # check that we actually received some values if len(result) == 0: logger.error(f"failed to get results for {metric}") list_of_metrics_without_results.append(metric) msg = ( "OCS Monitoring should provide some value(s) for tested rbd metrics, " "so that the list of metrics without results is empty.") assert list_of_metrics_without_results == [], msg
def test_monitoring_enabled(): """ OCS Monitoring is enabled after OCS installation (which is why this test has a post deployment marker) by asking for values of one ceph and one noobaa related metrics. """ prometheus = PrometheusAPI() if ( storagecluster_independent_check() and float(config.ENV_DATA["ocs_version"]) < 4.6 ): logger.info( f"Skipping ceph metrics because it is not enabled for external " f"mode for OCS {float(config.ENV_DATA['ocs_version'])}" ) else: # ask for values of ceph_pool_stored metric logger.info("Checking that ceph data are provided in OCS monitoring") result = prometheus.query("ceph_pool_stored") msg = "check that we actually received some values for a ceph query" assert len(result) > 0, msg for metric in result: _, value = metric["value"] assert_msg = "number of bytes in a pool isn't a positive integer or zero" assert int(value) >= 0, assert_msg # additional check that values makes at least some sense logger.info( "Checking that size of ceph_pool_stored result matches number of pools" ) ct_pod = pod.get_ceph_tools_pod() ceph_pools = ct_pod.exec_ceph_cmd("ceph osd pool ls") assert len(result) == len(ceph_pools) # again for a noobaa metric logger.info("Checking that MCG/NooBaa data are provided in OCS monitoring") result = prometheus.query("NooBaa_bucket_status") msg = "check that we actually received some values for a MCG/NooBaa query" assert len(result) > 0, msg for metric in result: _, value = metric["value"] assert int(value) >= 0, "bucket status isn't a positive integer or zero"
class ClusterLoad: """ A class for cluster load functionalities """ def __init__( self, project_factory=None, pvc_factory=None, sa_factory=None, pod_factory=None, target_percentage=None ): """ Initializer for ClusterLoad Args: project_factory (function): A call to project_factory function pvc_factory (function): A call to pvc_factory function sa_factory (function): A call to service_account_factory function pod_factory (function): A call to pod_factory function target_percentage (float): The percentage of cluster load that is required. The value should be greater than 0.1 and smaller than 0.95 """ self.prometheus_api = PrometheusAPI() self.pvc_factory = pvc_factory self.sa_factory = sa_factory self.pod_factory = pod_factory self.target_percentage = target_percentage self.cluster_limit = None self.dc_objs = list() self.pvc_objs = list() self.previous_iops = None self.current_iops = None self.rate = None self.pvc_size = int(get_osd_pods_memory_sum() * 0.5) self.sleep_time = 45 self.target_pods_number = None if project_factory: project_name = f"{defaults.BG_LOAD_NAMESPACE}-{uuid4().hex[:5]}" self.project = project_factory(project_name=project_name) def increase_load(self, rate, wait=True): """ Create a PVC, a service account and a DeploymentConfig of FIO pod Args: rate (str): FIO 'rate' value (e.g. '20M') wait (bool): True for waiting for IO to kick in on the newly created pod, False otherwise """ pvc_obj = self.pvc_factory( interface=constants.CEPHBLOCKPOOL, project=self.project, size=self.pvc_size, volume_mode=constants.VOLUME_MODE_BLOCK, ) self.pvc_objs.append(pvc_obj) service_account = self.sa_factory(pvc_obj.project) # Set new arguments with the updated file size to be used for # DeploymentConfig of FIO pod creation fio_dc_data = templating.load_yaml(constants.FIO_DC_YAML) args = fio_dc_data.get('spec').get('template').get( 'spec' ).get('containers')[0].get('args') new_args = [ x for x in args if not x.startswith('--filesize=') and not x.startswith('--rate=') ] io_file_size = f"{self.pvc_size * 1000 - 200}M" new_args.append(f"--filesize={io_file_size}") new_args.append(f"--rate={rate}") dc_obj = self.pod_factory( pvc=pvc_obj, pod_dict_path=constants.FIO_DC_YAML, raw_block_pv=True, deployment_config=True, service_account=service_account, command_args=new_args ) self.dc_objs.append(dc_obj) if wait: logger.info( f"Waiting {self.sleep_time} seconds for IO to kick-in on the newly " f"created FIO pod {dc_obj.name}" ) time.sleep(self.sleep_time) def decrease_load(self, wait=True): """ Delete DeploymentConfig with its pods and the PVC. Then, wait for the IO to be stopped Args: wait (bool): True for waiting for IO to drop after the deletion of the FIO pod, False otherwise """ dc_name = self.dc_objs[-1].name self.dc_objs[-1].delete() self.dc_objs[-1].ocp.wait_for_delete(dc_name) self.dc_objs.remove(self.dc_objs[-1]) self.pvc_objs[-1].delete() self.pvc_objs[-1].ocp.wait_for_delete(self.pvc_objs[-1].name) self.pvc_objs.remove(self.pvc_objs[-1]) if wait: logger.info( f"Waiting {self.sleep_time} seconds for IO to drop after " f"the deletion of {dc_name}" ) time.sleep(self.sleep_time) def increase_load_and_print_data(self, rate, wait=True): """ Increase load and print data Args: rate (str): FIO 'rate' value (e.g. '20M') wait (bool): True for waiting for IO to kick in on the newly created pod, False otherwise """ self.increase_load(rate=rate, wait=wait) self.previous_iops = self.current_iops self.current_iops = self.calc_trim_metric_mean(metric=constants.IOPS_QUERY) msg = f"Current: {self.current_iops:.2f} || Previous: {self.previous_iops:.2f}" logger.info(f"IOPS:{wrap_msg(msg)}") self.print_metrics() def reach_cluster_load_percentage(self): """ Reach the cluster limit and then drop to the given target percentage. The number of pods needed for the desired target percentage is determined by creating pods one by one, while examining the cluster latency. Once the latency is greater than 250 ms and it is growing exponentially, it means that the cluster limit has been reached. Then, dropping to the target percentage by deleting all pods and re-creating ones with smaller value of FIO 'rate' param. This leaves the number of pods needed running IO for cluster load to be around the desired percentage. """ if not self.target_percentage: logger.warning("The target percentage was not provided. Breaking") return if not 0.1 < self.target_percentage < 0.95: logger.warning( f"The target percentage is {self.target_percentage * 100}% which is " "not within the accepted range. Therefore, IO will not be started" ) return low_diff_counter = 0 cluster_limit = None latency_vals = list() time_to_wait = 60 * 30 time_before = time.time() self.current_iops = self.get_query(query=constants.IOPS_QUERY) # Creating FIO DeploymentConfig pods one by one, with a large value of FIO # 'rate' arg. This in order to determine the cluster limit faster. # Once determined, these pods will be deleted. Then, new FIO DC pods will be # created, with a smaller value of 'rate' param. This in order to be more # accurate with reaching the target percentage while True: wait = False if len(self.dc_objs) <= 1 else True self.increase_load_and_print_data(rate='250M', wait=wait) if self.current_iops > self.previous_iops: cluster_limit = self.current_iops latency = self.calc_trim_metric_mean(metric=constants.LATENCY_QUERY) * 1000 latency_vals.append(latency) logger.info(f"Latency values: {latency_vals}") iops_diff = (self.current_iops / self.previous_iops * 100) - 100 low_diff_counter += 1 if -15 < iops_diff < 10 else 0 cluster_used_space = get_percent_used_capacity() if len(latency_vals) > 1 and latency > 250: # Checking for an exponential growth. In case the latest latency sample # value is more than 128 times the first latency value sample, we can conclude # that the cluster limit in terms of IOPS, has been reached. # See https://blog.docbert.org/vdbench-curve/ for more details. # In other cases, when the first latency sample value is greater than 3 ms, # the multiplication factor we check according to, is lower, in order to # determine the cluster load faster. if latency > latency_vals[0] * 2 ** 7 or ( 3 < latency_vals[0] < 50 and len(latency_vals) > 5 ): logger.info( wrap_msg("The cluster limit was determined by latency growth") ) break # In case the latency is greater than 2 seconds, # most chances the limit has been reached elif latency > 2000: logger.info( wrap_msg(f"The limit was determined by the high latency - {latency} ms") ) break # For clusters that their nodes do not meet the minimum # resource requirements, the cluster limit is being reached # while the latency remains low. For that, the cluster limit # needs to be determined by the following condition of IOPS # diff between FIO pod creation iterations elif low_diff_counter > 3: logger.warning( wrap_msg( "Limit was determined by low IOPS diff between " f"iterations - {iops_diff:.2f}%" ) ) break elif time.time() > time_before + time_to_wait: logger.warning( wrap_msg( "Could not determine the cluster IOPS limit within" f"the given {time_to_wait} seconds timeout. Breaking" ) ) break elif cluster_used_space > 60: logger.warning( wrap_msg( f"Cluster used space is {cluster_used_space}%. Could " "not reach the cluster IOPS limit before the " "used spaced reached 60%. Breaking" ) ) break self.cluster_limit = cluster_limit logger.info(wrap_msg(f"The cluster IOPS limit is {self.cluster_limit:.2f}")) logger.info("Deleting all DC FIO pods that have large FIO rate") while self.dc_objs: self.decrease_load(wait=False) target_iops = self.cluster_limit * self.target_percentage range_map = RangeKeyDict( { (0, 500): (6, 0.82, 0.4), (500, 1000): (8, 0.84, 0.45), (1000, 1500): (10, 0.86, 0.5), (1500, 2000): (12, 0.88, 0.55), (2000, 2500): (14, 0.90, 0.6), (2500, 3000): (16, 0.92, 0.65), (3000, 3500): (18, 0.94, 0.7), (3500, math.inf): (20, 0.96, 0.75), } ) self.rate = f'{range_map[target_iops][0]}M' # Creating the first pod of small FIO 'rate' param, to speed up the process. # In the meantime, the load will drop, following the deletion of the # FIO pods with large FIO 'rate' param logger.info("Creating FIO pods, one by one, until the target percentage is reached") self.increase_load_and_print_data(rate=self.rate) msg = ( f"The target load, in IOPS, is: {target_iops}, which is " f"{self.target_percentage*100}% of the {self.cluster_limit} cluster limit" ) logger.info(wrap_msg(msg)) while self.current_iops < target_iops * range_map[target_iops][1]: wait = False if self.current_iops < target_iops * range_map[target_iops][2] else True self.increase_load_and_print_data(rate=self.rate, wait=wait) msg = f"The target load, of {self.target_percentage * 100}%, has been reached" logger.info(wrap_msg(msg)) self.target_pods_number = len(self.dc_objs) def get_query(self, query, mute_logs=False): """ Get query from Prometheus and parse it Args: query (str): Query to be done mute_logs (bool): True for muting the logs, False otherwise Returns: float: the query result """ now = datetime.now timestamp = datetime.timestamp return float( self.prometheus_api.query( query, str(timestamp(now())), mute_logs=mute_logs )[0]['value'][1] ) def calc_trim_metric_mean(self, metric, samples=5, mute_logs=False): """ Get the trimmed mean of a given metric Args: metric (str): The metric to calculate the average result for samples (int): The number of samples to take mute_logs (bool): True for muting the logs, False otherwise Returns: float: The average result for the metric """ vals = list() for i in range(samples): vals.append(round(self.get_query(metric, mute_logs), 5)) if i == samples - 1: break time.sleep(5) return round(get_trim_mean(vals), 5) def print_metrics(self, mute_logs=False): """ Print metrics Args: mute_logs (bool): True for muting the Prometheus logs, False otherwise """ high_latency = 200 metrics = { "throughput": self.get_query(constants.THROUGHPUT_QUERY, mute_logs=mute_logs) * ( constants.TP_CONVERSION.get(' B/s') ), "latency": self.get_query(constants.LATENCY_QUERY, mute_logs=mute_logs) * 1000, "iops": self.get_query(constants.IOPS_QUERY, mute_logs=mute_logs), "used_space": self.get_query(constants.USED_SPACE_QUERY, mute_logs=mute_logs) / 1e+9 } limit_msg = ( f" ({metrics.get('iops') / self.cluster_limit * 100:.2f}% of the " f"{self.cluster_limit:.2f} limit)" ) if self.cluster_limit else "" pods_msg = f" || Number of FIO pods: {len(self.dc_objs)}" if self.dc_objs else "" msg = ( f"Throughput: {metrics.get('throughput'):.2f} MB/s || " f"Latency: {metrics.get('latency'):.2f} ms || " f"IOPS: {metrics.get('iops'):.2f}{limit_msg} || " f"Used Space: {metrics.get('used_space'):.2f} GB{pods_msg}" ) logger.info(f"Cluster utilization:{wrap_msg(msg)}") if metrics.get('latency') > high_latency: logger.warning(f"Cluster latency is higher than {high_latency} ms!") def adjust_load_if_needed(self): """ Dynamically adjust the IO load based on the cluster latency. In case the latency goes beyond 250 ms, start deleting FIO pods. Once latency drops back below 100 ms, re-create the FIO pods to make sure that cluster load is around the target percentage """ latency = self.calc_trim_metric_mean( constants.LATENCY_QUERY, mute_logs=True ) if latency > 0.25 and len(self.dc_objs) > 0: msg = ( f"Latency is too high - {latency * 1000:.2f} ms." " Dropping the background load. Once the latency drops back to " "normal, the background load will be increased back" ) logger.warning(wrap_msg(msg)) self.decrease_load(wait=False) if latency < 0.1 and self.target_pods_number > len(self.dc_objs): msg = ( f"Latency is back to normal - {latency * 1000:.2f} ms. " f"Increasing back the load" ) logger.info(wrap_msg(msg)) self.increase_load(rate=self.rate, wait=False) def pause_load(self): """ Pause the cluster load """ logger.info(wrap_msg("Pausing the cluster load")) while self.dc_objs: self.decrease_load(wait=False) def resume_load(self): """ Resume the cluster load """ logger.info(wrap_msg("Resuming the cluster load")) while len(self.dc_objs) < self.target_pods_number: self.increase_load(rate=self.rate, wait=False)
def test_ceph_metrics_available(): """ Ceph metrics as listed in KNIP-634 should be provided via OCP Prometheus. Ceph Object Gateway https://docs.ceph.com/docs/master/radosgw/ is deployed on on-prem platforms only (such as VMWare - see BZ 1763150), so this test case ignores failures for ceph_rgw_* and ceph_objecter_* metrics when running on cloud platforms (such as AWS). """ # this list is taken from spreadsheet attached to KNIP-634 list_of_metrics = [ "ceph_bluestore_state_aio_wait_lat_sum", "ceph_paxos_store_state_latency_sum", "ceph_osd_op_out_bytes", "ceph_pg_incomplete", "ceph_bluestore_submit_lat_sum", "ceph_paxos_commit", "ceph_paxos_new_pn_latency_count", "ceph_osd_op_r_process_latency_count", "ceph_osd_flag_norebalance", "ceph_bluestore_submit_lat_count", "ceph_osd_in", "ceph_bluestore_kv_final_lat_sum", "ceph_paxos_collect_keys_sum", "ceph_paxos_accept_timeout", "ceph_paxos_begin_latency_count", "ceph_bluefs_wal_total_bytes", "ceph_osd_flag_nobackfill", "ceph_paxos_refresh", "ceph_bluestore_read_lat_count", "ceph_pg_degraded", "ceph_mon_num_sessions", "ceph_objecter_op_rmw", "ceph_bluefs_bytes_written_wal", "ceph_mon_num_elections", "ceph_rocksdb_compact", "ceph_bluestore_kv_sync_lat_sum", "ceph_osd_op_process_latency_count", "ceph_osd_op_w_prepare_latency_count", "ceph_pool_stored", "ceph_objecter_op_active", "ceph_pg_backfill_unfound", "ceph_num_objects_degraded", "ceph_osd_flag_nodeep_scrub", "ceph_osd_apply_latency_ms", "ceph_paxos_begin_latency_sum", "ceph_osd_flag_noin", "ceph_osd_op_r", "ceph_osd_op_rw_prepare_latency_sum", "ceph_paxos_new_pn", "ceph_rgw_qlen", "ceph_rgw_req", "ceph_rocksdb_get_latency_count", "ceph_pool_max_avail", "ceph_pool_rd", "ceph_rgw_cache_miss", "ceph_paxos_commit_latency_count", "ceph_bluestore_throttle_lat_count", "ceph_paxos_lease_ack_timeout", "ceph_bluestore_commit_lat_sum", "ceph_paxos_collect_bytes_sum", "ceph_cluster_total_used_raw_bytes", "ceph_pg_stale", "ceph_health_status", "ceph_pool_wr_bytes", "ceph_osd_op_rw_latency_count", "ceph_paxos_collect_uncommitted", "ceph_osd_op_rw_latency_sum", "ceph_paxos_share_state", "ceph_pool_stored_raw", "ceph_osd_op_r_prepare_latency_sum", "ceph_bluestore_kv_flush_lat_sum", "ceph_osd_op_rw_process_latency_sum", "ceph_osd_metadata", "ceph_rocksdb_rocksdb_write_memtable_time_count", "ceph_paxos_collect_latency_count", "ceph_pg_undersized", "ceph_osd_op_rw_prepare_latency_count", "ceph_paxos_collect_latency_sum", "ceph_rocksdb_rocksdb_write_delay_time_count", "ceph_objecter_op_rmw", "ceph_paxos_begin_bytes_sum", "ceph_pg_recovering", "ceph_pg_peering", "ceph_osd_numpg", "ceph_osd_flag_noout", "ceph_pg_inconsistent", "ceph_osd_stat_bytes", "ceph_rocksdb_submit_sync_latency_sum", "ceph_rocksdb_compact_queue_merge", "ceph_paxos_collect_bytes_count", "ceph_osd_op", "ceph_paxos_commit_keys_sum", "ceph_osd_op_rw_in_bytes", "ceph_osd_op_rw_out_bytes", "ceph_bluefs_bytes_written_sst", "ceph_rgw_put", "ceph_osd_op_rw_process_latency_count", "ceph_rocksdb_compact_queue_len", "ceph_pool_wr", "ceph_bluestore_throttle_lat_sum", "ceph_bluefs_slow_used_bytes", "ceph_osd_op_r_latency_sum", "ceph_bluestore_kv_flush_lat_count", "ceph_rocksdb_compact_range", "ceph_osd_op_latency_sum", "ceph_mon_session_add", "ceph_paxos_share_state_keys_count", "ceph_num_objects_misplaced", "ceph_paxos_collect", "ceph_osd_op_w_in_bytes", "ceph_osd_op_r_process_latency_sum", "ceph_paxos_start_peon", "ceph_cluster_total_bytes", "ceph_mon_session_trim", "ceph_pg_recovery_wait", "ceph_rocksdb_get_latency_sum", "ceph_rocksdb_submit_transaction_sync", "ceph_osd_op_rw", "ceph_paxos_store_state_keys_count", "ceph_rocksdb_rocksdb_write_delay_time_sum", "ceph_pool_objects", "ceph_pg_backfill_wait", "ceph_objecter_op_r", "ceph_objecter_op_active", "ceph_objecter_op_w", "ceph_osd_recovery_ops", "ceph_bluefs_logged_bytes", "ceph_rocksdb_get", "ceph_pool_metadata", "ceph_bluefs_db_total_bytes", "ceph_rgw_put_initial_lat_sum", "ceph_pg_recovery_toofull", "ceph_osd_op_w_latency_count", "ceph_rgw_put_initial_lat_count", "ceph_mon_metadata", "ceph_bluestore_commit_lat_count", "ceph_bluestore_state_aio_wait_lat_count", "ceph_pg_unknown", "ceph_paxos_begin_bytes_count", "ceph_pg_recovery_unfound", "ceph_pool_quota_bytes", "ceph_pg_snaptrim_wait", "ceph_paxos_start_leader", "ceph_pg_creating", "ceph_mon_election_call", "ceph_rocksdb_rocksdb_write_pre_and_post_time_count", "ceph_mon_session_rm", "ceph_cluster_total_used_bytes", "ceph_pg_active", "ceph_paxos_store_state", "ceph_pg_activating", "ceph_paxos_store_state_bytes_count", "ceph_osd_op_w_latency_sum", "ceph_rgw_keystone_token_cache_hit", "ceph_rocksdb_submit_latency_count", "ceph_pool_dirty", "ceph_paxos_commit_latency_sum", "ceph_rocksdb_rocksdb_write_memtable_time_sum", "ceph_rgw_metadata", "ceph_paxos_share_state_bytes_sum", "ceph_osd_op_process_latency_sum", "ceph_paxos_begin_keys_sum", "ceph_pg_snaptrim_error", "ceph_rgw_qactive", "ceph_pg_backfilling", "ceph_rocksdb_rocksdb_write_pre_and_post_time_sum", "ceph_bluefs_wal_used_bytes", "ceph_pool_rd_bytes", "ceph_pg_deep", "ceph_rocksdb_rocksdb_write_wal_time_sum", "ceph_osd_op_wip", "ceph_pg_backfill_toofull", "ceph_osd_flag_noup", "ceph_rgw_get_initial_lat_sum", "ceph_pg_scrubbing", "ceph_num_objects_unfound", "ceph_mon_quorum_status", "ceph_paxos_lease_timeout", "ceph_osd_op_r_out_bytes", "ceph_paxos_begin_keys_count", "ceph_bluestore_kv_sync_lat_count", "ceph_osd_op_prepare_latency_count", "ceph_bluefs_bytes_written_slow", "ceph_rocksdb_submit_latency_sum", "ceph_pg_repair", "ceph_osd_op_r_latency_count", "ceph_paxos_share_state_keys_sum", "ceph_paxos_store_state_bytes_sum", "ceph_osd_op_latency_count", "ceph_paxos_commit_bytes_count", "ceph_paxos_restart", "ceph_rgw_get_initial_lat_count", "ceph_pg_down", "ceph_bluefs_slow_total_bytes", "ceph_paxos_collect_timeout", "ceph_pg_peered", "ceph_osd_commit_latency_ms", "ceph_osd_op_w_process_latency_sum", "ceph_osd_weight", "ceph_paxos_collect_keys_count", "ceph_paxos_share_state_bytes_count", "ceph_osd_op_w_prepare_latency_sum", "ceph_bluestore_read_lat_sum", "ceph_osd_flag_noscrub", "ceph_osd_stat_bytes_used", "ceph_osd_flag_norecover", "ceph_pg_clean", "ceph_paxos_begin", "ceph_mon_election_win", "ceph_osd_op_w_process_latency_count", "ceph_rgw_get_b", "ceph_rgw_failed_req", "ceph_rocksdb_rocksdb_write_wal_time_count", "ceph_rgw_keystone_token_cache_miss", "ceph_disk_occupation", "ceph_pg_snaptrim", "ceph_paxos_store_state_keys_sum", "ceph_osd_numpg_removing", "ceph_pg_remapped", "ceph_paxos_commit_keys_count", "ceph_pg_forced_backfill", "ceph_paxos_new_pn_latency_sum", "ceph_osd_op_in_bytes", "ceph_paxos_store_state_latency_count", "ceph_paxos_refresh_latency_count", "ceph_rgw_get", "ceph_pg_total", "ceph_osd_op_r_prepare_latency_count", "ceph_rgw_cache_hit", "ceph_objecter_op_w", "ceph_rocksdb_submit_transaction", "ceph_objecter_op_r", "ceph_bluefs_num_files", "ceph_osd_up", "ceph_rgw_put_b", "ceph_mon_election_lose", "ceph_osd_op_prepare_latency_sum", "ceph_bluefs_db_used_bytes", "ceph_bluestore_kv_final_lat_count", "ceph_pool_quota_objects", "ceph_osd_flag_nodown", "ceph_pg_forced_recovery", "ceph_paxos_refresh_latency_sum", "ceph_osd_recovery_bytes", "ceph_osd_op_w", "ceph_paxos_commit_bytes_sum", "ceph_bluefs_log_bytes", "ceph_rocksdb_submit_sync_latency_count", "ceph_pool_num_bytes_recovered", "ceph_pool_num_objects_recovered", "ceph_pool_recovering_bytes_per_sec", "ceph_pool_recovering_keys_per_sec", "ceph_pool_recovering_objects_per_sec"] current_platform = config.ENV_DATA['platform'].lower() prometheus = PrometheusAPI() list_of_metrics_without_results = [] for metric in list_of_metrics: result = prometheus.query(metric) # check that we actually received some values if len(result) == 0: # Ceph Object Gateway https://docs.ceph.com/docs/master/radosgw/ is # deployed on on-prem platforms only, so we are going to ignore # missing metrics from these components on such platforms. is_rgw_metric = ( metric.startswith("ceph_rgw") or metric.startswith("ceph_objecter")) if current_platform in constants.CLOUD_PLATFORMS and is_rgw_metric: msg = ( f"failed to get results for {metric}, " f"but it is expected on {current_platform}") logger.info(msg) else: logger.error(f"failed to get results for {metric}") list_of_metrics_without_results.append(metric) msg = ( "OCS Monitoring should provide some value(s) for all tested metrics, " "so that the list of metrics without results is empty.") assert list_of_metrics_without_results == [], msg
class ClusterLoad: """ A class for cluster load functionalities """ def __init__(self, project_factory=None, pvc_factory=None, sa_factory=None, pod_factory=None, target_percentage=None): """ Initializer for ClusterLoad Args: pvc_factory (function): A call to pvc_factory function sa_factory (function): A call to service_account_factory function pod_factory (function): A call to pod_factory function target_percentage (float): The percentage of cluster load that is required. The value should be greater than 0 and smaller than 1 """ self.prometheus_api = PrometheusAPI() self.pvc_factory = pvc_factory self.sa_factory = sa_factory self.pod_factory = pod_factory self.target_percentage = target_percentage self.cluster_limit = None self.dc_objs = list() self.pvc_objs = list() self.name_suffix = 1 self.pvc_size = int(get_osd_pods_memory_sum() * 0.5) self.io_file_size = f"{self.pvc_size * 1000 - 200}M" self.sleep_time = 35 if project_factory: project_name = f"{defaults.BG_LOAD_NAMESPACE}-{uuid4().hex[:5]}" self.project = project_factory(project_name=project_name) def increase_load(self, rate=None, wait=True): """ Create a PVC, a service account and a DeploymentConfig of FIO pod Args: rate (str): FIO 'rate' value (e.g. '20M') wait (bool): True for waiting for IO to kick in on the newly created pod, False otherwise """ pvc_obj = self.pvc_factory( interface=constants.CEPHBLOCKPOOL, project=self.project, size=self.pvc_size, volume_mode=constants.VOLUME_MODE_BLOCK, ) self.pvc_objs.append(pvc_obj) service_account = self.sa_factory(pvc_obj.project) # Set new arguments with the updated file size to be used for # DeploymentConfig of FIO pod creation fio_dc_data = templating.load_yaml(constants.FIO_DC_YAML) args = fio_dc_data.get('spec').get('template').get('spec').get( 'containers')[0].get('args') new_args = [ x for x in args if not x.startswith('--filesize=') and not x.startswith('--rate=') ] new_args.append(f"--filesize={self.io_file_size}") new_args.append(f"--rate={rate}") self.name_suffix += 1 dc_obj = self.pod_factory(pvc=pvc_obj, pod_dict_path=constants.FIO_DC_YAML, raw_block_pv=True, deployment_config=True, service_account=service_account, command_args=new_args) self.dc_objs.append(dc_obj) if wait: logger.info( f"Waiting {self.sleep_time} seconds for IO to kick-in on the newly " f"created FIO pod {dc_obj.name}") time.sleep(self.sleep_time) def decrease_load(self, wait=True): """ Delete DeploymentConfig with its pods and the PVC. Then, wait for the IO to be stopped Args: wait (bool): True for waiting for IO to drop after the deletion of the FIO pod, False otherwise """ dc_name = self.dc_objs[-1].name self.dc_objs[-1].delete() self.dc_objs[-1].ocp.wait_for_delete(dc_name) self.dc_objs.remove(self.dc_objs[-1]) self.pvc_objs[-1].delete() self.pvc_objs[-1].ocp.wait_for_delete(self.pvc_objs[-1].name) self.pvc_objs.remove(self.pvc_objs[-1]) if wait: logger.info( f"Waiting {self.sleep_time} seconds for IO to drop after the deletion of {dc_name}" ) time.sleep(self.sleep_time) def reach_cluster_load_percentage(self): """ Reach the cluster limit and then drop to the given target percentage. The number of pods needed for the desired target percentage is determined by creating pods one by one, while examining the cluster latency. Once the latency is greater than 200 ms and it is growing exponentially, it means that the cluster limit has been reached. Then, dropping to the target percentage by deleting all pods and re-creating ones with smaller value of FIO 'rate' param. This leaves the number of pods needed running IO for cluster load to be around the desired percentage. """ if not 0.1 < self.target_percentage < 0.95: logger.warning( f"The target percentage is {self.target_percentage * 100}% which is " f"not within the accepted range. Therefore, IO will not be started" ) return low_diff_counter = 0 limit_reached = False cluster_limit = None latency_vals = list() time_to_wait = 60 * 30 time_before = time.time() current_iops = self.get_query(query=constants.IOPS_QUERY) msg = ("\n======================\nCurrent IOPS: {:.2f}" "\nPrevious IOPS: {:.2f}\n======================") # Creating FIO DeploymentConfig pods one by one, with a large value of FIO # 'rate' arg. This in order to determine the cluster limit faster. # Once determined, these pods will be deleted. Then, new FIO DC pods will be # created, with a smaller value of 'rate' param. This in order to be more # accurate with reaching the target percentage rate = '250M' while not limit_reached: self.increase_load(rate=rate) previous_iops = current_iops current_iops = self.get_query(query=constants.IOPS_QUERY) if current_iops > previous_iops: cluster_limit = current_iops logger.info( msg.format(current_iops, previous_iops, len(self.dc_objs))) self.print_metrics() latency = self.calc_trim_metric_mean( metric=constants.LATENCY_QUERY) * 1000 latency_vals.append(latency) logger.info(f"Latency values: {latency_vals}") if len(latency_vals) > 1 and latency > 250: # Checking for an exponential growth if latency > latency_vals[0] * 2**7: logger.info("Latency exponential growth was detected") limit_reached = True # In case the latency is greater than 3 seconds, # most chances the limit has been reached if latency > 3000: logger.info(f"Limit was determined by latency, which is " f"higher than 3 seconds - {latency} ms") limit_reached = True # For clusters that their nodes do not meet the minimum # resource requirements, the cluster limit is being reached # while the latency remains low. For that, the cluster limit # needs to be determined by the following condition of IOPS # diff between FIO pod creation iterations iops_diff = (current_iops / previous_iops * 100) - 100 low_diff_counter += 1 if -15 < iops_diff < 10 else 0 if low_diff_counter > 3: logger.warning( f"Limit was determined by low IOPS diff between " f"iterations - {iops_diff:.2f}%") limit_reached = True if time.time() > time_before + time_to_wait: logger.warning( f"Could not determine the cluster IOPS limit within" f"\nthe given {time_to_wait} seconds timeout. Breaking") limit_reached = True cluster_used_space = get_percent_used_capacity() if cluster_used_space > 60: logger.warning( f"Cluster used space is {cluster_used_space}%. Could " f"not reach the cluster IOPS limit before the " f"used spaced reached 60%. Breaking") limit_reached = True self.cluster_limit = cluster_limit logger.info( f"\n===================================\nThe cluster IOPS limit " f"is {self.cluster_limit:.2f}\n===================================" ) logger.info( f"Deleting all DC FIO pods that have FIO rate parameter of {rate}") while self.dc_objs: self.decrease_load(wait=False) # Creating the first pod of small FIO 'rate' param, to speed up the process. # In the meantime, the load will drop, following the deletion of the # FIO pods with large FIO 'rate' param rate = '15M' logger.info( f"Creating FIO pods with a rate parameter of {rate}, one by " f"one, until the target percentage is reached") self.increase_load(rate=rate) target_iops = self.cluster_limit * self.target_percentage current_iops = self.get_query(query=constants.IOPS_QUERY) logger.info(f"Target IOPS: {target_iops}") logger.info(f"Current IOPS: {current_iops}") while current_iops < target_iops * 0.95: wait = False if current_iops < target_iops / 2 else True self.increase_load(rate=rate, wait=wait) previous_iops = current_iops current_iops = self.get_query(query=constants.IOPS_QUERY) logger.info( msg.format(current_iops, previous_iops, len(self.dc_objs))) self.print_metrics() logger.info( f"\n========================================\n" f"The target load, of {self.target_percentage * 100}%, has been reached" f"\n==========================================") def get_query(self, query): """ Get query from Prometheus and parse it Args: query (str): Query to be done Returns: float: the query result """ now = datetime.now timestamp = datetime.timestamp return float( self.prometheus_api.query(query, str(timestamp(now())))[0]['value'][1]) def calc_trim_metric_mean(self, metric=constants.LATENCY_QUERY, samples=5): """ Get the trimmed mean of a given metric Args: metric (str): The metric to calculate the average result for samples (int): The number of samples to take Returns: float: The average result for the metric """ vals = list() for i in range(samples): vals.append(round(self.get_query(metric), 5)) if i == samples - 1: break time.sleep(5) return round(get_trim_mean(vals), 5) def get_metrics(self): """ Get different cluster load and utilization metrics """ return { "throughput": self.get_query(constants.THROUGHPUT_QUERY) * (constants.TP_CONVERSION.get(' B/s')), "latency": self.get_query(constants.LATENCY_QUERY) * 1000, "iops": self.get_query(constants.IOPS_QUERY), "used_space": self.get_query(constants.USED_SPACE_QUERY) / 1e+9 } def print_metrics(self): """ Print metrics """ high_latency = 500 metrics = self.get_metrics() limit_msg = "" pods_msg = "" if self.cluster_limit: limit_msg = ( f"({metrics.get('iops') / self.cluster_limit * 100:.2f}% of the " f"{self.cluster_limit:.1f} limit)\n") if self.dc_objs: pods_msg = (f"\nNumber of pods running FIO: {len(self.dc_objs)}") logger.info( f"\n===============================\n" f"Cluster throughput: {metrics.get('throughput'):.2f} MB/s\n" f"Cluster latency: {metrics.get('latency'):.2f} ms\n" f"Cluster IOPS: {metrics.get('iops'):.2f}\n{limit_msg}" f"Cluster used space: {metrics.get('used_space'):.2f} GB{pods_msg}" f"\n===============================") if metrics.get('latency') > high_latency: logger.warning( f"Cluster latency is higher than {high_latency} ms!")