class PoolFunctions: """ Contains various functions that help in altering the behaviour, working of pools and verify the changes """ def __init__(self, node: CephAdmin): """ initializes the env to run rados commands Args: node: CephAdmin object """ self.rados_obj = RadosOrchestrator(node=node) def verify_target_ratio_set(self, pool_name, ratio): """ Sets the "target_size_ratio" on the given pool and verifies it from the auto-scale status Args: pool_name: name of the pool ratio: ratio to be set Returns: True -> pass, False -> fail """ log.debug(f"Setting ratio: {ratio} on pool: {pool_name}") self.rados_obj.set_pool_property(pool=pool_name, props="target_size_ratio", value=ratio) # sleeping for 2 seconds for pg autoscaler updates the status and new PG's time.sleep(2) ratio_set = self.get_pg_autoscaler_value(pool_name, item="target_ratio") if not ratio_set == ratio: log.error("specified target ratio not set on the pool") return False return True def get_pg_autoscaler_value(self, pool_name, item): """ Fetches the target ratio set on the pool given Args: pool_name: name of the pool item: Value of the item to be fetched. Allowed values: actual_capacity_ratio|actual_raw_used|bias|capacity_ratio|crush_root_id|target_bytes| effective_target_ratio|logical_used|pg_autoscale_mode|pg_num_target|pool_id|raw_used|target_ratio| Returns: Requested value """ cmd = "ceph osd pool autoscale-status" autoscale_status = self.rados_obj.run_ceph_command(cmd=cmd) try: pool_details = [ details for details in autoscale_status if details["pool_name"] == pool_name ][0] except Exception: log.error("Pool not found") return pool_details[item] def fill_omap_entries(self, pool_name, **kwargs): """ creates key-value entries for objects on ceph pools and increase the omap entries on the pool eg : if obj_start, obj_end: 0, 3000 objects, with num_keys 1000, the method would create 3000 objects with 1k KW pairs each. so total 3000*1000 KW entries Args: pool_name: name of the pool where the KW pairs needed to be added to objects **kwargs: other args that can be passed Valid args: 1. obj_start: start count for object creation 2. obj_end : end count for object creation 3. num_keys_obj: Number of KW paris to be added to each object Returns: True -> pass, False -> fail """ # Getting the client node to perform the operations client_node = self.rados_obj.ceph_cluster.get_nodes(role="client")[0] obj_start = kwargs.get("obj_start", 0) obj_end = kwargs.get("obj_end", 2000) num_keys_obj = kwargs.get("num_keys_obj", 20000) log.debug(f"Writing {(obj_end - obj_start) * num_keys_obj} Key paris" f" to increase the omap entries on pool {pool_name}") script_loc = "https://raw.githubusercontent.com/red-hat-storage/cephci/master/utility/generate_omap_entries.py" client_node.exec_command( sudo=True, cmd=f"curl -k {script_loc} -O", ) # Setup Script pre-requisites : docopt client_node.exec_command(sudo=True, cmd="pip3 install docopt", long_running=True) cmd_options = f"--pool {pool_name} --start {obj_start} --end {obj_end} --key-count {num_keys_obj}" cmd = f"python3 generate_omap_entries.py {cmd_options}" client_node.exec_command(sudo=True, cmd=cmd, long_running=True) # removing the py file copied client_node.exec_command(sudo=True, cmd="rm -rf generate_omap_entries.py") log.debug("Checking the amount of omap entries created on the pool") pool_stats = self.rados_obj.run_ceph_command( cmd="ceph df detail")["pools"] for detail in pool_stats: if detail["name"] == pool_name: pool_1_stats = detail["stats"] total_omap_data = pool_1_stats["omap_bytes_used"] omap_data = pool_1_stats["stored_omap"] break if omap_data < 0: log.error("No omap entries written into pool") return False log.info(f"Wrote {omap_data} bytes of omap data on the pool." f"Total stored omap data on pool : {total_omap_data}") return True def do_rados_delete(self, pool_name: str, pg_id: str = None): """ deletes all the objects from the given pool / PG ID Args: 1. pool_name: name of the pool 2. [ pg_id ]: Pg ID (Optional, but when provided, should be passed along with pool name ) Returns: True -> pass, False -> fail """ obj_cmd = f"rados -p {pool_name} ls" if pg_id: obj_cmd = f"rados --pgid {pg_id} ls" delete_obj_list = self.rados_obj.run_ceph_command(cmd=obj_cmd, timeout=1000) for obj in delete_obj_list: cmd = f"rados -p {pool_name} rm {obj['name']}" self.rados_obj.node.shell([cmd], long_running=True) # Sleeping for 3 seconds for object reference to be deleted time.sleep(3) # Checking if object is still present in the pool out = self.rados_obj.run_ceph_command(cmd=obj_cmd, timeout=1000) rem_objs = [obj["name"] for obj in out] if obj["name"] in rem_objs: log.error(f"Object {obj['name']} not deleted in the pool") return False log.debug(f"deleted object: {obj['name']} from pool {pool_name}") log.info(f"Completed deleting all objects from pool {pool_name}") return True def create_pool_snap(self, pool_name: str): """ Creates snapshots of the given pool Args: pool_name: name of the pool Returns: Pass -> name of the snapshot created, Fail -> False """ # Checking if snapshots can be created on the supplied pool cmd = "ceph osd dump" pool_status = self.rados_obj.run_ceph_command(cmd=cmd, timeout=800) for detail in pool_status["pools"]: if detail["pool_name"] != pool_name: continue if "selfmanaged_snaps" in detail["flags_names"]: # bz: https://bugzilla.redhat.com/show_bug.cgi?id=1425803#c2 log.error( f"Pool {pool_name} is a self managed pool, cannot create snaps manually" ) return False # Creating snaps on the pool provided cmd = "uuidgen" out, err = self.rados_obj.node.shell([cmd]) uuid = out[0:5] snap_name = f"{pool_name}-snap-{uuid}" cmd = f"ceph osd pool mksnap {pool_name} {snap_name}" self.rados_obj.node.shell([cmd], long_running=True) # Checking if snap was created successfully if not self.check_snap_exists(snap_name=snap_name, pool_name=pool_name): log.error("Snapshot of pool not created") return False log.debug(f"Created snapshot {snap_name} on pool {pool_name}") return snap_name def check_snap_exists(self, snap_name: str, pool_name: str) -> bool: """ checks the existence of the snapshot name given on the pool Args: snap_name: Name of the snapshot pool_name: Name of the pool Returns: True -> Snapshot exists, False -> snapshot does not exist """ snap_list = self.get_snap_names(pool_name=pool_name) return True if snap_name in snap_list else False def get_snap_names(self, pool_name: str) -> list: """ Fetches the list of snapshots created on the given pool Args: pool_name: name of the pool Returns: list of the snaps created """ cmd = "ceph osd dump" pool_status = self.rados_obj.run_ceph_command(cmd=cmd, timeout=800) for detail in pool_status["pools"]: if detail["pool_name"] == pool_name: snap_list = [snap["name"] for snap in detail["pool_snaps"]] log.debug(f"snapshots on pool : {snap_list}") return snap_list def delete_pool_snap(self, pool_name: str, snap_name: str = None) -> bool: """ deletes snapshots of the given pool. If no snap name is provided, deletes all the snapshots on the pool Args: pool_name: name of the pool snap_name: name of the snapshot Returns: Pass -> snapshot Deleted, Fail -> snapshot not Deleted """ if snap_name: delete_list = list(snap_name) else: delete_list = self.get_snap_names(pool_name=pool_name) # Deleting snaps on the pool provided for snap in delete_list: cmd = f"ceph osd pool rmsnap {pool_name} {snap}" self.rados_obj.node.shell([cmd]) # Checking if snap was deleted successfully if self.check_snap_exists(snap_name=snap_name, pool_name=pool_name): log.error("Snapshot of pool exists") return False log.debug(f"deleted snapshot {snap} on pool {pool_name}") log.debug("Deleted provided snapshots on the pool") return True def get_bulk_details(self, pool_name: str) -> bool: """ Checks the status of bulk flag on the pool given Args: pool_name: Name of the pool Returns: True -> pass, False -> fail """ # Checking if the sent pool already exists. if pool_name not in self.rados_obj.list_pools(): log.error(f"Pool {pool_name} does not exist") return False # Getting the bulk status obj = self.rados_obj.get_pool_property(pool=pool_name, props="bulk") return obj["bulk"] def set_bulk_flag(self, pool_name: str) -> bool: """ Sets the bulk flag to true on existing pools Args: pool_name: Name of the pool Returns: True -> pass, False -> fail """ # Checking if the sent pool already exists. If does not, creating new pool if pool_name not in self.rados_obj.list_pools(): log.info( f"Pool {pool_name} does not exist, creating new pool with bulk enabled" ) if not self.rados_obj.create_pool(pool_name=pool_name, bulk=True): log.error("Failed to create the replicated Pool") return False # Enabling bulk on already existing pool if not self.rados_obj.set_pool_property( pool=pool_name, props="bulk", value="true"): log.error(f"Could not set the bulk flag on pool {pool_name}") return False # Sleeping for 2 seconds after pool create/Modify for PG's to be calculated with bulk time.sleep(2) # Checking if the bulk is enabled or not return self.get_bulk_details(pool_name=pool_name) def rm_bulk_flag(self, pool_name: str) -> bool: """ Removes the bulk flag on existing pools Args: pool_name: Name of the pool Returns: True -> pass, False -> fail """ # Checking if the sent pool already exists. if pool_name not in self.rados_obj.list_pools(): log.info(f"Pool {pool_name} does not exist") return False # Enabling bulk on already existing pool if not self.rados_obj.set_pool_property( pool=pool_name, props="bulk", value="false"): log.error(f"Could not unset the bulk flag on pool {pool_name}") return False # Sleeping for 2 seconds after pool create/Modify for PG's to be calculated with bulk time.sleep(2) # Checking if the bulk is enabled or not return not self.get_bulk_details(pool_name=pool_name) def get_target_pg_num_bulk_flag(self, pool_name: str) -> int: """ Fetches the target PG counts for the given pool from the autoscaler status Args: pool_name: Name of the pool Returns: PG Count """ # Checking the autoscaler status, final PG counts, bulk flags cmd = "ceph osd pool autoscale-status" pool_status = self.rados_obj.run_ceph_command(cmd=cmd) for entry in pool_status: if entry["pool_name"] == pool_name: return int(entry["pg_num_final"])
def verify_mon_db_trim(ceph_cluster, node: CephAdmin, **kwargs): """ The Mon DB size should be reduced by removing the old mappings regularly. To verify this behaviour, Creating various scenarios where the DB would be updated with new mappings and verify it DB is getting trimmed. Verifies BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1905339 https://bugzilla.redhat.com/show_bug.cgi?id=1829646 https://bugzilla.redhat.com/show_bug.cgi?id=1972281 https://bugzilla.redhat.com/show_bug.cgi?id=1943357 https://bugzilla.redhat.com/show_bug.cgi?id=1766702 Args: ceph_cluster (ceph.ceph.Ceph): ceph cluster node: Cephadm node where the commands need to be executed kwargs: Any other KV pairs that need to be passed for testing Returns: Exception hit for failure conditions. """ # Creating rados object to run rados commands rados_obj = RadosOrchestrator(node=node) mon_nodes = ceph_cluster.get_nodes(role="mon") osd_nodes = ceph_cluster.get_nodes(role="osd") client_node = ceph_cluster.get_nodes(role="client")[0] installer_node = ceph_cluster.get_nodes(role="installer")[0] daemon_info = rados_obj.run_ceph_command(cmd="ceph orch ps") mon_daemons = [ entry for entry in daemon_info if entry["daemon_type"] == "mon" ] # Duration for which we will sleep after the mon DB changes are made and mon would have begun trimming old mappings mon_db_trim_wait_dur = 1200 # List to capture the mon db size throughout the duration of the test to check the variations in DB size mon_db_size_list = list() mon_db_size_list.append(get_mondb_size(mon_nodes[0], mon_daemons)) # Collecting first and last commits to osdmap status = rados_obj.run_ceph_command(cmd="ceph report") init_commmits = { "osdmap_first_committed": float(status["osdmap_first_committed"]), "osdmap_last_committed": float(status["osdmap_last_committed"]), } # creating scenarios where the mon db would be updated with new info change_config_for_slow_ops(rados_obj=rados_obj, action="set", **kwargs) mon_db_size_list.append(get_mondb_size(mon_nodes[0], mon_daemons)) # Starting read and write on by creating a test pool . pool_name = "test_pool_ops" if not rados_obj.create_pool(pool_name=pool_name, crush_rule="stretch_rule"): error = "failed to create pool to run IO" raise TestCaseFailureException(error) cmd = f"rados --no-log-to-stderr -b 1024 -p {pool_name} bench 400 write --no-cleanup &" client_node.exec_command(sudo=True, cmd=cmd) mon_db_size_list.append(get_mondb_size(mon_nodes[0], mon_daemons)) # deleting a previously created pool to increase OSD operations and map changes # Pool created as part of suite set-up workflow. rados_obj.detete_pool(pool="delete_pool") # Proceeding to reboot 1 OSD from each host to trigger rebalance & Backfill cluster_fsid = rados_obj.run_ceph_command(cmd="ceph fsid")["fsid"] daemon_info = rados_obj.run_ceph_command(cmd="ceph orch ps") osd_daemons = [ entry for entry in daemon_info if entry["daemon_type"] == "osd" ] for onode in osd_nodes: for osd in osd_daemons: if re.search(osd["hostname"], onode.hostname): # Not using the container ID's provided in ceph orch ps command. # Bug : https://bugzilla.redhat.com/show_bug.cgi?id=1943494 # cmd = f"podman restart {osd['container_id']}" cmd = f"systemctl restart ceph-{cluster_fsid}@osd.{osd['daemon_id']}.service" log.info( f"rebooting osd-{osd['daemon_id']} on host {osd['hostname']}. Command {cmd}" ) onode.exec_command(sudo=True, cmd=cmd) # Sleeping for 5 seconds for status to be updated time.sleep(5) break # Re-weighting the OSd's based on usage to trigger rebalance # todo: Verify re-balancing process on OSD's ( PG movement across cluster) # todo: Add re-balancing based on crush item provided # BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1766702 rados_obj.reweight_crush_items() """ Waiting for 2 hours for cluster to get to active + clean state. Rationale: during cluster activities like backfill, rebalance, osd map change cause Mon DB to be updated. Hence, we can wait till mon DB updates are completed, after which DB size should be reduced, by trimming the old mappings once the new mappings are added. If cluster healthy state is reached within 2 hours, we exit the loop earlier, without waiting for stipulated time, But if cluster is still performing operations for long time, we would need at-least some data to make sure DB is not just increasing. ( DB is expected to increase when operations are in progress. Old mappings are removed when operations/ new updates are completed. ) """ end_time = datetime.datetime.now() + datetime.timedelta(seconds=7200) while end_time > datetime.datetime.now(): status_report = rados_obj.run_ceph_command(cmd="ceph report") ceph_health_status = status_report["health"] recovery_tuple = ("OSD_DOWN", "PG_AVAILABILITY", "PG_DEGRADED", "SLOW_OPS") daemon_info = rados_obj.run_ceph_command(cmd="ceph orch ps") mon_daemons = [ entry for entry in daemon_info if entry["daemon_type"] == "mon" ] mon_db_size_list.append(get_mondb_size(mon_nodes[0], mon_daemons)) # Checking for any health warnings that increase db size flag = (True if not any(key in ceph_health_status["checks"].keys() for key in recovery_tuple) else False) # Proceeding to check if all PG's are in active + clean if flag: for entry in status_report["num_pg_by_state"]: rec = ("remapped", "backfilling", "degraded") if any(key in rec for key in entry["state"].split("+")): flag = False if flag: log.info( f"The recovery and back-filling of the OSD is completed" f"Sleeping for {mon_db_trim_wait_dur} Seconds after clean for mon to remove old mappings" ) time.sleep(mon_db_trim_wait_dur / 2) mon_db_size_list.append(get_mondb_size(mon_nodes[0], mon_daemons)) time.sleep(mon_db_trim_wait_dur) break log.info( f"Waiting for active + clean. Active aletrs: {ceph_health_status['checks'].keys()}," f" checking status again in 1 minute") time.sleep(60) # Checking if any slow operations are reported and waiting for 'dur' for the slow_ops to be cleared end_time = datetime.datetime.now() + datetime.timedelta( seconds=mon_db_trim_wait_dur) while end_time > datetime.datetime.now(): if not get_slow_ops_data( node=node, installer=installer_node, action="current"): log.info("Operations in progress, checking again in 30 seconds") time.sleep(30) continue # Logging all the historic operations for reference / future enhancement get_slow_ops_data(node=node, installer=installer_node, action="historic") break # collecting the final size of the Mon DB and the OSD map epoch times daemon_info = rados_obj.run_ceph_command(cmd="ceph orch ps") mon_daemons = [ entry for entry in daemon_info if entry["daemon_type"] == "mon" ] final_db_size = get_mondb_size(mon_nodes[0], mon_daemons) final_status = rados_obj.run_ceph_command(cmd="ceph report") final_commmits = { "osdmap_first_committed": float(final_status["osdmap_first_committed"]), "osdmap_last_committed": float(final_status["osdmap_last_committed"]), } mon_db_max_size = max(mon_db_size_list) # Getting the trend of mon DB size when the operations were running on cluster. mon_db_size_size_change = list() for i in range(len(mon_db_size_list) - 1): mon_db_size_size_change.append(mon_db_size_list[i + 1] - mon_db_size_list[i]) # Reverting the config changes made for generation slow_ops change_config_for_slow_ops(rados_obj=rados_obj, action="rm", **kwargs) # Checking the final results if True not in list(map(lambda x: x <= 0, mon_db_size_size_change)): error = f"The mon DB is only increasing since the test begun. DB sizes {mon_db_size_list}" raise TestCaseFailureException(error) if not final_db_size <= mon_db_max_size: error = ( f"The mon DB size after cluster clean is higher than when operations were being performed.\n" f"max size during operations : {mon_db_max_size} , final DB size after clean {final_db_size}" ) log.error(error) raise TestCaseFailureException() # Initial update of OSD maps can be the same at the beginning and end of test if (final_commmits["osdmap_first_committed"] < init_commmits["osdmap_first_committed"]): error = ( f"The OSD map has not been updated of first commits\n" f"The commits are initial commits : {init_commmits}, final commits : {final_commmits}" ) log.error(error) raise TestCaseFailureException() # Final updates need to be more than initial updates as there are OSD map changes during the duration of test if (final_commmits["osdmap_last_committed"] <= init_commmits["osdmap_last_committed"]): error = ( f"The OSD map has not been updated of last commits\n" f"The commits are initial commits : {init_commmits}, final commits : {final_commmits}" ) log.error(error) raise TestCaseFailureException() # The number of OSD mappings present on cluster should not exceed 800 in total # https://tracker.ceph.com/issues/37875#note-1 if (final_commmits["osdmap_last_committed"] - final_commmits["osdmap_first_committed"]) > 800: error = ( f"There are still too many old commits in Mon DB. OSD map not trimmed as per needed\n" f"The commits are initial commits : {init_commmits}, final commits : {final_commmits}" ) log.error(error) raise TestCaseFailureException() # Checking the paxos trimming sizes if (int(final_status["paxos"]["last_committed"]) - int(final_status["paxos"]["first_committed"])) > 1000: error = ( f"There are still too many old commits in Mon DB.\n" f"The commits are initial commits : {final_status['paxos']['first_committed']}," f" final commits : {final_status['paxos']['last_committed']}") log.error(error) raise TestCaseFailureException() log.info("mon DB was trimmed successfully")
def run(ceph_cluster, **kw): """ Prepares the cluster to run rados tests. Actions Performed: 1. Create a Replicated and Erasure coded pools and write Objects into pools 2. Setup email alerts for sending errors/warnings on the cluster. Verifies Bugs: https://bugzilla.redhat.com/show_bug.cgi?id=1849894 https://bugzilla.redhat.com/show_bug.cgi?id=1878145 3. Enable logging into file and check file permissions Verifies Bug : https://bugzilla.redhat.com/show_bug.cgi?id=1884469 Args: ceph_cluster (ceph.ceph.Ceph): ceph cluster kw: Args that need to be passed to the test for initialization Returns: 1 -> Fail, 0 -> Pass """ log.info(run.__doc__) config = kw["config"] cephadm = CephAdmin(cluster=ceph_cluster, **config) rados_obj = RadosOrchestrator(node=cephadm) mon_obj = MonConfigMethods(rados_obj=rados_obj) out, err = cephadm.shell(["uuidgen"]) uuid = out.split("-")[0] if config.get("ec_pool"): ec_config = config.get("ec_pool") ec_config.setdefault("pool_name", f"ecpool_{uuid}") if not rados_obj.create_erasure_pool(name=uuid, **ec_config): log.error("Failed to create the EC Pool") return 1 if ec_config.get("test_overwrites_pool"): if not rados_obj.verify_ec_overwrites(**ec_config): log.error("Failed to create the EC Pool") return 1 else: if not rados_obj.bench_write(**ec_config): log.error("Failed to write objects into the EC Pool") return 1 rados_obj.bench_read(**ec_config) log.info( "Created the EC Pool, Finished writing data into the pool") if ec_config.get("delete_pool"): if not rados_obj.detete_pool(pool=ec_config["pool_name"]): log.error("Failed to delete EC Pool") return 1 if config.get("replicated_pool"): rep_config = config.get("replicated_pool") rep_config.setdefault("pool_name", f"repool_{uuid}") if not rados_obj.create_pool(**rep_config, ): log.error("Failed to create the replicated Pool") return 1 if not rados_obj.bench_write(**rep_config): log.error("Failed to write objects into the EC Pool") return 1 rados_obj.bench_read(**rep_config) log.info( "Created the replicated Pool, Finished writing data into the pool") if rep_config.get("delete_pool"): if not rados_obj.detete_pool(pool=rep_config["pool_name"]): log.error("Failed to delete replicated Pool") return 1 if config.get("set_pool_configs"): changes = config["set_pool_configs"] pool_name = changes["pool_name"] configurations = changes["configurations"] for conf in configurations.keys(): if not rados_obj.set_pool_property( pool=pool_name, props=conf, value=configurations[conf]): log.error(f"failed to set property {conf} on the cluster") return 1 log.info(f"made the config changes on the pool {pool_name}") if config.get("email_alerts"): alert_config = config.get("email_alerts") if not rados_obj.enable_email_alerts(**alert_config): log.error("Error while configuring email alerts") return 1 log.info("email alerts configured") if config.get("Verify_config_parameters"): test_config = config.get("Verify_config_parameters") test_node = ceph_cluster.get_nodes(role="osd")[0] for conf in test_config["configurations"]: for entry in conf.values(): if entry.get("location_type") == "host": entry["location_value"] = test_node.hostname if not mon_obj.set_config(**entry): log.error(f"Error setting config {conf}") return 1 log.info("done") pool_name = "test_pool_1" if not rados_obj.create_pool(pool_name=pool_name, pg_num=16): log.error("Failed to create the replicated Pool") return 1 rados_obj.bench_write(pool_name=pool_name, rados_write_duration=50) # Removing test configurations for conf in test_config["configurations"]: for entry in conf.values(): if entry.get("location_type") == "host": entry["location_value"] = test_node.hostname if not mon_obj.remove_config(**entry): log.error(f"Error setting config {conf}") return 1 log.info("finished removing values, passed") if config.get("log_to_file"): if not rados_obj.enable_file_logging(): log.error("Error while setting config to enable logging into file") return 1 log.info("Logging to file configured") if config.get("cluster_configuration_checks"): cls_config = config.get("cluster_configuration_checks") if not rados_obj.set_cluster_configuration_checks(**cls_config): log.error("Error while setting Cluster config checks") return 1 log.info("Set up cluster configuration checks") if config.get("configure_balancer"): balancer_config = config.get("configure_balancer") if not rados_obj.enable_balancer(**balancer_config): log.error("Error while setting up balancer on the Cluster") return 1 log.info("Set up Balancer on the cluster") if config.get("configure_pg_autoscaler"): autoscaler_config = config.get("configure_pg_autoscaler") if not rados_obj.configure_pg_autoscaler(**autoscaler_config): log.error("Error while setting up pg_autoscaler on the Cluster") return 1 log.info("Set up pg_autoscaler on the cluster") if config.get("enable_compression"): compression_conf = config["enable_compression"] pool_name = compression_conf["pool_name"] for conf in compression_conf["configurations"]: for entry in conf.values(): if not rados_obj.pool_inline_compression(pool_name=pool_name, **entry): log.error( f"Error setting compression on pool : {pool_name} for config {conf}" ) return 1 if not rados_obj.bench_write(**compression_conf): log.error("Failed to write objects into Pool") return 1 rados_obj.bench_read(**compression_conf) log.info( "Created the replicated Pool, Finished writing data into the pool" ) log.info("Completed compression tests") if config.get("delete_pools"): for name in config["delete_pools"]: if not rados_obj.detete_pool(name): log.error(f"the pool {name} could not be deleted") return 1 log.info("deleted all the given pools successfully") log.info("All Pre-requisites completed to run Rados suite") return 0
def run(ceph_cluster, **kw): """ Performs various pool related validation tests Returns: 1 -> Fail, 0 -> Pass """ log.info(run.__doc__) config = kw["config"] cephadm = CephAdmin(cluster=ceph_cluster, **config) rados_obj = RadosOrchestrator(node=cephadm) mon_obj = MonConfigMethods(rados_obj=rados_obj) pool_obj = PoolFunctions(node=cephadm) if config.get("ec_pool_recovery_improvement"): ec_config = config.get("ec_pool_recovery_improvement") if not rados_obj.create_erasure_pool(name="recovery", **ec_config): log.error("Failed to create the EC Pool") return 1 if not rados_obj.bench_write(**ec_config): log.error("Failed to write objects into the EC Pool") return 1 rados_obj.bench_read(**ec_config) log.info("Created the EC Pool, Finished writing data into the pool") # getting the acting set for the created pool acting_pg_set = rados_obj.get_pg_acting_set( pool_name=ec_config["pool_name"]) if len(acting_pg_set) != ec_config["k"] + ec_config["m"]: log.error( f"acting set consists of only these : {acting_pg_set} OSD's, less than k+m" ) return 1 log.info( f" Acting set of the pool consists of OSD's : {acting_pg_set}") log.info( f"Killing m, i.e {ec_config['m']} OSD's from acting set to verify recovery" ) stop_osds = [acting_pg_set.pop() for _ in range(ec_config["m"])] for osd_id in stop_osds: if not rados_obj.change_osd_state(action="stop", target=osd_id): log.error(f"Unable to stop the OSD : {osd_id}") return 1 log.info( "Stopped 'm' number of OSD's from, starting to wait for recovery") rados_obj.change_recover_threads(config=ec_config, action="set") # Sleeping for 25 seconds ( "osd_heartbeat_grace": "20" ) for osd's to be marked down time.sleep(25) # Waiting for up to 2.5 hours for the recovery to complete and PG's to enter active + Clean state end_time = datetime.datetime.now() + datetime.timedelta(seconds=9000) while end_time > datetime.datetime.now(): flag = True status_report = rados_obj.run_ceph_command(cmd="ceph report") # Proceeding to check if all PG's are in active + clean for entry in status_report["num_pg_by_state"]: rec = ( "backfilling", "degraded", "incomplete", "recovering", "recovery_wait", "backfilling_wait", "peered", "undersized", ) if any(key in rec for key in entry["state"].split("+")): flag = False if flag: log.info( "The recovery and back-filling of the OSD is completed") break log.info( f"Waiting for active + clean. Active aletrs: {status_report['health']['checks'].keys()}," f"PG States : {status_report['num_pg_by_state']}" f" checking status again in 1 minute") time.sleep(60) # getting the acting set for the created pool after recovery acting_pg_set = rados_obj.get_pg_acting_set( pool_name=ec_config["pool_name"]) if len(acting_pg_set) != ec_config["k"] + ec_config["m"]: log.error( f"acting set consists of only these : {acting_pg_set} OSD's, less than k+m" ) return 1 log.info( f" Acting set of the pool consists of OSD's : {acting_pg_set}") # Changing recovery threads back to default rados_obj.change_recover_threads(config=ec_config, action="rm") log.debug("Starting the stopped OSD's") for osd_id in stop_osds: if not rados_obj.change_osd_state(action="restart", target=osd_id): log.error(f"Unable to restart the OSD : {osd_id}") return 1 # Sleep for 5 seconds for OSD's to join the cluster time.sleep(5) if not flag: log.error( "The pool did not reach active + Clean state after recovery") return 1 # Deleting the pool created if not rados_obj.detete_pool(pool=ec_config["pool_name"]): log.error( f"the pool {ec_config['pool_name']} could not be deleted") return 1 log.info("Successfully tested EC pool recovery with K osd's surviving") return 0 if config.get("Compression_tests"): """ Create a 2 replicated pools: 1. Pool_1 : enable any compression algorithm(def snappy) and compression mode(aggressive/force). 2. Pool_2 : set compression mode to none Writing the same amount of data on 2 pools, size of pool with compression on would consume less space """ pool_config = config["Compression_tests"]["pool_config"] compression_config = config["Compression_tests"]["compression_config"] pool_1 = pool_config["pool-1"] pool_2 = pool_config["pool-2"] if config["Compression_tests"]["pool_type"] == "replicated": if not rados_obj.create_pool(pool_name=pool_1, **pool_config): log.error("could not create pool-1") return 1 if not rados_obj.create_pool(pool_name=pool_2, **pool_config): log.error("could not create pool-2") return 1 elif config["Compression_tests"]["pool_type"] == "erasure": pool_config["pool_name"] = pool_1 if not rados_obj.create_erasure_pool(name=pool_1, **pool_config): log.error("could not create pool-1") return 1 pool_config["pool_name"] = pool_2 if not rados_obj.create_erasure_pool(name=pool_2, **pool_config): log.error("could not create pool-2") return 1 del pool_config["pool_name"] log.debug("Created two pools to test compression") # Enabling compression on pool-1 if not rados_obj.pool_inline_compression(pool_name=pool_1, **compression_config): log.error( f"Error setting compression on pool : {pool_1} for config {compression_config}" ) return 1 # Writing the same amount of data into two pools if not rados_obj.bench_write(pool_name=pool_1, **pool_config): log.error( "Failed to write objects into Pool-1, with compression enabled" ) return 1 if not rados_obj.bench_write(pool_name=pool_2, **pool_config): log.error( "Failed to write objects into Pool-2, without compression enabled" ) return 1 # Sleeping for 5 seconds for status to be updated. time.sleep(5) log.debug( "Finished writing data into the two pools. Checking pool stats") try: pool_stats = rados_obj.run_ceph_command( cmd="ceph df detail")["pools"] pool_1_stats = [ detail for detail in pool_stats if detail["name"] == pool_1 ][0]["stats"] pool_2_stats = [ detail for detail in pool_stats if detail["name"] == pool_2 ][0]["stats"] except KeyError: log.error( "No stats about the pools requested found on the cluster") return 1 log.debug(f"Pool-1 stats: {pool_1_stats}") log.debug(f"Pool-2 stats: {pool_2_stats}") if pool_1_stats["compress_bytes_used"] < 0: log.error("No data stored under pool-1 is compressed") return 1 if pool_1_stats["kb_used"] >= pool_2_stats["kb_used"]: log.error("Compression has no effect on the pool size...") return 1 if config["Compression_tests"].get("verify_compression_ratio_set"): # added verification for test: CEPH-83571672 if not rados_obj.check_compression_size(pool_name=pool_1, **compression_config): log.error("data not compressed in accordance to ratio set") return 1 log.info("Pool size is less when compression is enabled") return 0 if config.get("test_autoscaler_bulk_feature"): """ Tests to verify the autoscaler bulk flag, which allows pools to make use of scale-down profile, making those pools start with full compliments of PG sets. Tests include 1. creating new pools with bulk, 2. enabling/disabling bulk flag on existing pools 3. Verify the PG changes when the flag is set/unset Verifies bugs : https://bugzilla.redhat.com/show_bug.cgi?id=2049851 """ regex = r"\s*(\d.\d)-rhel-\d" build = (re.search(regex, config.get("build", config.get("rhbuild")))).groups()[0] if not float(build) > 5.0: log.info( "Test running on version less than 5.1, skipping verifying bulk flags" ) return 0 # Creating a pool with bulk feature pool_name = config.get("pool_name") if not pool_obj.set_bulk_flag(pool_name=pool_name): log.error("Failed to create a pool with bulk features") return 1 # Checking the autoscaler status, final PG counts, bulk flags pg_target_init = pool_obj.get_target_pg_num_bulk_flag( pool_name=pool_name) # Unsetting the bulk flag and checking the change in the PG counts if not pool_obj.rm_bulk_flag(pool_name=pool_name): log.error("Failed to create a pool with bulk features") return 1 # Sleeping for 5 seconds for new PG num to bets et time.sleep(5) pg_target_interim = pool_obj.get_target_pg_num_bulk_flag( pool_name=pool_name) # The target PG's once the flag is disabled must be lesser than when enabled if pg_target_interim >= pg_target_init: log.error("PG's not reduced after bulk flag disabled") return 1 # Setting the bulk flag on pool again and checking the change in the PG counts if not pool_obj.set_bulk_flag(pool_name=pool_name): log.error("Failed to disable/remove bulk features on pool") return 1 # Sleeping for 5 seconds for new PG num to bets et time.sleep(5) pg_target_final = pool_obj.get_target_pg_num_bulk_flag( pool_name=pool_name) # The target PG's once the flag is disabled must be lesser than when enabled if pg_target_interim >= pg_target_final: log.error("PG's not Increased after bulk flag Enabled") return 1 if config.get("delete_pool"): rados_obj.detete_pool(pool=pool_name) log.info("Verified the workings of bulk flag") return 0 if config.get("verify_pool_target_ratio"): log.debug("Verifying target size ratio on pools") target_configs = config["verify_pool_target_ratio"]["configurations"] # Creating pools and starting the test for entry in target_configs.values(): log.debug(f"Creating {entry['pool_type']} pool on the cluster") if entry.get("pool_type", "replicated") == "erasure": method_should_succeed(rados_obj.create_erasure_pool, name=entry["pool_name"], **entry) else: method_should_succeed( rados_obj.create_pool, **entry, ) rados_obj.bench_write(**entry) if not pool_obj.verify_target_ratio_set( pool_name=entry["pool_name"], ratio=entry["target_size_ratio"]): log.error( f"Could not change the target ratio on the pool: {entry['pool_name']}" ) return 1 log.debug("Set the ratio. getting the projected pg's") rados_obj.change_recover_threads(config=config, action="set") log.debug( "Waiting for the rebalancing to complete on the cluster after the change" ) # Sleeping for 2 minutes for rebalancing to start & for new PG count to be updated. time.sleep(120) new_pg_count = int( pool_obj.get_pg_autoscaler_value(pool_name=entry["pool_name"], item="pg_num_target")) if new_pg_count <= entry["pg_num"]: log.error( f"Count of PG's not increased on the pool: {entry['pool_name']}" f"Initial creation count : {entry['pg_num']}" f"New count after setting num target : {new_pg_count}") return 1 res = wait_for_clean_pg_sets(rados_obj) if not res: log.error( "PG's in cluster are not active + Clean after the ratio change" ) return 1 if not pool_obj.verify_target_ratio_set( pool_name=entry["pool_name"], ratio=0.0): log.error( f"Could not remove the target ratio on the pool: {entry['pool_name']}" ) return 1 # Sleeping for 2 minutes for rebalancing to start & for new PG count to be updated. time.sleep(120) # Checking if after the removal of ratio, the PG count has reduced end_pg_count = int( pool_obj.get_pg_autoscaler_value(pool_name=entry["pool_name"], item="pg_num_target")) if end_pg_count >= new_pg_count: log.error( f"Count of PG's not changed/ reverted on the pool: {entry['pool_name']}" f" after removing the target ratios") return 1 rados_obj.change_recover_threads(config=config, action="rm") if entry.get("delete_pool", False): rados_obj.detete_pool(pool=entry["pool_name"]) log.info( f"Completed the test of target ratio on pool: {entry['pool_name']} " ) log.info("Target ratio tests completed") return 0 if config.get("verify_mon_target_pg_per_osd"): pg_conf = config.get("verify_mon_target_pg_per_osd") if not mon_obj.set_config(**pg_conf): log.error("Could not set the value for mon_target_pg_per_osd ") return 1 mon_obj.remove_config(**pg_conf) log.info("Set and verified the value for mon_target_pg_per_osd ") return 0 if config.get("verify_pg_num_min"): log.debug("Verifying pg_num_min on pools") target_configs = config["verify_pg_num_min"]["configurations"] # Creating pools and starting the test for entry in target_configs.values(): log.debug(f"Creating {entry['pool_type']} pool on the cluster") if entry.get("pool_type", "replicated") == "erasure": method_should_succeed(rados_obj.create_erasure_pool, name=entry["pool_name"], **entry) else: method_should_succeed( rados_obj.create_pool, **entry, ) rados_obj.bench_write(**entry) if not rados_obj.set_pool_property(pool=entry["pool_name"], props="pg_num_min", value=entry["pg_num_min"]): log.error("Could not set the pg_min_size on the pool") return 1 if entry.get("delete_pool", False): rados_obj.detete_pool(pool=entry["pool_name"]) log.info( f"Completed the test of pg_min_num on pool: {entry['pool_name']} " ) log.info("pg_min_num tests completed") return 0
def run(ceph_cluster, **kw): """ enables connectivity mode and deploys stretch cluster with arbiter mon node Actions Performed: 1. Disables the automatic crush map update 2. Collects the OSD daemons in the cluster and split them into 2 sites. 3. If add capacity is selected, only half of the OSD's will be added to various sites initially. 4. Adds the stretch rule into crush map. 5. Adding monitors into the 2 sites. 6. Create a replicated pool and deploy stretch mode. 7. Create a test pool, write some data and perform add capacity. ( add osd nodes into two sites ) 8. Check for the bump in election epochs throughout. 9. Check the acting set in PG for 4 OSD's. 2 from each site. Verifies bugs: [1]. https://bugzilla.redhat.com/show_bug.cgi?id=1937088 [2]. https://bugzilla.redhat.com/show_bug.cgi?id=1952763 Args: ceph_cluster (ceph.ceph.Ceph): ceph cluster """ log.info("Deploying stretch cluster with arbiter mon node") log.info(run.__doc__) config = kw.get("config") cephadm = CephAdmin(cluster=ceph_cluster, **config) rados_obj = RadosOrchestrator(node=cephadm) mon_obj = MonElectionStrategies(rados_obj=rados_obj) client_node = ceph_cluster.get_nodes(role="client")[0] tiebreaker_node = ceph_cluster.get_nodes(role="installer")[0] if not client_node and not tiebreaker_node: log.error( "Admin client and tie breaker node not configured, Cannot modify crush rules for stretch cluster" ) return 1 mon_state = get_mon_details(node=cephadm) if len(list(mon_state["monitors"])) < 5: log.error( f"Minimum of 5 Mon daemons needed to deploy a stretch cluster, found : {len(mon_state['monitors'])}" ) return 1 osd_details = get_osd_details(node=cephadm) if len(osd_details.keys()) < 4: log.error( f"Minimum of 4 osd daemons needed to deploy a stretch cluster, found : {len(osd_details.keys())}" ) return 1 if config.get("verify_forced_recovery"): log.info("Verifying forced recovery and healthy in stretch environment") pool_name = "stretch_pool_recovery" if not rados_obj.create_pool(pool_name=pool_name, pg_num=16): log.error("Failed to create the replicated Pool") return 1 # getting the acting set for the created pool acting_pg_set = rados_obj.get_pg_acting_set(pool_name=pool_name) log.info( f"Killing 2 OSD's from acting set : {acting_pg_set} to verify recovery" ) stop_osds = [acting_pg_set.pop() for _ in range(2)] for osd_id in stop_osds: if not rados_obj.change_osd_state(action="stop", target=osd_id): log.error(f"Unable to stop the OSD : {osd_id}") return 1 # Sleeping for 25 seconds ( "osd_heartbeat_grace": "20" ) for osd's to be marked down time.sleep(25) log.info("Stopped 2 OSD's from acting set, starting to wait for recovery") rados_obj.change_recover_threads(config=config, action="set") if not rados_obj.bench_write(pool_name=pool_name, **config): log.error("Failed to write objects into the Pool") return 1 log.debug("Triggering forced recovery in stretch mode") cmd = "ceph osd force_recovery_stretch_mode --yes-i-really-mean-it" rados_obj.run_ceph_command(cmd) log.info("Triggered the recovery in stretch mode") log.debug("Starting the stopped OSD's") for osd_id in stop_osds: if not rados_obj.change_osd_state(action="restart", target=osd_id): log.error(f"Unable to restart the OSD : {osd_id}") return 1 # there was data written into pool when the OSD's were down. # Verifying if data is recovered and placed into the OSD's after bringing them back res = wait_for_clean_pg_sets(rados_obj) if not res: log.error("PG's in cluster are not active + Clean ") return 1 log.debug("Forcing the stretch cluster into healthy mode") cmd = "ceph osd force_healthy_stretch_mode --yes-i-really-mean-it" rados_obj.run_ceph_command(cmd) log.info("Cluster has successfully recovered and is in healthy state") return 0 # Finding and Deleting any stray EC pools that might have been left on cluster pool_dump = rados_obj.run_ceph_command(cmd="ceph osd dump") for entry in pool_dump["pools"]: if entry["type"] != 1 and entry["crush_rule"] != 0: log.info( f"A non-replicated pool found : {entry['pool_name']}, proceeding to delete pool" ) if not rados_obj.detete_pool(pool=entry["pool_name"]): log.error(f"the pool {entry['pool_name']} could not be deleted") return 1 log.debug("No pools other than replicated found on cluster") # disabling automatic crush update cmd = "ceph config set osd osd_crush_update_on_start false" cephadm.shell([cmd]) site1 = config.get("site1", "site1") site2 = config.get("site2", "site2") # Collecting osd details and split them into Sita A and Site B sorted_osds = sort_osd_sites(all_osd_details=osd_details) site_a_osds = sorted_osds[0] site_b_osds = sorted_osds[1] if config.get("perform_add_capacity"): site_a_osds = sorted_osds[0][: (len(sorted_osds[0]) // 2)] site_b_osds = sorted_osds[1][: (len(sorted_osds[1]) // 2)] if not set_osd_sites( node=cephadm, osds=site_a_osds, site=site1, all_osd_details=osd_details, ): log.error("Failed to move the OSD's into sites") return 1 if not set_osd_sites( node=cephadm, osds=site_b_osds, site=site2, all_osd_details=osd_details, ): log.error("Failed to move the OSD's into sites") return 1 # collecting mon map to be compared after strtech cluster deployment stretch_rule_name = "stretch_rule" if not setup_crush_rule( node=client_node, rule_name=stretch_rule_name, site1=site1, site2=site2 ): log.error("Failed to Add crush rules in the crush map") return 1 # Setting the election strategy to connectivity mode if not mon_obj.set_election_strategy(mode="connectivity"): log.error("could not set election strategy to connectivity mode") return 1 # Sleeping for 5 sec for the strategy to be active time.sleep(5) init_mon_state = get_mon_details(node=cephadm) # Checking if mon elections happened after changing election strategy if mon_state["epoch"] > init_mon_state["epoch"]: log.error("Election epoch not bumped up after setting the connectivity mode.") return 1 # Checking updated election strategy in mon map strategy = mon_obj.get_election_strategy() if strategy != 3: log.error( f"cluster created election strategy other than connectivity, i.e {strategy}" ) return 1 log.info("Enabled connectivity mode on the cluster") log.info(f"selecting mon : {tiebreaker_node} as tie breaker monitor on site 3") if not set_mon_sites( node=cephadm, tiebreaker_node=tiebreaker_node, site1=site1, site2=site2 ): log.error("Failed to ad monitors into respective sites") return 1 # All the existing pools should be automatically changed with stretch rule. Creating a test pool pool_name = "test_pool_1" if not rados_obj.create_pool(pool_name=pool_name, pg_num=16): log.error("Failed to create the replicated Pool") return 1 log.info("Monitors added to respective sites. enabling stretch rule") cmd = f"/bin/ceph mon enable_stretch_mode {tiebreaker_node.hostname} {stretch_rule_name} datacenter" try: cephadm.shell([cmd]) except Exception as err: log.error( f"Error while enabling stretch rule on the datacenter. Command : {cmd}" ) log.error(err) return 1 if get_mon_details(node=cephadm)["epoch"] < init_mon_state["epoch"]: log.error("Election epoch not bumped up after Enabling strech mode") return 1 # Increasing backfill/rebalance threads so that cluster will re-balance it faster rados_obj.change_recover_threads(config=config, action="set") # wait for active + clean after deployment of stretch mode # checking the state after deployment coz of BZ : https://bugzilla.redhat.com/show_bug.cgi?id=2025800 res = wait_for_clean_pg_sets(rados_obj) if not res: status_report = rados_obj.run_ceph_command(cmd="ceph report") # Proceeding to check if all PG's are in active + clean for entry in status_report["num_pg_by_state"]: rec = ("remapped", "peering") if any(key in rec for key in entry["state"].split("+")): log.error( "PG's in cluster are stuck in remapped+peering after stretch deployment." ) return 1 if config.get("perform_add_capacity"): pool_name = "test_stretch_pool" if not rados_obj.create_pool( pool_name=pool_name, crush_rule=stretch_rule_name, ): log.error("Failed to create the replicated Pool") return 1 do_rados_put(mon=client_node, pool=pool_name, nobj=100) log.info("Performing add Capacity after the deployment of stretch cluster") site_a_osds = [osd for osd in sorted_osds[0] if osd not in site_a_osds] site_b_osds = [osd for osd in sorted_osds[1] if osd not in site_b_osds] if not set_osd_sites( node=cephadm, osds=site_a_osds, site=site1, all_osd_details=osd_details, ): log.error("Failed to move the OSD's into sites") return 1 if not set_osd_sites( node=cephadm, osds=site_b_osds, site=site2, all_osd_details=osd_details, ): log.error("Failed to move the OSD's into sites") return 1 flag = wait_for_clean_pg_sets(rados_obj) if not flag: log.error( "The cluster did not reach active + Clean state after add capacity" ) return 1 with parallel() as p: p.spawn(do_rados_get, client_node, pool_name, 10) for res in p: log.info(res) log.info("Successfully completed Add Capacity scenario") rados_obj.change_recover_threads(config=config, action="rm") # Checking if the pools have been updated with the new crush rules acting_set = rados_obj.get_pg_acting_set(pool_name=pool_name) if len(acting_set) != 4: log.error( f"There are {len(acting_set)} OSD's in PG. OSDs: {acting_set}. Stretch cluster requires 4" ) return 1 log.info(f"Acting set : {acting_set} Consists of 4 OSD's per PG") log.info("Stretch rule with arbiter monitor node set up successfully") return 0
def run(ceph_cluster, **kw): """ Performs various pool related validation tests Returns: 1 -> Fail, 0 -> Pass """ log.info(run.__doc__) config = kw["config"] cephadm = CephAdmin(cluster=ceph_cluster, **config) rados_obj = RadosOrchestrator(node=cephadm) mon_obj = MonConfigMethods(rados_obj=rados_obj) if config.get("ec_pool_recovery_improvement"): ec_config = config.get("ec_pool_recovery_improvement") if not rados_obj.create_erasure_pool(name="recovery", **ec_config): log.error("Failed to create the EC Pool") return 1 if not rados_obj.bench_write(**ec_config): log.error("Failed to write objects into the EC Pool") return 1 rados_obj.bench_read(**ec_config) log.info("Created the EC Pool, Finished writing data into the pool") # getting the acting set for the created pool acting_pg_set = rados_obj.get_pg_acting_set( pool_name=ec_config["pool_name"]) if len(acting_pg_set) != ec_config["k"] + ec_config["m"]: log.error( f"acting set consists of only these : {acting_pg_set} OSD's, less than k+m" ) return 1 log.info( f" Acting set of the pool consists of OSD's : {acting_pg_set}") log.info( f"Killing m, i.e {ec_config['m']} OSD's from acting set to verify recovery" ) stop_osds = [acting_pg_set.pop() for _ in range(ec_config["m"])] for osd_id in stop_osds: if not rados_obj.change_osd_state(action="stop", target=osd_id): log.error(f"Unable to stop the OSD : {osd_id}") return 1 log.info( "Stopped 'm' number of OSD's from, starting to wait for recovery") rados_obj.change_recover_threads(config=ec_config, action="set") # Sleeping for 25 seconds ( "osd_heartbeat_grace": "20" ) for osd's to be marked down time.sleep(25) # Waiting for up to 2.5 hours for the recovery to complete and PG's to enter active + Clean state end_time = datetime.datetime.now() + datetime.timedelta(seconds=9000) while end_time > datetime.datetime.now(): flag = True status_report = rados_obj.run_ceph_command(cmd="ceph report") # Proceeding to check if all PG's are in active + clean for entry in status_report["num_pg_by_state"]: rec = ( "backfilling", "degraded", "incomplete", "recovering", "recovery_wait", "backfilling_wait", "peered", "undersized", ) if any(key in rec for key in entry["state"].split("+")): flag = False if flag: log.info( "The recovery and back-filling of the OSD is completed") break log.info( f"Waiting for active + clean. Active aletrs: {status_report['health']['checks'].keys()}," f"PG States : {status_report['num_pg_by_state']}" f" checking status again in 1 minute") time.sleep(60) # getting the acting set for the created pool after recovery acting_pg_set = rados_obj.get_pg_acting_set( pool_name=ec_config["pool_name"]) if len(acting_pg_set) != ec_config["k"] + ec_config["m"]: log.error( f"acting set consists of only these : {acting_pg_set} OSD's, less than k+m" ) return 1 log.info( f" Acting set of the pool consists of OSD's : {acting_pg_set}") # Changing recovery threads back to default rados_obj.change_recover_threads(config=ec_config, action="rm") log.debug("Starting the stopped OSD's") for osd_id in stop_osds: if not rados_obj.change_osd_state(action="restart", target=osd_id): log.error(f"Unable to restart the OSD : {osd_id}") return 1 # Sleep for 5 seconds for OSD's to join the cluster time.sleep(5) if not flag: log.error( "The pool did not reach active + Clean state after recovery") return 1 # Deleting the pool created if not rados_obj.detete_pool(pool=ec_config["pool_name"]): log.error( f"the pool {ec_config['pool_name']} could not be deleted") return 1 log.info("Successfully tested EC pool recovery with K osd's surviving") return 0 if config.get("Compression_tests"): """ Create a 2 replicated pools: 1. Pool_1 : enable any compression algorithm(def snappy) and compression mode(aggressive/force). 2. Pool_2 : set compression mode to none Writing the same amount of data on 2 pools, size of pool with compression on would consume less space """ pool_config = config["Compression_tests"]["pool_config"] compression_config = config["Compression_tests"]["compression_config"] pool_1 = pool_config["pool-1"] pool_2 = pool_config["pool-2"] if config["Compression_tests"]["pool_type"] == "replicated": if not rados_obj.create_pool(pool_name=pool_1, **pool_config): log.error("could not create pool-1") return 1 if not rados_obj.create_pool(pool_name=pool_2, **pool_config): log.error("could not create pool-2") return 1 elif config["Compression_tests"]["pool_type"] == "erasure": pool_config["pool_name"] = pool_1 if not rados_obj.create_erasure_pool(name=pool_1, **pool_config): log.error("could not create pool-1") return 1 pool_config["pool_name"] = pool_2 if not rados_obj.create_erasure_pool(name=pool_2, **pool_config): log.error("could not create pool-2") return 1 del pool_config["pool_name"] log.debug("Created two pools to test compression") # Enabling compression on pool-1 if not rados_obj.pool_inline_compression(pool_name=pool_1, **compression_config): log.error( f"Error setting compression on pool : {pool_1} for config {compression_config}" ) return 1 # Writing the same amount of data into two pools if not rados_obj.bench_write(pool_name=pool_1, **pool_config): log.error( "Failed to write objects into Pool-1, with compression enabled" ) return 1 if not rados_obj.bench_write(pool_name=pool_2, **pool_config): log.error( "Failed to write objects into Pool-2, without compression enabled" ) return 1 # Sleeping for 5 seconds for status to be updated. time.sleep(5) log.debug( "Finished writing data into the two pools. Checking pool stats") try: pool_stats = rados_obj.run_ceph_command( cmd="ceph df detail")["pools"] pool_1_stats = [ detail for detail in pool_stats if detail["name"] == pool_1 ][0]["stats"] pool_2_stats = [ detail for detail in pool_stats if detail["name"] == pool_2 ][0]["stats"] except KeyError: log.error( "No stats about the pools requested found on the cluster") return 1 log.debug(f"Pool-1 stats: {pool_1_stats}") log.debug(f"Pool-2 stats: {pool_2_stats}") if pool_1_stats["compress_bytes_used"] < 0: log.error("No data stored under pool-1 is compressed") return 1 if pool_1_stats["kb_used"] >= pool_2_stats["kb_used"]: log.error("Compression has no effect on the pool size...") return 1 if config["Compression_tests"].get("verify_compression_ratio_set"): # added verification for test: CEPH-83571672 if not rados_obj.check_compression_size(pool_name=pool_1, **compression_config): log.error("data not compressed in accordance to ratio set") return 1 log.info("Pool size is less when compression is enabled") return 0 if config.get("check_autoscaler_profile"): """ Verifies that the default auto-scaler profile on 5.1 builds in scale-up Verifies bugs : 1. https://bugzilla.redhat.com/show_bug.cgi?id=2021738 """ build = config.get("build", config.get("rhbuild")) autoscale_conf = config.get("check_autoscaler_profile") regex = r"5.1-rhel-\d{1}" if re.search(regex, build): log.info( "Test running on 5.1 builds, checking the default autoscaler profile" ) if not mon_obj.verify_set_config(**autoscale_conf): log.error( f"The default value for autoscaler profile is not scale-up in buld {build}" ) return 1 log.info(f"Autoscale profile is scale-up in release : {build}") else: log.debug( f"The profile is already scale-up by default in release : {build}" ) return 0
def run(ceph_cluster, **kw): """ enables connectivity mode and deploys stretch cluster with arbiter mon node Actions Performed: 1. Disables the automatic crush map update 2. Collects the OSD daemons in the cluster and split them into 2 sites. 3. If add capacity is selected, only half of the OSD's will be added to various sites initially. 4. Adds the stretch rule into crush map. 5. Adding monitors into the 2 sites. 6. Create a replicated pool and deploy stretch mode. 7. Create a test pool, write some data and perform add capacity. ( add osd nodes into two sites ) 8. Check for the bump in election epochs throughout. 9. Check the acting set in PG for 4 OSD's. 2 from each site. Verifies bugs: [1]. https://bugzilla.redhat.com/show_bug.cgi?id=1937088 [2]. https://bugzilla.redhat.com/show_bug.cgi?id=1952763 Args: ceph_cluster (ceph.ceph.Ceph): ceph cluster """ log.info("Deploying stretch cluster with arbiter mon node") log.info(run.__doc__) config = kw.get("config") cephadm = CephAdmin(cluster=ceph_cluster, **config) rados_obj = RadosOrchestrator(node=cephadm) mon_obj = MonElectionStrategies(rados_obj=rados_obj) client_node = ceph_cluster.get_nodes(role="client")[0] tiebreaker_node = ceph_cluster.get_nodes(role="installer")[0] if not client_node and not tiebreaker_node: log.error( "Admin client and tie breaker node not configured, Cannot modify crush rules for stretch cluster" ) return 1 mon_state = get_mon_details(node=cephadm) if len(list(mon_state["monitors"])) < 5: log.error( f"Minimum of 5 Mon daemons needed to deploy a stretch cluster, found : {len(mon_state['monitors'])}" ) return 1 osd_details = get_osd_details(node=cephadm) if len(osd_details.keys()) < 4: log.error( f"Minimum of 4 osd daemons needed to deploy a stretch cluster, found : {len(osd_details.keys())}" ) return 1 # Finding and Deleting any stray EC pools that might have been left on cluster pool_dump = rados_obj.run_ceph_command(cmd="ceph osd dump") for entry in pool_dump["pools"]: if entry["type"] != 1 and entry["crush_rule"] != 0: log.info( f"A non-replicated pool found : {entry['pool_name']}, proceeding to delete pool" ) if not rados_obj.detete_pool(pool=entry["pool_name"]): log.error( f"the pool {entry['pool_name']} could not be deleted") return 1 log.debug("No pools other than replicated found on cluster") # disabling automatic crush update cmd = "ceph config set osd osd_crush_update_on_start false" cephadm.shell([cmd]) site1 = config.get("site1", "site1") site2 = config.get("site2", "site2") # Collecting osd details and split them into Sita A and Site B sorted_osds = sort_osd_sites(all_osd_details=osd_details) site_a_osds = sorted_osds[0] site_b_osds = sorted_osds[1] if config.get("perform_add_capacity"): site_a_osds = sorted_osds[0][:(len(sorted_osds[0]) // 2)] site_b_osds = sorted_osds[1][:(len(sorted_osds[1]) // 2)] if not set_osd_sites( node=cephadm, osds=site_a_osds, site=site1, all_osd_details=osd_details, ): log.error("Failed to move the OSD's into sites") return 1 if not set_osd_sites( node=cephadm, osds=site_b_osds, site=site2, all_osd_details=osd_details, ): log.error("Failed to move the OSD's into sites") return 1 # collecting mon map to be compared after strtech cluster deployment stretch_rule_name = "stretch_rule" if not setup_crush_rule(node=client_node, rule_name=stretch_rule_name, site1=site1, site2=site2): log.error("Failed to Add crush rules in the crush map") return 1 # Setting the election strategy to connectivity mode if not mon_obj.set_election_strategy(mode="connectivity"): log.error("could not set election strategy to connectivity mode") return 1 # Sleeping for 5 sec for the strategy to be active time.sleep(5) init_mon_state = get_mon_details(node=cephadm) # Checking if mon elections happened after changing election strategy if mon_state["epoch"] > init_mon_state["epoch"]: log.error( "Election epoch not bumped up after setting the connectivity mode." ) return 1 # Checking updated election strategy in mon map strategy = mon_obj.get_election_strategy() if strategy != 3: log.error( f"cluster created election strategy other than connectivity, i.e {strategy}" ) return 1 log.info("Enabled connectivity mode on the cluster") log.info( f"selecting mon : {tiebreaker_node} as tie breaker monitor on site 3") if not set_mon_sites(node=cephadm, tiebreaker_node=tiebreaker_node, site1=site1, site2=site2): log.error("Failed to ad monitors into respective sites") return 1 # All the existing pools should be automatically changed with stretch rule. Creating a test pool pool_name = "test_pool_1" if not rados_obj.create_pool(pool_name=pool_name, pg_num=16): log.error("Failed to create the replicated Pool") return 1 log.info("Monitors added to respective sites. enabling stretch rule") cmd = f"/bin/ceph mon enable_stretch_mode {tiebreaker_node.hostname} {stretch_rule_name} datacenter" try: cephadm.shell([cmd]) except Exception as err: log.error( f"Error while enabling stretch rule on the datacenter. Command : {cmd}" ) log.error(err) return 1 if get_mon_details(node=cephadm)["epoch"] < init_mon_state["epoch"]: log.error("Election epoch not bumped up after Enabling strech mode") return 1 if config.get("perform_add_capacity"): pool_name = "test_stretch_pool" if not rados_obj.create_pool( pool_name=pool_name, crush_rule=stretch_rule_name, ): log.error("Failed to create the replicated Pool") return 1 do_rados_put(mon=client_node, pool=pool_name, nobj=1000) # Increasing backfill/rebalance threads so that cluster will re-balance it faster after add capacity rados_obj.change_recover_threads(config=config, action="set") log.info( "Performing add Capacity after the deployment of stretch cluster") site_a_osds = [osd for osd in sorted_osds[0] if osd not in site_a_osds] site_b_osds = [osd for osd in sorted_osds[1] if osd not in site_b_osds] if not set_osd_sites( node=cephadm, osds=site_a_osds, site=site1, all_osd_details=osd_details, ): log.error("Failed to move the OSD's into sites") return 1 if not set_osd_sites( node=cephadm, osds=site_b_osds, site=site2, all_osd_details=osd_details, ): log.error("Failed to move the OSD's into sites") return 1 # Waiting for up to 2.5 hours for the PG's to enter active + Clean state after add capacity # Automation for bug : [1] & [2] end_time = datetime.datetime.now() + datetime.timedelta(seconds=9000) while end_time > datetime.datetime.now(): flag = True status_report = rados_obj.run_ceph_command(cmd="ceph report") # Proceeding to check if all PG's are in active + clean for entry in status_report["num_pg_by_state"]: rec = ( "remapped", "backfilling", "degraded", "incomplete", "peering", "recovering", "recovery_wait", "peering", "undersized", "backfilling_wait", ) if any(key in rec for key in entry["state"].split("+")): flag = False if flag: log.info( "The recovery and back-filling of the OSD is completed") break log.info( f"Waiting for active + clean. Active aletrs: {status_report['health']['checks'].keys()}," f"PG States : {status_report['num_pg_by_state']}" f" checking status again in 2 minutes") time.sleep(120) rados_obj.change_recover_threads(config=config, action="rm") if not flag: log.error( "The cluster did not reach active + Clean state after add capacity" ) return 1 with parallel() as p: p.spawn(do_rados_get, client_node, pool_name, 10) for res in p: log.info(res) # Checking if the pools have been updated with the new crush rules acting_set = rados_obj.get_pg_acting_set(pool_name=pool_name) if len(acting_set) != 4: log.error( f"There are {len(acting_set)} OSD's in PG. OSDs: {acting_set}. Stretch cluster requires 4" ) return 1 log.info(f"Acting set : {acting_set} Consists of 4 OSD's per PG") log.info("Stretch rule with arbiter monitor node set up successfully") return 0