def run(ceph_cluster, **kw): """ Prepares the cluster to run rados tests. Actions Performed: 1. Create a Replicated and Erasure coded pools and write Objects into pools 2. Setup email alerts for sending errors/warnings on the cluster. Verifies Bugs: https://bugzilla.redhat.com/show_bug.cgi?id=1849894 https://bugzilla.redhat.com/show_bug.cgi?id=1878145 3. Enable logging into file and check file permissions Verifies Bug : https://bugzilla.redhat.com/show_bug.cgi?id=1884469 Args: ceph_cluster (ceph.ceph.Ceph): ceph cluster kw: Args that need to be passed to the test for initialization Returns: 1 -> Fail, 0 -> Pass """ log.info(run.__doc__) config = kw["config"] cephadm = CephAdmin(cluster=ceph_cluster, **config) rados_obj = RadosOrchestrator(node=cephadm) mon_obj = MonConfigMethods(rados_obj=rados_obj) out, err = cephadm.shell(["uuidgen"]) uuid = out.split("-")[0] if config.get("ec_pool"): ec_config = config.get("ec_pool") ec_config.setdefault("pool_name", f"ecpool_{uuid}") if not rados_obj.create_erasure_pool(name=uuid, **ec_config): log.error("Failed to create the EC Pool") return 1 if ec_config.get("test_overwrites_pool"): if not rados_obj.verify_ec_overwrites(**ec_config): log.error("Failed to create the EC Pool") return 1 else: if not rados_obj.bench_write(**ec_config): log.error("Failed to write objects into the EC Pool") return 1 rados_obj.bench_read(**ec_config) log.info( "Created the EC Pool, Finished writing data into the pool") if ec_config.get("delete_pool"): if not rados_obj.detete_pool(pool=ec_config["pool_name"]): log.error("Failed to delete EC Pool") return 1 if config.get("replicated_pool"): rep_config = config.get("replicated_pool") rep_config.setdefault("pool_name", f"repool_{uuid}") if not rados_obj.create_pool(**rep_config, ): log.error("Failed to create the replicated Pool") return 1 if not rados_obj.bench_write(**rep_config): log.error("Failed to write objects into the EC Pool") return 1 rados_obj.bench_read(**rep_config) log.info( "Created the replicated Pool, Finished writing data into the pool") if rep_config.get("delete_pool"): if not rados_obj.detete_pool(pool=rep_config["pool_name"]): log.error("Failed to delete replicated Pool") return 1 if config.get("set_pool_configs"): changes = config["set_pool_configs"] pool_name = changes["pool_name"] configurations = changes["configurations"] for conf in configurations.keys(): if not rados_obj.set_pool_property( pool=pool_name, props=conf, value=configurations[conf]): log.error(f"failed to set property {conf} on the cluster") return 1 log.info(f"made the config changes on the pool {pool_name}") if config.get("email_alerts"): alert_config = config.get("email_alerts") if not rados_obj.enable_email_alerts(**alert_config): log.error("Error while configuring email alerts") return 1 log.info("email alerts configured") if config.get("Verify_config_parameters"): test_config = config.get("Verify_config_parameters") test_node = ceph_cluster.get_nodes(role="osd")[0] for conf in test_config["configurations"]: for entry in conf.values(): if entry.get("location_type") == "host": entry["location_value"] = test_node.hostname if not mon_obj.set_config(**entry): log.error(f"Error setting config {conf}") return 1 log.info("done") pool_name = "test_pool_1" if not rados_obj.create_pool(pool_name=pool_name, pg_num=16): log.error("Failed to create the replicated Pool") return 1 rados_obj.bench_write(pool_name=pool_name, rados_write_duration=50) # Removing test configurations for conf in test_config["configurations"]: for entry in conf.values(): if entry.get("location_type") == "host": entry["location_value"] = test_node.hostname if not mon_obj.remove_config(**entry): log.error(f"Error setting config {conf}") return 1 log.info("finished removing values, passed") if config.get("log_to_file"): if not rados_obj.enable_file_logging(): log.error("Error while setting config to enable logging into file") return 1 log.info("Logging to file configured") if config.get("cluster_configuration_checks"): cls_config = config.get("cluster_configuration_checks") if not rados_obj.set_cluster_configuration_checks(**cls_config): log.error("Error while setting Cluster config checks") return 1 log.info("Set up cluster configuration checks") if config.get("configure_balancer"): balancer_config = config.get("configure_balancer") if not rados_obj.enable_balancer(**balancer_config): log.error("Error while setting up balancer on the Cluster") return 1 log.info("Set up Balancer on the cluster") if config.get("configure_pg_autoscaler"): autoscaler_config = config.get("configure_pg_autoscaler") if not rados_obj.configure_pg_autoscaler(**autoscaler_config): log.error("Error while setting up pg_autoscaler on the Cluster") return 1 log.info("Set up pg_autoscaler on the cluster") if config.get("enable_compression"): compression_conf = config["enable_compression"] pool_name = compression_conf["pool_name"] for conf in compression_conf["configurations"]: for entry in conf.values(): if not rados_obj.pool_inline_compression(pool_name=pool_name, **entry): log.error( f"Error setting compression on pool : {pool_name} for config {conf}" ) return 1 if not rados_obj.bench_write(**compression_conf): log.error("Failed to write objects into Pool") return 1 rados_obj.bench_read(**compression_conf) log.info( "Created the replicated Pool, Finished writing data into the pool" ) log.info("Completed compression tests") if config.get("delete_pools"): for name in config["delete_pools"]: if not rados_obj.detete_pool(name): log.error(f"the pool {name} could not be deleted") return 1 log.info("deleted all the given pools successfully") log.info("All Pre-requisites completed to run Rados suite") return 0
class PoolFunctions: """ Contains various functions that help in altering the behaviour, working of pools and verify the changes """ def __init__(self, node: CephAdmin): """ initializes the env to run rados commands Args: node: CephAdmin object """ self.rados_obj = RadosOrchestrator(node=node) def verify_target_ratio_set(self, pool_name, ratio): """ Sets the "target_size_ratio" on the given pool and verifies it from the auto-scale status Args: pool_name: name of the pool ratio: ratio to be set Returns: True -> pass, False -> fail """ log.debug(f"Setting ratio: {ratio} on pool: {pool_name}") self.rados_obj.set_pool_property(pool=pool_name, props="target_size_ratio", value=ratio) # sleeping for 2 seconds for pg autoscaler updates the status and new PG's time.sleep(2) ratio_set = self.get_pg_autoscaler_value(pool_name, item="target_ratio") if not ratio_set == ratio: log.error("specified target ratio not set on the pool") return False return True def get_pg_autoscaler_value(self, pool_name, item): """ Fetches the target ratio set on the pool given Args: pool_name: name of the pool item: Value of the item to be fetched. Allowed values: actual_capacity_ratio|actual_raw_used|bias|capacity_ratio|crush_root_id|target_bytes| effective_target_ratio|logical_used|pg_autoscale_mode|pg_num_target|pool_id|raw_used|target_ratio| Returns: Requested value """ cmd = "ceph osd pool autoscale-status" autoscale_status = self.rados_obj.run_ceph_command(cmd=cmd) try: pool_details = [ details for details in autoscale_status if details["pool_name"] == pool_name ][0] except Exception: log.error("Pool not found") return pool_details[item] def fill_omap_entries(self, pool_name, **kwargs): """ creates key-value entries for objects on ceph pools and increase the omap entries on the pool eg : if obj_start, obj_end: 0, 3000 objects, with num_keys 1000, the method would create 3000 objects with 1k KW pairs each. so total 3000*1000 KW entries Args: pool_name: name of the pool where the KW pairs needed to be added to objects **kwargs: other args that can be passed Valid args: 1. obj_start: start count for object creation 2. obj_end : end count for object creation 3. num_keys_obj: Number of KW paris to be added to each object Returns: True -> pass, False -> fail """ # Getting the client node to perform the operations client_node = self.rados_obj.ceph_cluster.get_nodes(role="client")[0] obj_start = kwargs.get("obj_start", 0) obj_end = kwargs.get("obj_end", 2000) num_keys_obj = kwargs.get("num_keys_obj", 20000) log.debug(f"Writing {(obj_end - obj_start) * num_keys_obj} Key paris" f" to increase the omap entries on pool {pool_name}") script_loc = "https://raw.githubusercontent.com/red-hat-storage/cephci/master/utility/generate_omap_entries.py" client_node.exec_command( sudo=True, cmd=f"curl -k {script_loc} -O", ) # Setup Script pre-requisites : docopt client_node.exec_command(sudo=True, cmd="pip3 install docopt", long_running=True) cmd_options = f"--pool {pool_name} --start {obj_start} --end {obj_end} --key-count {num_keys_obj}" cmd = f"python3 generate_omap_entries.py {cmd_options}" client_node.exec_command(sudo=True, cmd=cmd, long_running=True) # removing the py file copied client_node.exec_command(sudo=True, cmd="rm -rf generate_omap_entries.py") log.debug("Checking the amount of omap entries created on the pool") pool_stats = self.rados_obj.run_ceph_command( cmd="ceph df detail")["pools"] for detail in pool_stats: if detail["name"] == pool_name: pool_1_stats = detail["stats"] total_omap_data = pool_1_stats["omap_bytes_used"] omap_data = pool_1_stats["stored_omap"] break if omap_data < 0: log.error("No omap entries written into pool") return False log.info(f"Wrote {omap_data} bytes of omap data on the pool." f"Total stored omap data on pool : {total_omap_data}") return True def do_rados_delete(self, pool_name: str, pg_id: str = None): """ deletes all the objects from the given pool / PG ID Args: 1. pool_name: name of the pool 2. [ pg_id ]: Pg ID (Optional, but when provided, should be passed along with pool name ) Returns: True -> pass, False -> fail """ obj_cmd = f"rados -p {pool_name} ls" if pg_id: obj_cmd = f"rados --pgid {pg_id} ls" delete_obj_list = self.rados_obj.run_ceph_command(cmd=obj_cmd, timeout=1000) for obj in delete_obj_list: cmd = f"rados -p {pool_name} rm {obj['name']}" self.rados_obj.node.shell([cmd], long_running=True) # Sleeping for 3 seconds for object reference to be deleted time.sleep(3) # Checking if object is still present in the pool out = self.rados_obj.run_ceph_command(cmd=obj_cmd, timeout=1000) rem_objs = [obj["name"] for obj in out] if obj["name"] in rem_objs: log.error(f"Object {obj['name']} not deleted in the pool") return False log.debug(f"deleted object: {obj['name']} from pool {pool_name}") log.info(f"Completed deleting all objects from pool {pool_name}") return True def create_pool_snap(self, pool_name: str): """ Creates snapshots of the given pool Args: pool_name: name of the pool Returns: Pass -> name of the snapshot created, Fail -> False """ # Checking if snapshots can be created on the supplied pool cmd = "ceph osd dump" pool_status = self.rados_obj.run_ceph_command(cmd=cmd, timeout=800) for detail in pool_status["pools"]: if detail["pool_name"] != pool_name: continue if "selfmanaged_snaps" in detail["flags_names"]: # bz: https://bugzilla.redhat.com/show_bug.cgi?id=1425803#c2 log.error( f"Pool {pool_name} is a self managed pool, cannot create snaps manually" ) return False # Creating snaps on the pool provided cmd = "uuidgen" out, err = self.rados_obj.node.shell([cmd]) uuid = out[0:5] snap_name = f"{pool_name}-snap-{uuid}" cmd = f"ceph osd pool mksnap {pool_name} {snap_name}" self.rados_obj.node.shell([cmd], long_running=True) # Checking if snap was created successfully if not self.check_snap_exists(snap_name=snap_name, pool_name=pool_name): log.error("Snapshot of pool not created") return False log.debug(f"Created snapshot {snap_name} on pool {pool_name}") return snap_name def check_snap_exists(self, snap_name: str, pool_name: str) -> bool: """ checks the existence of the snapshot name given on the pool Args: snap_name: Name of the snapshot pool_name: Name of the pool Returns: True -> Snapshot exists, False -> snapshot does not exist """ snap_list = self.get_snap_names(pool_name=pool_name) return True if snap_name in snap_list else False def get_snap_names(self, pool_name: str) -> list: """ Fetches the list of snapshots created on the given pool Args: pool_name: name of the pool Returns: list of the snaps created """ cmd = "ceph osd dump" pool_status = self.rados_obj.run_ceph_command(cmd=cmd, timeout=800) for detail in pool_status["pools"]: if detail["pool_name"] == pool_name: snap_list = [snap["name"] for snap in detail["pool_snaps"]] log.debug(f"snapshots on pool : {snap_list}") return snap_list def delete_pool_snap(self, pool_name: str, snap_name: str = None) -> bool: """ deletes snapshots of the given pool. If no snap name is provided, deletes all the snapshots on the pool Args: pool_name: name of the pool snap_name: name of the snapshot Returns: Pass -> snapshot Deleted, Fail -> snapshot not Deleted """ if snap_name: delete_list = list(snap_name) else: delete_list = self.get_snap_names(pool_name=pool_name) # Deleting snaps on the pool provided for snap in delete_list: cmd = f"ceph osd pool rmsnap {pool_name} {snap}" self.rados_obj.node.shell([cmd]) # Checking if snap was deleted successfully if self.check_snap_exists(snap_name=snap_name, pool_name=pool_name): log.error("Snapshot of pool exists") return False log.debug(f"deleted snapshot {snap} on pool {pool_name}") log.debug("Deleted provided snapshots on the pool") return True def get_bulk_details(self, pool_name: str) -> bool: """ Checks the status of bulk flag on the pool given Args: pool_name: Name of the pool Returns: True -> pass, False -> fail """ # Checking if the sent pool already exists. if pool_name not in self.rados_obj.list_pools(): log.error(f"Pool {pool_name} does not exist") return False # Getting the bulk status obj = self.rados_obj.get_pool_property(pool=pool_name, props="bulk") return obj["bulk"] def set_bulk_flag(self, pool_name: str) -> bool: """ Sets the bulk flag to true on existing pools Args: pool_name: Name of the pool Returns: True -> pass, False -> fail """ # Checking if the sent pool already exists. If does not, creating new pool if pool_name not in self.rados_obj.list_pools(): log.info( f"Pool {pool_name} does not exist, creating new pool with bulk enabled" ) if not self.rados_obj.create_pool(pool_name=pool_name, bulk=True): log.error("Failed to create the replicated Pool") return False # Enabling bulk on already existing pool if not self.rados_obj.set_pool_property( pool=pool_name, props="bulk", value="true"): log.error(f"Could not set the bulk flag on pool {pool_name}") return False # Sleeping for 2 seconds after pool create/Modify for PG's to be calculated with bulk time.sleep(2) # Checking if the bulk is enabled or not return self.get_bulk_details(pool_name=pool_name) def rm_bulk_flag(self, pool_name: str) -> bool: """ Removes the bulk flag on existing pools Args: pool_name: Name of the pool Returns: True -> pass, False -> fail """ # Checking if the sent pool already exists. if pool_name not in self.rados_obj.list_pools(): log.info(f"Pool {pool_name} does not exist") return False # Enabling bulk on already existing pool if not self.rados_obj.set_pool_property( pool=pool_name, props="bulk", value="false"): log.error(f"Could not unset the bulk flag on pool {pool_name}") return False # Sleeping for 2 seconds after pool create/Modify for PG's to be calculated with bulk time.sleep(2) # Checking if the bulk is enabled or not return not self.get_bulk_details(pool_name=pool_name) def get_target_pg_num_bulk_flag(self, pool_name: str) -> int: """ Fetches the target PG counts for the given pool from the autoscaler status Args: pool_name: Name of the pool Returns: PG Count """ # Checking the autoscaler status, final PG counts, bulk flags cmd = "ceph osd pool autoscale-status" pool_status = self.rados_obj.run_ceph_command(cmd=cmd) for entry in pool_status: if entry["pool_name"] == pool_name: return int(entry["pg_num_final"])
def run(ceph_cluster, **kw): """ Performs various pool related validation tests Returns: 1 -> Fail, 0 -> Pass """ log.info(run.__doc__) config = kw["config"] cephadm = CephAdmin(cluster=ceph_cluster, **config) rados_obj = RadosOrchestrator(node=cephadm) mon_obj = MonConfigMethods(rados_obj=rados_obj) pool_obj = PoolFunctions(node=cephadm) if config.get("ec_pool_recovery_improvement"): ec_config = config.get("ec_pool_recovery_improvement") if not rados_obj.create_erasure_pool(name="recovery", **ec_config): log.error("Failed to create the EC Pool") return 1 if not rados_obj.bench_write(**ec_config): log.error("Failed to write objects into the EC Pool") return 1 rados_obj.bench_read(**ec_config) log.info("Created the EC Pool, Finished writing data into the pool") # getting the acting set for the created pool acting_pg_set = rados_obj.get_pg_acting_set( pool_name=ec_config["pool_name"]) if len(acting_pg_set) != ec_config["k"] + ec_config["m"]: log.error( f"acting set consists of only these : {acting_pg_set} OSD's, less than k+m" ) return 1 log.info( f" Acting set of the pool consists of OSD's : {acting_pg_set}") log.info( f"Killing m, i.e {ec_config['m']} OSD's from acting set to verify recovery" ) stop_osds = [acting_pg_set.pop() for _ in range(ec_config["m"])] for osd_id in stop_osds: if not rados_obj.change_osd_state(action="stop", target=osd_id): log.error(f"Unable to stop the OSD : {osd_id}") return 1 log.info( "Stopped 'm' number of OSD's from, starting to wait for recovery") rados_obj.change_recover_threads(config=ec_config, action="set") # Sleeping for 25 seconds ( "osd_heartbeat_grace": "20" ) for osd's to be marked down time.sleep(25) # Waiting for up to 2.5 hours for the recovery to complete and PG's to enter active + Clean state end_time = datetime.datetime.now() + datetime.timedelta(seconds=9000) while end_time > datetime.datetime.now(): flag = True status_report = rados_obj.run_ceph_command(cmd="ceph report") # Proceeding to check if all PG's are in active + clean for entry in status_report["num_pg_by_state"]: rec = ( "backfilling", "degraded", "incomplete", "recovering", "recovery_wait", "backfilling_wait", "peered", "undersized", ) if any(key in rec for key in entry["state"].split("+")): flag = False if flag: log.info( "The recovery and back-filling of the OSD is completed") break log.info( f"Waiting for active + clean. Active aletrs: {status_report['health']['checks'].keys()}," f"PG States : {status_report['num_pg_by_state']}" f" checking status again in 1 minute") time.sleep(60) # getting the acting set for the created pool after recovery acting_pg_set = rados_obj.get_pg_acting_set( pool_name=ec_config["pool_name"]) if len(acting_pg_set) != ec_config["k"] + ec_config["m"]: log.error( f"acting set consists of only these : {acting_pg_set} OSD's, less than k+m" ) return 1 log.info( f" Acting set of the pool consists of OSD's : {acting_pg_set}") # Changing recovery threads back to default rados_obj.change_recover_threads(config=ec_config, action="rm") log.debug("Starting the stopped OSD's") for osd_id in stop_osds: if not rados_obj.change_osd_state(action="restart", target=osd_id): log.error(f"Unable to restart the OSD : {osd_id}") return 1 # Sleep for 5 seconds for OSD's to join the cluster time.sleep(5) if not flag: log.error( "The pool did not reach active + Clean state after recovery") return 1 # Deleting the pool created if not rados_obj.detete_pool(pool=ec_config["pool_name"]): log.error( f"the pool {ec_config['pool_name']} could not be deleted") return 1 log.info("Successfully tested EC pool recovery with K osd's surviving") return 0 if config.get("Compression_tests"): """ Create a 2 replicated pools: 1. Pool_1 : enable any compression algorithm(def snappy) and compression mode(aggressive/force). 2. Pool_2 : set compression mode to none Writing the same amount of data on 2 pools, size of pool with compression on would consume less space """ pool_config = config["Compression_tests"]["pool_config"] compression_config = config["Compression_tests"]["compression_config"] pool_1 = pool_config["pool-1"] pool_2 = pool_config["pool-2"] if config["Compression_tests"]["pool_type"] == "replicated": if not rados_obj.create_pool(pool_name=pool_1, **pool_config): log.error("could not create pool-1") return 1 if not rados_obj.create_pool(pool_name=pool_2, **pool_config): log.error("could not create pool-2") return 1 elif config["Compression_tests"]["pool_type"] == "erasure": pool_config["pool_name"] = pool_1 if not rados_obj.create_erasure_pool(name=pool_1, **pool_config): log.error("could not create pool-1") return 1 pool_config["pool_name"] = pool_2 if not rados_obj.create_erasure_pool(name=pool_2, **pool_config): log.error("could not create pool-2") return 1 del pool_config["pool_name"] log.debug("Created two pools to test compression") # Enabling compression on pool-1 if not rados_obj.pool_inline_compression(pool_name=pool_1, **compression_config): log.error( f"Error setting compression on pool : {pool_1} for config {compression_config}" ) return 1 # Writing the same amount of data into two pools if not rados_obj.bench_write(pool_name=pool_1, **pool_config): log.error( "Failed to write objects into Pool-1, with compression enabled" ) return 1 if not rados_obj.bench_write(pool_name=pool_2, **pool_config): log.error( "Failed to write objects into Pool-2, without compression enabled" ) return 1 # Sleeping for 5 seconds for status to be updated. time.sleep(5) log.debug( "Finished writing data into the two pools. Checking pool stats") try: pool_stats = rados_obj.run_ceph_command( cmd="ceph df detail")["pools"] pool_1_stats = [ detail for detail in pool_stats if detail["name"] == pool_1 ][0]["stats"] pool_2_stats = [ detail for detail in pool_stats if detail["name"] == pool_2 ][0]["stats"] except KeyError: log.error( "No stats about the pools requested found on the cluster") return 1 log.debug(f"Pool-1 stats: {pool_1_stats}") log.debug(f"Pool-2 stats: {pool_2_stats}") if pool_1_stats["compress_bytes_used"] < 0: log.error("No data stored under pool-1 is compressed") return 1 if pool_1_stats["kb_used"] >= pool_2_stats["kb_used"]: log.error("Compression has no effect on the pool size...") return 1 if config["Compression_tests"].get("verify_compression_ratio_set"): # added verification for test: CEPH-83571672 if not rados_obj.check_compression_size(pool_name=pool_1, **compression_config): log.error("data not compressed in accordance to ratio set") return 1 log.info("Pool size is less when compression is enabled") return 0 if config.get("test_autoscaler_bulk_feature"): """ Tests to verify the autoscaler bulk flag, which allows pools to make use of scale-down profile, making those pools start with full compliments of PG sets. Tests include 1. creating new pools with bulk, 2. enabling/disabling bulk flag on existing pools 3. Verify the PG changes when the flag is set/unset Verifies bugs : https://bugzilla.redhat.com/show_bug.cgi?id=2049851 """ regex = r"\s*(\d.\d)-rhel-\d" build = (re.search(regex, config.get("build", config.get("rhbuild")))).groups()[0] if not float(build) > 5.0: log.info( "Test running on version less than 5.1, skipping verifying bulk flags" ) return 0 # Creating a pool with bulk feature pool_name = config.get("pool_name") if not pool_obj.set_bulk_flag(pool_name=pool_name): log.error("Failed to create a pool with bulk features") return 1 # Checking the autoscaler status, final PG counts, bulk flags pg_target_init = pool_obj.get_target_pg_num_bulk_flag( pool_name=pool_name) # Unsetting the bulk flag and checking the change in the PG counts if not pool_obj.rm_bulk_flag(pool_name=pool_name): log.error("Failed to create a pool with bulk features") return 1 # Sleeping for 5 seconds for new PG num to bets et time.sleep(5) pg_target_interim = pool_obj.get_target_pg_num_bulk_flag( pool_name=pool_name) # The target PG's once the flag is disabled must be lesser than when enabled if pg_target_interim >= pg_target_init: log.error("PG's not reduced after bulk flag disabled") return 1 # Setting the bulk flag on pool again and checking the change in the PG counts if not pool_obj.set_bulk_flag(pool_name=pool_name): log.error("Failed to disable/remove bulk features on pool") return 1 # Sleeping for 5 seconds for new PG num to bets et time.sleep(5) pg_target_final = pool_obj.get_target_pg_num_bulk_flag( pool_name=pool_name) # The target PG's once the flag is disabled must be lesser than when enabled if pg_target_interim >= pg_target_final: log.error("PG's not Increased after bulk flag Enabled") return 1 if config.get("delete_pool"): rados_obj.detete_pool(pool=pool_name) log.info("Verified the workings of bulk flag") return 0 if config.get("verify_pool_target_ratio"): log.debug("Verifying target size ratio on pools") target_configs = config["verify_pool_target_ratio"]["configurations"] # Creating pools and starting the test for entry in target_configs.values(): log.debug(f"Creating {entry['pool_type']} pool on the cluster") if entry.get("pool_type", "replicated") == "erasure": method_should_succeed(rados_obj.create_erasure_pool, name=entry["pool_name"], **entry) else: method_should_succeed( rados_obj.create_pool, **entry, ) rados_obj.bench_write(**entry) if not pool_obj.verify_target_ratio_set( pool_name=entry["pool_name"], ratio=entry["target_size_ratio"]): log.error( f"Could not change the target ratio on the pool: {entry['pool_name']}" ) return 1 log.debug("Set the ratio. getting the projected pg's") rados_obj.change_recover_threads(config=config, action="set") log.debug( "Waiting for the rebalancing to complete on the cluster after the change" ) # Sleeping for 2 minutes for rebalancing to start & for new PG count to be updated. time.sleep(120) new_pg_count = int( pool_obj.get_pg_autoscaler_value(pool_name=entry["pool_name"], item="pg_num_target")) if new_pg_count <= entry["pg_num"]: log.error( f"Count of PG's not increased on the pool: {entry['pool_name']}" f"Initial creation count : {entry['pg_num']}" f"New count after setting num target : {new_pg_count}") return 1 res = wait_for_clean_pg_sets(rados_obj) if not res: log.error( "PG's in cluster are not active + Clean after the ratio change" ) return 1 if not pool_obj.verify_target_ratio_set( pool_name=entry["pool_name"], ratio=0.0): log.error( f"Could not remove the target ratio on the pool: {entry['pool_name']}" ) return 1 # Sleeping for 2 minutes for rebalancing to start & for new PG count to be updated. time.sleep(120) # Checking if after the removal of ratio, the PG count has reduced end_pg_count = int( pool_obj.get_pg_autoscaler_value(pool_name=entry["pool_name"], item="pg_num_target")) if end_pg_count >= new_pg_count: log.error( f"Count of PG's not changed/ reverted on the pool: {entry['pool_name']}" f" after removing the target ratios") return 1 rados_obj.change_recover_threads(config=config, action="rm") if entry.get("delete_pool", False): rados_obj.detete_pool(pool=entry["pool_name"]) log.info( f"Completed the test of target ratio on pool: {entry['pool_name']} " ) log.info("Target ratio tests completed") return 0 if config.get("verify_mon_target_pg_per_osd"): pg_conf = config.get("verify_mon_target_pg_per_osd") if not mon_obj.set_config(**pg_conf): log.error("Could not set the value for mon_target_pg_per_osd ") return 1 mon_obj.remove_config(**pg_conf) log.info("Set and verified the value for mon_target_pg_per_osd ") return 0 if config.get("verify_pg_num_min"): log.debug("Verifying pg_num_min on pools") target_configs = config["verify_pg_num_min"]["configurations"] # Creating pools and starting the test for entry in target_configs.values(): log.debug(f"Creating {entry['pool_type']} pool on the cluster") if entry.get("pool_type", "replicated") == "erasure": method_should_succeed(rados_obj.create_erasure_pool, name=entry["pool_name"], **entry) else: method_should_succeed( rados_obj.create_pool, **entry, ) rados_obj.bench_write(**entry) if not rados_obj.set_pool_property(pool=entry["pool_name"], props="pg_num_min", value=entry["pg_num_min"]): log.error("Could not set the pg_min_size on the pool") return 1 if entry.get("delete_pool", False): rados_obj.detete_pool(pool=entry["pool_name"]) log.info( f"Completed the test of pg_min_num on pool: {entry['pool_name']} " ) log.info("pg_min_num tests completed") return 0