def run(ceph_cluster, **kw):
"""
Prepares the cluster to run rados tests.
Actions Performed:
1. Create a Replicated and Erasure coded pools and write Objects into pools
2. Setup email alerts for sending errors/warnings on the cluster.
Verifies Bugs:
https://bugzilla.redhat.com/show_bug.cgi?id=1849894
https://bugzilla.redhat.com/show_bug.cgi?id=1878145
3. Enable logging into file and check file permissions
Verifies Bug : https://bugzilla.redhat.com/show_bug.cgi?id=1884469
Args:
ceph_cluster (ceph.ceph.Ceph): ceph cluster
kw: Args that need to be passed to the test for initialization
Returns:
1 -> Fail, 0 -> Pass
"""
log.info(run.__doc__)
config = kw["config"]
cephadm = CephAdmin(cluster=ceph_cluster, **config)
rados_obj = RadosOrchestrator(node=cephadm)
mon_obj = MonConfigMethods(rados_obj=rados_obj)
out, err = cephadm.shell(["uuidgen"])
uuid = out.split("-")[0]
if config.get("ec_pool"):
ec_config = config.get("ec_pool")
ec_config.setdefault("pool_name", f"ecpool_{uuid}")
if not rados_obj.create_erasure_pool(name=uuid, **ec_config):
log.error("Failed to create the EC Pool")
return 1
if ec_config.get("test_overwrites_pool"):
if not rados_obj.verify_ec_overwrites(**ec_config):
log.error("Failed to create the EC Pool")
return 1
else:
if not rados_obj.bench_write(**ec_config):
log.error("Failed to write objects into the EC Pool")
return 1
rados_obj.bench_read(**ec_config)
log.info(
"Created the EC Pool, Finished writing data into the pool")
if ec_config.get("delete_pool"):
if not rados_obj.detete_pool(pool=ec_config["pool_name"]):
log.error("Failed to delete EC Pool")
return 1
if config.get("replicated_pool"):
rep_config = config.get("replicated_pool")
rep_config.setdefault("pool_name", f"repool_{uuid}")
if not rados_obj.create_pool(**rep_config, ):
log.error("Failed to create the replicated Pool")
return 1
if not rados_obj.bench_write(**rep_config):
log.error("Failed to write objects into the EC Pool")
return 1
rados_obj.bench_read(**rep_config)
log.info(
"Created the replicated Pool, Finished writing data into the pool")
if rep_config.get("delete_pool"):
if not rados_obj.detete_pool(pool=rep_config["pool_name"]):
log.error("Failed to delete replicated Pool")
return 1
if config.get("set_pool_configs"):
changes = config["set_pool_configs"]
pool_name = changes["pool_name"]
configurations = changes["configurations"]
for conf in configurations.keys():
if not rados_obj.set_pool_property(
pool=pool_name, props=conf, value=configurations[conf]):
log.error(f"failed to set property {conf} on the cluster")
return 1
log.info(f"made the config changes on the pool {pool_name}")
if config.get("email_alerts"):
alert_config = config.get("email_alerts")
if not rados_obj.enable_email_alerts(**alert_config):
log.error("Error while configuring email alerts")
return 1
log.info("email alerts configured")
if config.get("Verify_config_parameters"):
test_config = config.get("Verify_config_parameters")
test_node = ceph_cluster.get_nodes(role="osd")[0]
for conf in test_config["configurations"]:
for entry in conf.values():
if entry.get("location_type") == "host":
entry["location_value"] = test_node.hostname
if not mon_obj.set_config(**entry):
log.error(f"Error setting config {conf}")
return 1
log.info("done")
pool_name = "test_pool_1"
if not rados_obj.create_pool(pool_name=pool_name, pg_num=16):
log.error("Failed to create the replicated Pool")
return 1
rados_obj.bench_write(pool_name=pool_name, rados_write_duration=50)
# Removing test configurations
for conf in test_config["configurations"]:
for entry in conf.values():
if entry.get("location_type") == "host":
entry["location_value"] = test_node.hostname
if not mon_obj.remove_config(**entry):
log.error(f"Error setting config {conf}")
return 1
log.info("finished removing values, passed")
if config.get("log_to_file"):
if not rados_obj.enable_file_logging():
log.error("Error while setting config to enable logging into file")
return 1
log.info("Logging to file configured")
if config.get("cluster_configuration_checks"):
cls_config = config.get("cluster_configuration_checks")
if not rados_obj.set_cluster_configuration_checks(**cls_config):
log.error("Error while setting Cluster config checks")
return 1
log.info("Set up cluster configuration checks")
if config.get("configure_balancer"):
balancer_config = config.get("configure_balancer")
if not rados_obj.enable_balancer(**balancer_config):
log.error("Error while setting up balancer on the Cluster")
return 1
log.info("Set up Balancer on the cluster")
if config.get("configure_pg_autoscaler"):
autoscaler_config = config.get("configure_pg_autoscaler")
if not rados_obj.configure_pg_autoscaler(**autoscaler_config):
log.error("Error while setting up pg_autoscaler on the Cluster")
return 1
log.info("Set up pg_autoscaler on the cluster")
if config.get("enable_compression"):
compression_conf = config["enable_compression"]
pool_name = compression_conf["pool_name"]
for conf in compression_conf["configurations"]:
for entry in conf.values():
if not rados_obj.pool_inline_compression(pool_name=pool_name,
**entry):
log.error(
f"Error setting compression on pool : {pool_name} for config {conf}"
)
return 1
if not rados_obj.bench_write(**compression_conf):
log.error("Failed to write objects into Pool")
return 1
rados_obj.bench_read(**compression_conf)
log.info(
"Created the replicated Pool, Finished writing data into the pool"
)
log.info("Completed compression tests")
if config.get("delete_pools"):
for name in config["delete_pools"]:
if not rados_obj.detete_pool(name):
log.error(f"the pool {name} could not be deleted")
return 1
log.info("deleted all the given pools successfully")
log.info("All Pre-requisites completed to run Rados suite")
return 0
class PoolFunctions:
"""
Contains various functions that help in altering the behaviour, working of pools and verify the changes
"""
def __init__(self, node: CephAdmin):
"""
initializes the env to run rados commands
Args:
node: CephAdmin object
"""
self.rados_obj = RadosOrchestrator(node=node)
def verify_target_ratio_set(self, pool_name, ratio):
"""
Sets the "target_size_ratio" on the given pool and verifies it from the auto-scale status
Args:
pool_name: name of the pool
ratio: ratio to be set
Returns: True -> pass, False -> fail
"""
log.debug(f"Setting ratio: {ratio} on pool: {pool_name}")
self.rados_obj.set_pool_property(pool=pool_name,
props="target_size_ratio",
value=ratio)
# sleeping for 2 seconds for pg autoscaler updates the status and new PG's
time.sleep(2)
ratio_set = self.get_pg_autoscaler_value(pool_name,
item="target_ratio")
if not ratio_set == ratio:
log.error("specified target ratio not set on the pool")
return False
return True
def get_pg_autoscaler_value(self, pool_name, item):
"""
Fetches the target ratio set on the pool given
Args:
pool_name: name of the pool
item: Value of the item to be fetched.
Allowed values: actual_capacity_ratio|actual_raw_used|bias|capacity_ratio|crush_root_id|target_bytes|
effective_target_ratio|logical_used|pg_autoscale_mode|pg_num_target|pool_id|raw_used|target_ratio|
Returns: Requested value
"""
cmd = "ceph osd pool autoscale-status"
autoscale_status = self.rados_obj.run_ceph_command(cmd=cmd)
try:
pool_details = [
details for details in autoscale_status
if details["pool_name"] == pool_name
][0]
except Exception:
log.error("Pool not found")
return pool_details[item]
def fill_omap_entries(self, pool_name, **kwargs):
"""
creates key-value entries for objects on ceph pools and increase the omap entries on the pool
eg : if obj_start, obj_end: 0, 3000 objects, with num_keys 1000, the method would create 3000 objects with 1k
KW pairs each. so total 3000*1000 KW entries
Args:
pool_name: name of the pool where the KW pairs needed to be added to objects
**kwargs: other args that can be passed
Valid args:
1. obj_start: start count for object creation
2. obj_end : end count for object creation
3. num_keys_obj: Number of KW paris to be added to each object
Returns: True -> pass, False -> fail
"""
# Getting the client node to perform the operations
client_node = self.rados_obj.ceph_cluster.get_nodes(role="client")[0]
obj_start = kwargs.get("obj_start", 0)
obj_end = kwargs.get("obj_end", 2000)
num_keys_obj = kwargs.get("num_keys_obj", 20000)
log.debug(f"Writing {(obj_end - obj_start) * num_keys_obj} Key paris"
f" to increase the omap entries on pool {pool_name}")
script_loc = "https://raw.githubusercontent.com/red-hat-storage/cephci/master/utility/generate_omap_entries.py"
client_node.exec_command(
sudo=True,
cmd=f"curl -k {script_loc} -O",
)
# Setup Script pre-requisites : docopt
client_node.exec_command(sudo=True,
cmd="pip3 install docopt",
long_running=True)
cmd_options = f"--pool {pool_name} --start {obj_start} --end {obj_end} --key-count {num_keys_obj}"
cmd = f"python3 generate_omap_entries.py {cmd_options}"
client_node.exec_command(sudo=True, cmd=cmd, long_running=True)
# removing the py file copied
client_node.exec_command(sudo=True,
cmd="rm -rf generate_omap_entries.py")
log.debug("Checking the amount of omap entries created on the pool")
pool_stats = self.rados_obj.run_ceph_command(
cmd="ceph df detail")["pools"]
for detail in pool_stats:
if detail["name"] == pool_name:
pool_1_stats = detail["stats"]
total_omap_data = pool_1_stats["omap_bytes_used"]
omap_data = pool_1_stats["stored_omap"]
break
if omap_data < 0:
log.error("No omap entries written into pool")
return False
log.info(f"Wrote {omap_data} bytes of omap data on the pool."
f"Total stored omap data on pool : {total_omap_data}")
return True
def do_rados_delete(self, pool_name: str, pg_id: str = None):
"""
deletes all the objects from the given pool / PG ID
Args:
1. pool_name: name of the pool
2. [ pg_id ]: Pg ID (Optional, but when provided, should be passed along with pool name )
Returns: True -> pass, False -> fail
"""
obj_cmd = f"rados -p {pool_name} ls"
if pg_id:
obj_cmd = f"rados --pgid {pg_id} ls"
delete_obj_list = self.rados_obj.run_ceph_command(cmd=obj_cmd,
timeout=1000)
for obj in delete_obj_list:
cmd = f"rados -p {pool_name} rm {obj['name']}"
self.rados_obj.node.shell([cmd], long_running=True)
# Sleeping for 3 seconds for object reference to be deleted
time.sleep(3)
# Checking if object is still present in the pool
out = self.rados_obj.run_ceph_command(cmd=obj_cmd, timeout=1000)
rem_objs = [obj["name"] for obj in out]
if obj["name"] in rem_objs:
log.error(f"Object {obj['name']} not deleted in the pool")
return False
log.debug(f"deleted object: {obj['name']} from pool {pool_name}")
log.info(f"Completed deleting all objects from pool {pool_name}")
return True
def create_pool_snap(self, pool_name: str):
"""
Creates snapshots of the given pool
Args:
pool_name: name of the pool
Returns: Pass -> name of the snapshot created, Fail -> False
"""
# Checking if snapshots can be created on the supplied pool
cmd = "ceph osd dump"
pool_status = self.rados_obj.run_ceph_command(cmd=cmd, timeout=800)
for detail in pool_status["pools"]:
if detail["pool_name"] != pool_name:
continue
if "selfmanaged_snaps" in detail["flags_names"]:
# bz: https://bugzilla.redhat.com/show_bug.cgi?id=1425803#c2
log.error(
f"Pool {pool_name} is a self managed pool, cannot create snaps manually"
)
return False
# Creating snaps on the pool provided
cmd = "uuidgen"
out, err = self.rados_obj.node.shell([cmd])
uuid = out[0:5]
snap_name = f"{pool_name}-snap-{uuid}"
cmd = f"ceph osd pool mksnap {pool_name} {snap_name}"
self.rados_obj.node.shell([cmd], long_running=True)
# Checking if snap was created successfully
if not self.check_snap_exists(snap_name=snap_name,
pool_name=pool_name):
log.error("Snapshot of pool not created")
return False
log.debug(f"Created snapshot {snap_name} on pool {pool_name}")
return snap_name
def check_snap_exists(self, snap_name: str, pool_name: str) -> bool:
"""
checks the existence of the snapshot name given on the pool
Args:
snap_name: Name of the snapshot
pool_name: Name of the pool
Returns: True -> Snapshot exists, False -> snapshot does not exist
"""
snap_list = self.get_snap_names(pool_name=pool_name)
return True if snap_name in snap_list else False
def get_snap_names(self, pool_name: str) -> list:
"""
Fetches the list of snapshots created on the given pool
Args:
pool_name: name of the pool
Returns: list of the snaps created
"""
cmd = "ceph osd dump"
pool_status = self.rados_obj.run_ceph_command(cmd=cmd, timeout=800)
for detail in pool_status["pools"]:
if detail["pool_name"] == pool_name:
snap_list = [snap["name"] for snap in detail["pool_snaps"]]
log.debug(f"snapshots on pool : {snap_list}")
return snap_list
def delete_pool_snap(self, pool_name: str, snap_name: str = None) -> bool:
"""
deletes snapshots of the given pool. If no snap name is provided, deletes all the snapshots on the pool
Args:
pool_name: name of the pool
snap_name: name of the snapshot
Returns: Pass -> snapshot Deleted, Fail -> snapshot not Deleted
"""
if snap_name:
delete_list = list(snap_name)
else:
delete_list = self.get_snap_names(pool_name=pool_name)
# Deleting snaps on the pool provided
for snap in delete_list:
cmd = f"ceph osd pool rmsnap {pool_name} {snap}"
self.rados_obj.node.shell([cmd])
# Checking if snap was deleted successfully
if self.check_snap_exists(snap_name=snap_name,
pool_name=pool_name):
log.error("Snapshot of pool exists")
return False
log.debug(f"deleted snapshot {snap} on pool {pool_name}")
log.debug("Deleted provided snapshots on the pool")
return True
def get_bulk_details(self, pool_name: str) -> bool:
"""
Checks the status of bulk flag on the pool given
Args:
pool_name: Name of the pool
Returns: True -> pass, False -> fail
"""
# Checking if the sent pool already exists.
if pool_name not in self.rados_obj.list_pools():
log.error(f"Pool {pool_name} does not exist")
return False
# Getting the bulk status
obj = self.rados_obj.get_pool_property(pool=pool_name, props="bulk")
return obj["bulk"]
def set_bulk_flag(self, pool_name: str) -> bool:
"""
Sets the bulk flag to true on existing pools
Args:
pool_name: Name of the pool
Returns: True -> pass, False -> fail
"""
# Checking if the sent pool already exists. If does not, creating new pool
if pool_name not in self.rados_obj.list_pools():
log.info(
f"Pool {pool_name} does not exist, creating new pool with bulk enabled"
)
if not self.rados_obj.create_pool(pool_name=pool_name, bulk=True):
log.error("Failed to create the replicated Pool")
return False
# Enabling bulk on already existing pool
if not self.rados_obj.set_pool_property(
pool=pool_name, props="bulk", value="true"):
log.error(f"Could not set the bulk flag on pool {pool_name}")
return False
# Sleeping for 2 seconds after pool create/Modify for PG's to be calculated with bulk
time.sleep(2)
# Checking if the bulk is enabled or not
return self.get_bulk_details(pool_name=pool_name)
def rm_bulk_flag(self, pool_name: str) -> bool:
"""
Removes the bulk flag on existing pools
Args:
pool_name: Name of the pool
Returns: True -> pass, False -> fail
"""
# Checking if the sent pool already exists.
if pool_name not in self.rados_obj.list_pools():
log.info(f"Pool {pool_name} does not exist")
return False
# Enabling bulk on already existing pool
if not self.rados_obj.set_pool_property(
pool=pool_name, props="bulk", value="false"):
log.error(f"Could not unset the bulk flag on pool {pool_name}")
return False
# Sleeping for 2 seconds after pool create/Modify for PG's to be calculated with bulk
time.sleep(2)
# Checking if the bulk is enabled or not
return not self.get_bulk_details(pool_name=pool_name)
def get_target_pg_num_bulk_flag(self, pool_name: str) -> int:
"""
Fetches the target PG counts for the given pool from the autoscaler status
Args:
pool_name: Name of the pool
Returns: PG Count
"""
# Checking the autoscaler status, final PG counts, bulk flags
cmd = "ceph osd pool autoscale-status"
pool_status = self.rados_obj.run_ceph_command(cmd=cmd)
for entry in pool_status:
if entry["pool_name"] == pool_name:
return int(entry["pg_num_final"])
def run(ceph_cluster, **kw):
"""
Performs various pool related validation tests
Returns:
1 -> Fail, 0 -> Pass
"""
log.info(run.__doc__)
config = kw["config"]
cephadm = CephAdmin(cluster=ceph_cluster, **config)
rados_obj = RadosOrchestrator(node=cephadm)
mon_obj = MonConfigMethods(rados_obj=rados_obj)
pool_obj = PoolFunctions(node=cephadm)
if config.get("ec_pool_recovery_improvement"):
ec_config = config.get("ec_pool_recovery_improvement")
if not rados_obj.create_erasure_pool(name="recovery", **ec_config):
log.error("Failed to create the EC Pool")
return 1
if not rados_obj.bench_write(**ec_config):
log.error("Failed to write objects into the EC Pool")
return 1
rados_obj.bench_read(**ec_config)
log.info("Created the EC Pool, Finished writing data into the pool")
# getting the acting set for the created pool
acting_pg_set = rados_obj.get_pg_acting_set(
pool_name=ec_config["pool_name"])
if len(acting_pg_set) != ec_config["k"] + ec_config["m"]:
log.error(
f"acting set consists of only these : {acting_pg_set} OSD's, less than k+m"
)
return 1
log.info(
f" Acting set of the pool consists of OSD's : {acting_pg_set}")
log.info(
f"Killing m, i.e {ec_config['m']} OSD's from acting set to verify recovery"
)
stop_osds = [acting_pg_set.pop() for _ in range(ec_config["m"])]
for osd_id in stop_osds:
if not rados_obj.change_osd_state(action="stop", target=osd_id):
log.error(f"Unable to stop the OSD : {osd_id}")
return 1
log.info(
"Stopped 'm' number of OSD's from, starting to wait for recovery")
rados_obj.change_recover_threads(config=ec_config, action="set")
# Sleeping for 25 seconds ( "osd_heartbeat_grace": "20" ) for osd's to be marked down
time.sleep(25)
# Waiting for up to 2.5 hours for the recovery to complete and PG's to enter active + Clean state
end_time = datetime.datetime.now() + datetime.timedelta(seconds=9000)
while end_time > datetime.datetime.now():
flag = True
status_report = rados_obj.run_ceph_command(cmd="ceph report")
# Proceeding to check if all PG's are in active + clean
for entry in status_report["num_pg_by_state"]:
rec = (
"backfilling",
"degraded",
"incomplete",
"recovering",
"recovery_wait",
"backfilling_wait",
"peered",
"undersized",
)
if any(key in rec for key in entry["state"].split("+")):
flag = False
if flag:
log.info(
"The recovery and back-filling of the OSD is completed")
break
log.info(
f"Waiting for active + clean. Active aletrs: {status_report['health']['checks'].keys()},"
f"PG States : {status_report['num_pg_by_state']}"
f" checking status again in 1 minute")
time.sleep(60)
# getting the acting set for the created pool after recovery
acting_pg_set = rados_obj.get_pg_acting_set(
pool_name=ec_config["pool_name"])
if len(acting_pg_set) != ec_config["k"] + ec_config["m"]:
log.error(
f"acting set consists of only these : {acting_pg_set} OSD's, less than k+m"
)
return 1
log.info(
f" Acting set of the pool consists of OSD's : {acting_pg_set}")
# Changing recovery threads back to default
rados_obj.change_recover_threads(config=ec_config, action="rm")
log.debug("Starting the stopped OSD's")
for osd_id in stop_osds:
if not rados_obj.change_osd_state(action="restart", target=osd_id):
log.error(f"Unable to restart the OSD : {osd_id}")
return 1
# Sleep for 5 seconds for OSD's to join the cluster
time.sleep(5)
if not flag:
log.error(
"The pool did not reach active + Clean state after recovery")
return 1
# Deleting the pool created
if not rados_obj.detete_pool(pool=ec_config["pool_name"]):
log.error(
f"the pool {ec_config['pool_name']} could not be deleted")
return 1
log.info("Successfully tested EC pool recovery with K osd's surviving")
return 0
if config.get("Compression_tests"):
"""
Create a 2 replicated pools:
1. Pool_1 : enable any compression algorithm(def snappy) and compression mode(aggressive/force).
2. Pool_2 : set compression mode to none
Writing the same amount of data on 2 pools, size of pool with compression on would consume less space
"""
pool_config = config["Compression_tests"]["pool_config"]
compression_config = config["Compression_tests"]["compression_config"]
pool_1 = pool_config["pool-1"]
pool_2 = pool_config["pool-2"]
if config["Compression_tests"]["pool_type"] == "replicated":
if not rados_obj.create_pool(pool_name=pool_1, **pool_config):
log.error("could not create pool-1")
return 1
if not rados_obj.create_pool(pool_name=pool_2, **pool_config):
log.error("could not create pool-2")
return 1
elif config["Compression_tests"]["pool_type"] == "erasure":
pool_config["pool_name"] = pool_1
if not rados_obj.create_erasure_pool(name=pool_1, **pool_config):
log.error("could not create pool-1")
return 1
pool_config["pool_name"] = pool_2
if not rados_obj.create_erasure_pool(name=pool_2, **pool_config):
log.error("could not create pool-2")
return 1
del pool_config["pool_name"]
log.debug("Created two pools to test compression")
# Enabling compression on pool-1
if not rados_obj.pool_inline_compression(pool_name=pool_1,
**compression_config):
log.error(
f"Error setting compression on pool : {pool_1} for config {compression_config}"
)
return 1
# Writing the same amount of data into two pools
if not rados_obj.bench_write(pool_name=pool_1, **pool_config):
log.error(
"Failed to write objects into Pool-1, with compression enabled"
)
return 1
if not rados_obj.bench_write(pool_name=pool_2, **pool_config):
log.error(
"Failed to write objects into Pool-2, without compression enabled"
)
return 1
# Sleeping for 5 seconds for status to be updated.
time.sleep(5)
log.debug(
"Finished writing data into the two pools. Checking pool stats")
try:
pool_stats = rados_obj.run_ceph_command(
cmd="ceph df detail")["pools"]
pool_1_stats = [
detail for detail in pool_stats if detail["name"] == pool_1
][0]["stats"]
pool_2_stats = [
detail for detail in pool_stats if detail["name"] == pool_2
][0]["stats"]
except KeyError:
log.error(
"No stats about the pools requested found on the cluster")
return 1
log.debug(f"Pool-1 stats: {pool_1_stats}")
log.debug(f"Pool-2 stats: {pool_2_stats}")
if pool_1_stats["compress_bytes_used"] < 0:
log.error("No data stored under pool-1 is compressed")
return 1
if pool_1_stats["kb_used"] >= pool_2_stats["kb_used"]:
log.error("Compression has no effect on the pool size...")
return 1
if config["Compression_tests"].get("verify_compression_ratio_set"):
# added verification for test: CEPH-83571672
if not rados_obj.check_compression_size(pool_name=pool_1,
**compression_config):
log.error("data not compressed in accordance to ratio set")
return 1
log.info("Pool size is less when compression is enabled")
return 0
if config.get("test_autoscaler_bulk_feature"):
"""
Tests to verify the autoscaler bulk flag, which allows pools to make use of
scale-down profile, making those pools start with full compliments of PG sets.
Tests include
1. creating new pools with bulk,
2. enabling/disabling bulk flag on existing pools
3. Verify the PG changes when the flag is set/unset
Verifies bugs : https://bugzilla.redhat.com/show_bug.cgi?id=2049851
"""
regex = r"\s*(\d.\d)-rhel-\d"
build = (re.search(regex,
config.get("build",
config.get("rhbuild")))).groups()[0]
if not float(build) > 5.0:
log.info(
"Test running on version less than 5.1, skipping verifying bulk flags"
)
return 0
# Creating a pool with bulk feature
pool_name = config.get("pool_name")
if not pool_obj.set_bulk_flag(pool_name=pool_name):
log.error("Failed to create a pool with bulk features")
return 1
# Checking the autoscaler status, final PG counts, bulk flags
pg_target_init = pool_obj.get_target_pg_num_bulk_flag(
pool_name=pool_name)
# Unsetting the bulk flag and checking the change in the PG counts
if not pool_obj.rm_bulk_flag(pool_name=pool_name):
log.error("Failed to create a pool with bulk features")
return 1
# Sleeping for 5 seconds for new PG num to bets et
time.sleep(5)
pg_target_interim = pool_obj.get_target_pg_num_bulk_flag(
pool_name=pool_name)
# The target PG's once the flag is disabled must be lesser than when enabled
if pg_target_interim >= pg_target_init:
log.error("PG's not reduced after bulk flag disabled")
return 1
# Setting the bulk flag on pool again and checking the change in the PG counts
if not pool_obj.set_bulk_flag(pool_name=pool_name):
log.error("Failed to disable/remove bulk features on pool")
return 1
# Sleeping for 5 seconds for new PG num to bets et
time.sleep(5)
pg_target_final = pool_obj.get_target_pg_num_bulk_flag(
pool_name=pool_name)
# The target PG's once the flag is disabled must be lesser than when enabled
if pg_target_interim >= pg_target_final:
log.error("PG's not Increased after bulk flag Enabled")
return 1
if config.get("delete_pool"):
rados_obj.detete_pool(pool=pool_name)
log.info("Verified the workings of bulk flag")
return 0
if config.get("verify_pool_target_ratio"):
log.debug("Verifying target size ratio on pools")
target_configs = config["verify_pool_target_ratio"]["configurations"]
# Creating pools and starting the test
for entry in target_configs.values():
log.debug(f"Creating {entry['pool_type']} pool on the cluster")
if entry.get("pool_type", "replicated") == "erasure":
method_should_succeed(rados_obj.create_erasure_pool,
name=entry["pool_name"],
**entry)
else:
method_should_succeed(
rados_obj.create_pool,
**entry,
)
rados_obj.bench_write(**entry)
if not pool_obj.verify_target_ratio_set(
pool_name=entry["pool_name"],
ratio=entry["target_size_ratio"]):
log.error(
f"Could not change the target ratio on the pool: {entry['pool_name']}"
)
return 1
log.debug("Set the ratio. getting the projected pg's")
rados_obj.change_recover_threads(config=config, action="set")
log.debug(
"Waiting for the rebalancing to complete on the cluster after the change"
)
# Sleeping for 2 minutes for rebalancing to start & for new PG count to be updated.
time.sleep(120)
new_pg_count = int(
pool_obj.get_pg_autoscaler_value(pool_name=entry["pool_name"],
item="pg_num_target"))
if new_pg_count <= entry["pg_num"]:
log.error(
f"Count of PG's not increased on the pool: {entry['pool_name']}"
f"Initial creation count : {entry['pg_num']}"
f"New count after setting num target : {new_pg_count}")
return 1
res = wait_for_clean_pg_sets(rados_obj)
if not res:
log.error(
"PG's in cluster are not active + Clean after the ratio change"
)
return 1
if not pool_obj.verify_target_ratio_set(
pool_name=entry["pool_name"], ratio=0.0):
log.error(
f"Could not remove the target ratio on the pool: {entry['pool_name']}"
)
return 1
# Sleeping for 2 minutes for rebalancing to start & for new PG count to be updated.
time.sleep(120)
# Checking if after the removal of ratio, the PG count has reduced
end_pg_count = int(
pool_obj.get_pg_autoscaler_value(pool_name=entry["pool_name"],
item="pg_num_target"))
if end_pg_count >= new_pg_count:
log.error(
f"Count of PG's not changed/ reverted on the pool: {entry['pool_name']}"
f" after removing the target ratios")
return 1
rados_obj.change_recover_threads(config=config, action="rm")
if entry.get("delete_pool", False):
rados_obj.detete_pool(pool=entry["pool_name"])
log.info(
f"Completed the test of target ratio on pool: {entry['pool_name']} "
)
log.info("Target ratio tests completed")
return 0
if config.get("verify_mon_target_pg_per_osd"):
pg_conf = config.get("verify_mon_target_pg_per_osd")
if not mon_obj.set_config(**pg_conf):
log.error("Could not set the value for mon_target_pg_per_osd ")
return 1
mon_obj.remove_config(**pg_conf)
log.info("Set and verified the value for mon_target_pg_per_osd ")
return 0
if config.get("verify_pg_num_min"):
log.debug("Verifying pg_num_min on pools")
target_configs = config["verify_pg_num_min"]["configurations"]
# Creating pools and starting the test
for entry in target_configs.values():
log.debug(f"Creating {entry['pool_type']} pool on the cluster")
if entry.get("pool_type", "replicated") == "erasure":
method_should_succeed(rados_obj.create_erasure_pool,
name=entry["pool_name"],
**entry)
else:
method_should_succeed(
rados_obj.create_pool,
**entry,
)
rados_obj.bench_write(**entry)
if not rados_obj.set_pool_property(pool=entry["pool_name"],
props="pg_num_min",
value=entry["pg_num_min"]):
log.error("Could not set the pg_min_size on the pool")
return 1
if entry.get("delete_pool", False):
rados_obj.detete_pool(pool=entry["pool_name"])
log.info(
f"Completed the test of pg_min_num on pool: {entry['pool_name']} "
)
log.info("pg_min_num tests completed")
return 0