def run(ceph_cluster, **kw):
"""
Automates OSD re-balance test scenarios.
1. Create replicated and/or erasure pool/pools
2. Identify the first osd to be removed
3. Fetch the host by daemon_type=osd and osd id
4. Fetch container id and device path
5. Mark osd out and wait for pgs to be active+clean
6. Remove OSD
7. Zap device and wait for device not present
8. Identify the second osd to be removed
9. Fetch the host by daemon_type=osd and osd id
10. Fetch container id and device path
11. Mark osd out
12. Add first osd and wait for device present and pgs to be active+clean
"""
try:
log.info(run.__doc__)
config = kw["config"]
cephadm = CephAdmin(cluster=ceph_cluster, **config)
rados_obj = RadosOrchestrator(node=cephadm)
client_node = ceph_cluster.get_nodes(role="client")[0]
log.info("Running osd in progress rebalance tests")
pool = create_pools(config, rados_obj, client_node)
should_not_be_empty(pool, "Failed to retrieve pool details")
write_to_pools(config, rados_obj, client_node)
rados_obj.change_recover_threads(config=pool, action="set")
acting_pg_set = rados_obj.get_pg_acting_set(
pool_name=pool["pool_name"])
log.info(f"Acting set {acting_pg_set}")
should_not_be_empty(acting_pg_set, "Failed to retrieve acting pg set")
osd_id = acting_pg_set[0]
host = rados_obj.fetch_host_node(daemon_type="osd", daemon_id=osd_id)
should_not_be_empty(host, "Failed to fetch host details")
dev_path = get_device_path(host, osd_id)
log.debug(
f"osd1 device path : {dev_path}, osd_id : {osd_id}, host.hostname : {host.hostname}"
)
utils.set_osd_devices_unamanged(ceph_cluster, unmanaged=True)
method_should_succeed(utils.set_osd_out, ceph_cluster, osd_id)
method_should_succeed(wait_for_clean_pg_sets, rados_obj)
utils.osd_remove(ceph_cluster, osd_id)
method_should_succeed(wait_for_clean_pg_sets, rados_obj)
method_should_succeed(utils.zap_device, ceph_cluster, host.hostname,
dev_path)
method_should_succeed(wait_for_device, host, osd_id, action="remove")
osd_id1 = acting_pg_set[1]
host1 = rados_obj.fetch_host_node(daemon_type="osd", daemon_id=osd_id1)
should_not_be_empty(host1, "Failed to fetch host details")
dev_path1 = get_device_path(host1, osd_id1)
log.debug(
f"osd2 device path : {dev_path1}, osd_id : {osd_id1}, host.hostname : {host1.hostname}"
)
method_should_succeed(utils.set_osd_out, ceph_cluster, osd_id1)
utils.add_osd(ceph_cluster, host.hostname, dev_path, osd_id)
method_should_succeed(wait_for_device, host, osd_id, action="add")
method_should_succeed(wait_for_clean_pg_sets, rados_obj)
acting_pg_set1 = rados_obj.get_pg_acting_set(
pool_name=pool["pool_name"])
if len(acting_pg_set) != len(acting_pg_set1):
log.error(
f"Acting pg set count before {acting_pg_set} and after {acting_pg_set1} rebalance mismatched"
)
return 1
if pool.get("rados_put", False):
do_rados_get(client_node, pool["pool_name"], 1)
utils.set_osd_devices_unamanged(ceph_cluster, unmanaged=False)
rados_obj.change_recover_threads(config=pool, action="rm")
if config.get("delete_pools"):
for name in config["delete_pools"]:
method_should_succeed(rados_obj.detete_pool, name)
log.info("deleted all the given pools successfully")
return 0
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
"""
Automates OSD re-balance test scenarios.
1. Create replicated and/or erasure pool/pools
2. Identify the osd to be removed
3. Fetch the host by daemon_type=osd and osd id
4. Fetch container id and device path
5. Mark osd out and wait for pgs to be active+clean
6. Remove OSD
7. Zap device and wait for device not present
8. Add OSD and wait for device present and pgs to be active+clean
"""
log.info(run.__doc__)
config = kw["config"]
cephadm = CephAdmin(cluster=ceph_cluster, **config)
rados_obj = RadosOrchestrator(node=cephadm)
client_node = ceph_cluster.get_nodes(role="client")[0]
log.info("Running create pool test case")
if config.get("create_pools"):
pools = config.get("create_pools")
for each_pool in pools:
cr_pool = each_pool["create_pool"]
if cr_pool.get("pool_type", "replicated") == "erasure":
method_should_succeed(rados_obj.create_erasure_pool,
name=cr_pool["pool_name"],
**cr_pool)
else:
method_should_succeed(rados_obj.create_pool,
pool_name=cr_pool["pool_name"],
**cr_pool)
method_should_succeed(rados_obj.bench_write, **cr_pool)
pool = random.choice(pools)["create_pool"]
if not pool:
log.error("Failed to retrieve pool details")
return 1
rados_obj.change_recover_threads(config=pool, action="set")
acting_pg_set = rados_obj.get_pg_acting_set(pool_name=pool["pool_name"])
log.info(f"Acting set {acting_pg_set}")
if not acting_pg_set:
log.error("Failed to retrieve acting pg set")
return 1
osd_id = acting_pg_set[0]
host = rados_obj.fetch_host_node(daemon_type="osd", daemon_id=osd_id)
if not host:
log.error("Failed to fetch host details")
return 1
# fetch container id
out, _ = host.exec_command(sudo=True, cmd="podman ps --format json")
container_id = [
item["Names"][0] for item in json.loads(out.read().decode())
if f"osd.{osd_id}" in item["Command"]
][0]
if not container_id:
log.error("Failed to retrieve container id")
return 1
# fetch device path by osd_id
vol_out, _ = host.exec_command(
sudo=True,
cmd=f"podman exec {container_id} ceph-volume lvm list --format json",
)
volume_out = vol_out.read().decode()
dev_path = [
v[0]["devices"][0] for k, v in json.loads(volume_out).items()
if str(k) == str(osd_id)
][0]
if not dev_path:
log.error("Failed to get device path")
return 1
log.debug(
f"device path : {dev_path}, osd_id : {osd_id}, host.hostname : {host.hostname}"
)
utils.set_osd_devices_unamanged(ceph_cluster, unmanaged=True)
method_should_succeed(utils.set_osd_out, ceph_cluster, osd_id)
method_should_succeed(wait_for_clean_pg_sets, rados_obj)
utils.osd_remove(ceph_cluster, osd_id)
method_should_succeed(wait_for_clean_pg_sets, rados_obj)
method_should_succeed(utils.zap_device, ceph_cluster, host.hostname,
dev_path)
method_should_succeed(wait_for_device,
host,
container_id,
osd_id,
action="remove")
utils.add_osd(ceph_cluster, host.hostname, dev_path, osd_id)
method_should_succeed(wait_for_device,
host,
container_id,
osd_id,
action="add")
method_should_succeed(wait_for_clean_pg_sets, rados_obj)
do_rados_put(mon=client_node, pool=pool["pool_name"], nobj=1000)
method_should_succeed(wait_for_clean_pg_sets, rados_obj)
utils.set_osd_devices_unamanged(ceph_cluster, unmanaged=False)
rados_obj.change_recover_threads(config=pool, action="rm")
if config.get("delete_pools"):
for name in config["delete_pools"]:
method_should_succeed(rados_obj.detete_pool, name)
log.info("deleted all the given pools successfully")
return 0