def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11222"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get("config")
num_of_osds = config.get("num_of_osds")
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
cluster_health_beforeIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
dir1 = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
for client in client_info["clients"]:
log.info("Creating directory:")
client.exec_command(cmd="sudo mkdir %s%s" %
(client_info["mounting_dir"], dir1))
log.info("Creating directories with breadth and depth:")
client.exec_command(
sudo=True,
cmd=
f"python3 smallfile/smallfile_cli.py --operation create --threads 10 --file-size 4 --files 1000 "
f"--files-per-dir 10 --dirs-per-dir 2 --top {client_info['mounting_dir']}{dir1}",
long_running=True,
timeout=300,
)
return_counts = fs_util.io_verify(client)
result = fs_util.rc_verify("", return_counts)
print(result)
break
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="smallfile",
)
for op in p:
return_counts, rc = op
result1 = fs_util.rc_verify("", return_counts)
print(result1)
for client in client_info["clients"]:
client.exec_command(cmd="sudo rm -rf %s%s" %
(client_info["mounting_dir"], dir1))
break
for client in client_info["clients"]:
log.info("Creating directories with breadth and depth:")
client.exec_command(
sudo=True,
cmd="python3 smallfile/smallfile_cli.py "
"--operation create --threads 10 "
" --file-size 4 --files 1000 "
"--files-per-dir 10 --dirs-per-dir 2"
" --top %s%s" % (client_info["mounting_dir"], dir1),
long_running=True,
timeout=300,
)
return_counts = fs_util.io_verify(client)
result = fs_util.rc_verify("", return_counts)
print(result)
log.info("Renaming the dirs:")
client.exec_command(
sudo=True,
cmd="python3 smallfile/smallfile_cli.py "
"--operation rename --threads 10 --file-size 4"
" --file-size 4 --files 1000 "
"--files-per-dir 10 --dirs-per-dir 2"
" --top %s%s" % (client_info["mounting_dir"], dir1),
long_running=True,
timeout=300,
)
return_counts = fs_util.io_verify(client)
result = fs_util.rc_verify("", return_counts)
print(result)
break
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="smallfile",
)
for op in p:
return_counts, rc = op
result2 = fs_util.rc_verify("", return_counts)
print(result2)
cluster_health_afterIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
client1[0].exec_command(cmd="sudo mkdir %s%s" %
(client_info["mounting_dir"], dir_name))
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"read",
"m",
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client3,
client_info["mounting_dir"],
"write",
"m",
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client2,
client_info["mounting_dir"],
"read",
"m",
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"write",
"m",
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client3,
client_info["mounting_dir"],
dir_name=dir_name,
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dir_name,
0,
5,
iotype="smallfile",
)
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("11223", return_counts)
print(result)
if cluster_health_beforeIO == cluster_health_afterIO:
print("Testcase %s passed" % (tc))
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
if rc == 0:
log.info("Cleaning up successfull")
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11222'
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
client_info, rc = fs_util.get_clients()
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
cluster_health_beforeIO = check_ceph_healthly(client_info['mon_node'],
12, 1, None, 300)
dir1 = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
for client in client_info['clients']:
log.info("Creating directory:")
client.exec_command(cmd='sudo mkdir %s%s' %
(client_info['mounting_dir'], dir1))
log.info("Creating directories with breadth and depth:")
out, rc = client.exec_command(
cmd='sudo crefi %s%s --fop create --multi -b 10 -d 10 '
'--random --min=1K --max=10K' %
(client_info['mounting_dir'], dir1))
print(out.read().decode())
break
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='fio')
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir1,
0,
100,
iotype='touch')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='dd')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='crefi')
for op in p:
return_counts, rc = op
result1 = fs_util.rc_verify('', return_counts)
print(result1)
for client in client_info['clients']:
client.exec_command(cmd='sudo rm -rf %s%s' %
(client_info['mounting_dir'], dir1))
break
for client in client_info['clients']:
log.info("Creating directories with breadth and depth:")
out, rc = client.exec_command(
cmd='sudo crefi %s%s --fop create --multi -b 10 -d 10 '
'--random --min=1K --max=10K' %
(client_info['mounting_dir'], dir1))
print(out.read().decode())
log.info("Renaming the dirs:")
out, rc = client.exec_command(
cmd='sudo crefi '
'%s%s --fop rename --multi -b 10 -d 10 --random '
'--min=1K --max=10K' % (client_info['mounting_dir'], dir1))
print(out.read().decode())
break
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='fio')
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir1,
0,
100,
iotype='touch')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='dd')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='crefi')
for op in p:
return_counts, rc = op
result2 = fs_util.rc_verify('', return_counts)
print(result2)
cluster_health_afterIO = check_ceph_healthly(client_info['mon_node'],
12, 1, None, 300)
if cluster_health_beforeIO == cluster_health_afterIO:
print("Testcase %s passed" % (tc))
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11219,11224"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
config = kw.get("config")
num_of_osds = config.get("num_of_osds")
fs_util = FsUtils(ceph_cluster)
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
cluster_health_beforeIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
log.error("Activate multiple mdss failed")
return 1
client1[0].exec_command(
sudo=True,
cmd=f"mkdir {client_info['mounting_dir']}{dir_name}",
)
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(fs_util.read_write_IO, client2,
client_info["mounting_dir"], "g", "read")
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
2,
iotype="smallfile",
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"g",
"readwrite",
)
p.spawn(fs_util.read_write_IO, client3,
client_info["mounting_dir"])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
if result == "Data validation success":
print("Data validation success")
dirs, rc = fs_util.mkdir(client1, 0, 3,
client_info["mounting_dir"], dir_name)
if rc == 0:
log.info("Directories created")
else:
raise CommandFailed("Directory creation failed")
dirs = dirs.split("\n")
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[1],
0,
1,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile",
)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_create",
fnum=10,
fsize=1024,
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[1],
0,
1,
iotype="smallfile_create",
fnum=10,
fsize=1024,
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile_create",
fnum=10,
fsize=1024,
)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_delete",
fnum=10,
fsize=1024,
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[1],
0,
1,
iotype="smallfile_delete",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile_delete",
fnum=10,
fsize=1024,
)
for op in p:
return_counts, rc = op
cluster_health_afterIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
log.info("Execution of Test case CEPH-%s ended" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
if cluster_health_beforeIO == cluster_health_afterIO:
print(result)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=10,
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[0],
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[0],
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile",
)
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"read",
dir_name=dirs[2],
)
p.spawn(
fs_util.read_write_IO,
client2,
client_info["mounting_dir"],
"g",
"read",
dir_name=dirs[0],
)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=10,
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[0],
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[0],
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile",
)
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"read",
dir_name=dirs[0],
)
p.spawn(
fs_util.read_write_IO,
client2,
client_info["mounting_dir"],
"g",
"read",
dir_name=dirs[0],
)
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
if rc == 0:
log.info("Cleaning up successfull")
else:
return 1
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
if rc == 0:
log.info("Cleaning up successfull")
else:
return 1
log.info("Execution of Test case CEPH-%s ended" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
print(result)
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11255_11336-fuse client'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
config = kw.get('config')
num_of_osds = config.get('num_of_osds')
fs_util = FsUtils(ceph_cluster)
build = config.get('build', config.get('rhbuild'))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("creating auth failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(
client3,
client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(
client4,
client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
cluster_health_beforeIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds, len(
client_info['mon_node']), build, None, 300)
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
rc = fs_util.standby_rank(
client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank')
if rc == 0:
log.info("Added standby ranks")
else:
log.error("Failed to add standby ranks")
return 1
client1[0].exec_command(
cmd='sudo mkdir %s%s' %
(client_info['mounting_dir'], dir_name))
if client1[0].node.exit_status == 0:
log.info("Dir created")
else:
raise CommandFailed('Dir creation failed')
rc1 = fs_util.fstab_entry(
client1,
client_info['mounting_dir'],
action='doEntry')
rc2 = fs_util.fstab_entry(
client2,
client_info['mounting_dir'],
action='doEntry')
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
raise CommandFailed("FsEntry failed")
rc1 = fs_util.fstab_entry(
client3,
client_info['mounting_dir'],
action='doEntry', mon_node_ip=client_info['mon_node_ip'])
rc2 = fs_util.fstab_entry(
client4,
client_info['mounting_dir'],
action='doEntry', mon_node_ip=client_info['mon_node_ip'])
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
raise CommandFailed("FsEntry failed")
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client3,
client_info['mounting_dir'], 'g', 'read')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir_name,
0,
50,
iotype='fio')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
1,
iotype='smallfile_create', fnum=1000, fsize=100)
p.spawn(
fs_util.stress_io,
client4,
client_info['mounting_dir'],
dir_name,
0,
1,
iotype='crefi')
p.spawn(fs_util.reboot, client1[0])
res = []
with parallel() as p:
for node in client_info['mds_nodes']:
p.spawn(fs_util.heartbeat_map, node)
for op in p:
res.append(op)
print(res)
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client4,
client_info['mounting_dir'], 'g', 'read')
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir_name,
0,
1,
iotype='fio')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='dd')
p.spawn(
fs_util.stress_io,
client4,
client_info['mounting_dir'],
dir_name,
0,
500,
iotype='touch'),
p.spawn(fs_util.reboot, client2[0])
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds, len(
client_info['mon_node']), build, None, 300)
if cluster_health_afterIO == cluster_health_beforeIO:
log.info('cluster is healthy')
else:
log.error("cluster is not healty")
return 1
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='fio')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='dd')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
500,
iotype='touch')
for node in client_info['osd_nodes']:
p.spawn(fs_util.reboot, node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='fio'),
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='dd')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
500,
iotype='touch')
for node in client_info['osd_nodes']:
fs_util.network_disconnect(node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='fio')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='dd')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
500,
iotype='touch')
for node in client_info['osd_nodes']:
fs_util.pid_kill(node, 'osd')
time.sleep(100)
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds, len(
client_info['mon_node']), build, None, 300)
if cluster_health_beforeIO == cluster_health_afterIO:
log.info("Cluster is healthy")
else:
return 1
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(
client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
rc = fs_util.standby_rank(
client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
if rc == 0:
log.info("removed standby ranks")
rc1 = fs_util.fstab_entry(
client1,
client_info['mounting_dir'],
action='revertEntry')
rc2 = fs_util.fstab_entry(
client2,
client_info['mounting_dir'],
action='revertEntry')
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
return 1
else:
rc = fs_util.client_clean_up(
client_info['fuse_clients'],
'',
client_info['mounting_dir'],
'umount')
rc = fs_util.standby_rank(
client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
if rc == 0:
log.info("removed standby ranks")
rc1 = fs_util.fstab_entry(
client1,
client_info['mounting_dir'],
action='revertEntry')
rc2 = fs_util.fstab_entry(
client2,
client_info['mounting_dir'],
action='revertEntry')
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
if rc == 0:
log.info('Cleaning up successfull')
log.info("Execution of Test cases CEPH-%s ended:" % (tc))
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(
client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
else:
rc = fs_util.client_clean_up(
client_info['fuse_clients'],
'',
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11222"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get("config")
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
cluster_health_beforeIO = check_ceph_healthly(client_info["mon_node"],
12, 1, build, None, 300)
dir1 = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
for client in client_info["clients"]:
log.info("Creating directory:")
client.exec_command(cmd="sudo mkdir %s%s" %
(client_info["mounting_dir"], dir1))
log.info("Creating directories with breadth and depth:")
out, rc = client.exec_command(
cmd="sudo crefi %s%s --fop create --multi -b 10 -d 10 "
"--random --min=1K --max=10K" %
(client_info["mounting_dir"], dir1))
print(out.read().decode())
break
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="crefi",
)
for op in p:
return_counts, rc = op
result1 = fs_util.rc_verify("", return_counts)
print(result1)
for client in client_info["clients"]:
client.exec_command(cmd="sudo rm -rf %s%s" %
(client_info["mounting_dir"], dir1))
break
for client in client_info["clients"]:
log.info("Creating directories with breadth and depth:")
out, rc = client.exec_command(
cmd="sudo crefi %s%s --fop create --multi -b 10 -d 10 "
"--random --min=1K --max=10K" %
(client_info["mounting_dir"], dir1))
print(out.read().decode())
log.info("Renaming the dirs:")
out, rc = client.exec_command(
cmd="sudo crefi "
"%s%s --fop rename --multi -b 10 -d 10 --random "
"--min=1K --max=10K" % (client_info["mounting_dir"], dir1))
print(out.read().decode())
break
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="crefi",
)
for op in p:
return_counts, rc = op
result2 = fs_util.rc_verify("", return_counts)
print(result2)
cluster_health_afterIO = check_ceph_healthly(client_info["mon_node"],
12, 1, build, None, 300)
if cluster_health_beforeIO == cluster_health_afterIO:
print("Testcase %s passed" % (tc))
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
if rc == 0:
log.info("Cleaning up successfull")
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11254-fuse_clients"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
config = kw.get("config")
num_of_osds = config.get("num_of_osds")
fs_util = FsUtils(ceph_cluster)
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(
client3, client_info["mounting_dir"], client_info["mon_node_ip"]
)
rc4 = fs_util.kernel_mount(
client4, client_info["mounting_dir"], client_info["mon_node_ip"]
)
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
cluster_health_beforeIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
rc = fs_util.standby_rank(
client_info["mds_nodes"], client_info["mon_node"], todo="add_rank"
)
if rc == 0:
log.info("Added standby ranks")
else:
raise Exception("Adding standby ranks failed")
client1[0].exec_command(
cmd="sudo mkdir %s%s" % (client_info["mounting_dir"], dir_name)
)
if client1[0].node.exit_status == 0:
log.info("Dir created")
else:
fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
)
rc1 = fs_util.fstab_entry(
client1, client_info["mounting_dir"], action="doEntry"
)
rc2 = fs_util.fstab_entry(
client2, client_info["mounting_dir"], action="doEntry"
)
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
raise CommandFailed("FsEntry failed")
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(
fs_util.read_write_IO, client3, client_info["mounting_dir"], "g", "read"
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
50,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=100,
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dir_name,
0,
1,
iotype="smallfile",
)
p.spawn(fs_util.reboot, client1[0])
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(
fs_util.read_write_IO, client4, client_info["mounting_dir"], "g", "read"
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
1,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
p.spawn(fs_util.reboot, client2[0])
cluster_health_afterIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
if cluster_health_afterIO == cluster_health_beforeIO:
log.info("cluster is healthy")
else:
log.error("cluster is not healty")
return 1
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
p.spawn(fs_util.reboot, node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
fs_util.network_disconnect(node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
fs_util.pid_kill(node, "mon")
cluster_health_afterIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
if cluster_health_beforeIO == cluster_health_afterIO:
log.info("Cluster is healthy")
else:
return 1
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
rc = fs_util.standby_rank(
client_info["mds_nodes"],
client_info["mon_node"],
todo="add_rank_revert",
)
if rc == 0:
log.info("removed standby ranks")
rc1 = fs_util.fstab_entry(
client1, client_info["mounting_dir"], action="revertEntry"
)
rc2 = fs_util.fstab_entry(
client2, client_info["mounting_dir"], action="revertEntry"
)
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
return 1
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"], "", client_info["mounting_dir"], "umount"
)
rc = fs_util.standby_rank(
client_info["mds_nodes"],
client_info["mon_node"],
todo="add_rank_revert",
)
if rc == 0:
log.info("removed standby ranks")
else:
return 1
rc1 = fs_util.fstab_entry(
client1, client_info["mounting_dir"], action="revertEntry"
)
rc2 = fs_util.fstab_entry(
client2, client_info["mounting_dir"], action="revertEntry"
)
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
return 1
if rc == 0:
log.info("Cleaning up successfull")
log.info("Execution of Test cases CEPH-%s ended:" % (tc))
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"], "", client_info["mounting_dir"], "umount"
)
if rc == 0:
log.info("Cleaning up successfull")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11256-fuse'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get('config')
num_of_osds = config.get('num_of_osds')
client_info, rc = fs_util.get_clients()
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise Exception("kernel mount failed")
cluster_health_beforeIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), None, 300)
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
rc = fs_util.standby_rank(client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank')
if rc == 0:
log.info("Added standby ranks")
else:
raise Exception("Adding standby ranks failed")
dirs, rc = fs_util.mkdir(client1, 0, 4, client_info['mounting_dir'],
dir_name)
if rc == 0:
log.info("Directories created")
else:
raise CommandFailed("Directory creation failed")
dirs = dirs.split('\n')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
'',
'',
iotype='smallfile_create',
fnum=1000,
fsize=1024)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[3],
0,
10,
iotype='dd')
for node in client_info['mds_nodes']:
p.spawn(fs_util.reboot, node)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[1],
'',
'',
iotype='smallfile_create',
fnum=1000,
fsize=1024)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[3],
0,
10,
iotype='dd')
for node in client_info['mds_nodes']:
p.spawn(fs_util.daemon_systemctl, node, 'mds',
'active_mds_restart')
with parallel() as p:
for node in client_info['mds_nodes']:
p.spawn(fs_util.heartbeat_map, node)
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), None, 300)
if cluster_health_afterIO == cluster_health_beforeIO:
log.info('cluster is healthy')
else:
log.error("cluster is not healty")
return 1
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[3],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='dd')
for node in client_info['mds_nodes']:
fs_util.network_disconnect(node)
with parallel() as p:
for node in client_info['mds_nodes']:
p.spawn(fs_util.heartbeat_map, node)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='crefi')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[3],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='dd')
for node in client_info['mds_nodes']:
fs_util.pid_kill(node, 'mds')
with parallel() as p:
for node in client_info['mds_nodes']:
p.spawn(fs_util.heartbeat_map, node)
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), None, 300)
if cluster_health_beforeIO == cluster_health_afterIO:
log.info("Cluster is healthy")
else:
return 1
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
if rc == 0:
log.info('Cleaning up successfull')
fs_util.standby_rank(client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
log.info("Execution of Test cases CEPH-%s ended:" % (tc))
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
rc = fs_util.standby_rank(client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
rc = fs_util.standby_rank(client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
if rc == 0:
log.info('Cleaning up successfull')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11219,11224'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
config = kw.get('config')
num_of_osds = config.get('num_of_osds')
fs_util = FsUtils(ceph_cluster)
build = config.get('build', config.get('rhbuild'))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
cluster_health_beforeIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), build, None, 300)
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
log.error("Activate multiple mdss failed")
return 1
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client2,
client_info['mounting_dir'], 'g', 'read')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
2,
iotype='crefi')
p.spawn(fs_util.read_write_IO, client4,
client_info['mounting_dir'], 'g', 'readwrite')
p.spawn(fs_util.read_write_IO, client3,
client_info['mounting_dir'])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify('', return_counts)
if result == 'Data validation success':
print("Data validation success")
dirs, rc = fs_util.mkdir(client1, 0, 3,
client_info['mounting_dir'], dir_name)
if rc == 0:
log.info("Directories created")
else:
raise CommandFailed("Directory creation failed")
dirs = dirs.split('\n')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
100,
iotype='touch')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='dd')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='crefi')
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_delete',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='smallfile_delete')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_delete',
fnum=10,
fsize=1024)
for op in p:
return_counts, rc = op
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), build, None, 300)
log.info("Execution of Test case CEPH-%s ended" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
if cluster_health_beforeIO == cluster_health_afterIO:
print(result)
print('-----------------------------------------')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_create',
fnum=1000,
fsize=10)
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
5,
iotype='fio')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[0],
0,
10,
iotype='dd')
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
print('-------------------------------------------------------')
with parallel() as p:
p.spawn(fs_util.read_write_IO,
client1,
client_info['mounting_dir'],
'g',
'read',
dir_name=dirs[0])
p.spawn(fs_util.read_write_IO,
client2,
client_info['mounting_dir'],
'g',
'read',
dir_name=dirs[0])
print('-------------------------------------------------------')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_create',
fnum=1000,
fsize=10)
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
5,
iotype='fio')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[0],
0,
10,
iotype='dd')
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
print('-------------------------------------------------------')
with parallel() as p:
p.spawn(fs_util.read_write_IO,
client1,
client_info['mounting_dir'],
'g',
'read',
dir_name=dirs[0])
p.spawn(fs_util.read_write_IO,
client2,
client_info['mounting_dir'],
'g',
'read',
dir_name=dirs[0])
print('-------------------------------------------------------')
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
else:
return 1
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
else:
return 1
log.info("Execution of Test case CEPH-%s ended" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
print(result)
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
