def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11222"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get("config")
num_of_osds = config.get("num_of_osds")
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
cluster_health_beforeIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
dir1 = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
for client in client_info["clients"]:
log.info("Creating directory:")
client.exec_command(cmd="sudo mkdir %s%s" %
(client_info["mounting_dir"], dir1))
log.info("Creating directories with breadth and depth:")
client.exec_command(
sudo=True,
cmd=
f"python3 smallfile/smallfile_cli.py --operation create --threads 10 --file-size 4 --files 1000 "
f"--files-per-dir 10 --dirs-per-dir 2 --top {client_info['mounting_dir']}{dir1}",
long_running=True,
timeout=300,
)
return_counts = fs_util.io_verify(client)
result = fs_util.rc_verify("", return_counts)
print(result)
break
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="smallfile",
)
for op in p:
return_counts, rc = op
result1 = fs_util.rc_verify("", return_counts)
print(result1)
for client in client_info["clients"]:
client.exec_command(cmd="sudo rm -rf %s%s" %
(client_info["mounting_dir"], dir1))
break
for client in client_info["clients"]:
log.info("Creating directories with breadth and depth:")
client.exec_command(
sudo=True,
cmd="python3 smallfile/smallfile_cli.py "
"--operation create --threads 10 "
" --file-size 4 --files 1000 "
"--files-per-dir 10 --dirs-per-dir 2"
" --top %s%s" % (client_info["mounting_dir"], dir1),
long_running=True,
timeout=300,
)
return_counts = fs_util.io_verify(client)
result = fs_util.rc_verify("", return_counts)
print(result)
log.info("Renaming the dirs:")
client.exec_command(
sudo=True,
cmd="python3 smallfile/smallfile_cli.py "
"--operation rename --threads 10 --file-size 4"
" --file-size 4 --files 1000 "
"--files-per-dir 10 --dirs-per-dir 2"
" --top %s%s" % (client_info["mounting_dir"], dir1),
long_running=True,
timeout=300,
)
return_counts = fs_util.io_verify(client)
result = fs_util.rc_verify("", return_counts)
print(result)
break
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="smallfile",
)
for op in p:
return_counts, rc = op
result2 = fs_util.rc_verify("", return_counts)
print(result2)
cluster_health_afterIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
client1[0].exec_command(cmd="sudo mkdir %s%s" %
(client_info["mounting_dir"], dir_name))
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"read",
"m",
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client3,
client_info["mounting_dir"],
"write",
"m",
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client2,
client_info["mounting_dir"],
"read",
"m",
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"write",
"m",
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
dir_name=dir_name,
)
p.spawn(
fs_util.read_write_IO,
client3,
client_info["mounting_dir"],
dir_name=dir_name,
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dir_name,
0,
5,
iotype="smallfile",
)
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("11223", return_counts)
print(result)
if cluster_health_beforeIO == cluster_health_afterIO:
print("Testcase %s passed" % (tc))
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
if rc == 0:
log.info("Cleaning up successfull")
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
config = kw.get('config')
num_of_dirs = config.get('num_of_dirs')
tc = '11227'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
build = config.get('build', config.get('rhbuild'))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client2,
client_info['mounting_dir'], 'g', 'read')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
'',
0,
2,
iotype='crefi',
)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
'',
0,
2,
iotype='crefi')
p.spawn(fs_util.read_write_IO, client4,
client_info['mounting_dir'], 'g', 'readwrite')
p.spawn(fs_util.read_write_IO, client3,
client_info['mounting_dir'])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify('', return_counts)
client1[0].exec_command(cmd='sudo mkdir %s%s' %
(client_info['mounting_dir'], 'testdir'))
if result == 'Data validation success':
print("Data validation success")
fs_util.activate_multiple_mdss(client_info['mds_nodes'])
log.info("Execution of Test case CEPH-%s started:" % (tc))
num_of_dirs = int(num_of_dirs / 5)
with parallel() as p:
p.spawn(fs_util.mkdir_bulk, client1, 0, num_of_dirs * 1,
client_info['mounting_dir'] + 'testdir/', dir_name)
p.spawn(fs_util.mkdir_bulk, client2, num_of_dirs * 1 + 1,
num_of_dirs * 2,
client_info['mounting_dir'] + 'testdir/', dir_name)
p.spawn(fs_util.mkdir_bulk, client1, num_of_dirs * 2 + 1,
num_of_dirs * 3,
client_info['mounting_dir'] + 'testdir/', dir_name)
p.spawn(fs_util.mkdir_bulk, client2, num_of_dirs * 3 + 1,
num_of_dirs * 4,
client_info['mounting_dir'] + 'testdir/', dir_name)
p.spawn(fs_util.mkdir_bulk, client1, num_of_dirs * 4 + 1,
num_of_dirs * 5,
client_info['mounting_dir'] + 'testdir/', dir_name)
for op in p:
rc = op
if rc == 0:
log.info('Directories created successfully')
else:
raise CommandFailed('Directory creation failed')
with parallel() as p:
p.spawn(fs_util.max_dir_io, client1,
client_info['mounting_dir'] + '/testdir/', dir_name, 0,
num_of_dirs * 1, 10)
p.spawn(fs_util.max_dir_io, client2,
client_info['mounting_dir'] + '/testdir/', dir_name,
num_of_dirs * 1, num_of_dirs * 2, 10)
rc = fs_util.check_mount_exists(client1[0])
if rc == 0:
fs_util.pinning(client1, 0, 10,
client_info['mounting_dir'] + 'testdir/',
dir_name, 0)
p.spawn(fs_util.max_dir_io, client3,
client_info['mounting_dir'] + '/testdir/', dir_name,
num_of_dirs * 3, num_of_dirs * 4, 10)
p.spawn(fs_util.max_dir_io, client4,
client_info['mounting_dir'] + '/testdir/', dir_name,
num_of_dirs * 4, num_of_dirs * 5, 10)
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client1,
client_info['mounting_dir'] + '/testdir/', dir_name, 0,
10, 100, fs_util.mds_fail_over,
client_info['mds_nodes'])
with parallel() as p:
p.spawn(fs_util.pinning, client2, 10, num_of_dirs * 1,
client_info['mounting_dir'] + '/testdir/', dir_name, 1)
p.spawn(fs_util.pinning, client3, num_of_dirs * 1,
num_of_dirs * 2,
client_info['mounting_dir'] + '/testdir/', dir_name, 1)
p.spawn(fs_util.pinning, client4, num_of_dirs * 2,
num_of_dirs * 3,
client_info['mounting_dir'] + '/testdir/', dir_name, 1)
p.spawn(fs_util.pinning, client1, num_of_dirs * 3,
num_of_dirs * 4,
client_info['mounting_dir'] + '/testdir/', dir_name, 1)
p.spawn(fs_util.pinning, client3, num_of_dirs * 4,
num_of_dirs * 5,
client_info['mounting_dir'] + '/testdir/', dir_name, 1)
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client1,
client_info['mounting_dir'] + '/testdir/', dir_name, 0,
10, 100, fs_util.mds_fail_over,
client_info['mds_nodes'])
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client2,
client_info['mounting_dir'] + '/testdir/', dir_name, 0,
10, 100, fs_util.mds_fail_over,
client_info['mds_nodes'])
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client3,
client_info['mounting_dir'] + '/testdir/', dir_name, 0,
10, 100, fs_util.mds_fail_over,
client_info['mds_nodes'])
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client4,
client_info['mounting_dir'] + '/testdir/', dir_name, 0,
10, 100, fs_util.mds_fail_over,
client_info['mds_nodes'])
log.info("Execution of Test case CEPH-%s ended:" % (tc))
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc_client = fs_util.client_clean_up(
client_info['fuse_clients'], client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
rc_mds = fs_util.mds_cleanup(client_info['mds_nodes'], None)
else:
rc_client = fs_util.client_clean_up(
client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
rc_mds = fs_util.mds_cleanup(client_info['mds_nodes'], None)
if rc_client == 0 and rc_mds == 0:
log.info('Cleaning up successfull')
else:
return 1
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc_client = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
rc_mds = fs_util.mds_cleanup(client_info['mds_nodes'], None)
else:
rc_client = fs_util.client_clean_up(client_info['fuse_clients'],
'',
client_info['mounting_dir'],
'umount')
rc_mds = fs_util.mds_cleanup(client_info['mds_nodes'], None)
if rc_client == 0 and rc_mds == 0:
log.info('Cleaning up successfull')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11230'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
client_info, rc = fs_util.get_clients()
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(
client3,
client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(
client4,
client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client2,
client_info['mounting_dir'], 'g', 'read')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
'',
0,
1,
iotype='crefi',
)
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
1,
iotype='crefi'
)
p.spawn(fs_util.read_write_IO, client4,
client_info['mounting_dir'], 'g', 'readwrite')
p.spawn(fs_util.read_write_IO, client3,
client_info['mounting_dir'])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify('', return_counts)
if result == 'Data validation success':
print("Data validation success")
fs_util.activate_multiple_mdss(client_info['mds_nodes'])
log.info("Execution of Test case CEPH-%s started:" % (tc))
for client in client1:
client.exec_command(
cmd='sudo mkdir %s%s_{1..50}' %
(client_info['mounting_dir'], dir_name))
if client.node.exit_status == 0:
log.info("directories created succcessfully")
else:
raise CommandFailed("directories creation failed")
with parallel() as p:
p.spawn(
fs_util.pinning,
client1,
1,
25,
client_info['mounting_dir'],
dir_name,
0)
p.spawn(
fs_util.pinning,
client3,
26,
50,
client_info['mounting_dir'],
dir_name,
1)
with parallel() as p:
p.spawn(
fs_util.max_dir_io,
client1,
client_info['mounting_dir'],
dir_name,
1,
25,
1000)
p.spawn(
fs_util.max_dir_io,
client3,
client_info['mounting_dir'],
dir_name,
26,
50,
1000)
with parallel() as p:
p.spawn(
fs_util.pinned_dir_io_mdsfailover,
client1,
client_info['mounting_dir'],
dir_name,
1,
25,
10,
fs_util.mds_fail_over,
client_info['mds_nodes'])
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(
fs_util.pinned_dir_io_mdsfailover,
client4,
client_info['mounting_dir'],
dir_name,
26,
50,
20,
fs_util.mds_fail_over,
client_info['mds_nodes'])
for op in p:
return_counts, rc = op
log.info("Execution of Test case CEPH-%s ended:" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
log.info(result)
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc_client = fs_util.client_clean_up(
client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
rc_mds = fs_util.mds_cleanup(client_info['mds_nodes'], None)
else:
rc_client = fs_util.client_clean_up(
client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
rc_mds = fs_util.mds_cleanup(client_info['mds_nodes'], None)
if rc_client == 0 and rc_mds == 0:
log.info('Cleaning up successfull')
else:
return 1
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'],
'',
client_info['mounting_dir'], 'umount')
if rc == 0:
log.info('Cleaning up successfull')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11219,11224"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
config = kw.get("config")
num_of_osds = config.get("num_of_osds")
fs_util = FsUtils(ceph_cluster)
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
cluster_health_beforeIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
log.error("Activate multiple mdss failed")
return 1
client1[0].exec_command(
sudo=True,
cmd=f"mkdir {client_info['mounting_dir']}{dir_name}",
)
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(fs_util.read_write_IO, client2,
client_info["mounting_dir"], "g", "read")
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
2,
iotype="smallfile",
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"g",
"readwrite",
)
p.spawn(fs_util.read_write_IO, client3,
client_info["mounting_dir"])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
if result == "Data validation success":
print("Data validation success")
dirs, rc = fs_util.mkdir(client1, 0, 3,
client_info["mounting_dir"], dir_name)
if rc == 0:
log.info("Directories created")
else:
raise CommandFailed("Directory creation failed")
dirs = dirs.split("\n")
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[1],
0,
1,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile",
)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_create",
fnum=10,
fsize=1024,
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[1],
0,
1,
iotype="smallfile_create",
fnum=10,
fsize=1024,
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile_create",
fnum=10,
fsize=1024,
)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_delete",
fnum=10,
fsize=1024,
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[1],
0,
1,
iotype="smallfile_delete",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile_delete",
fnum=10,
fsize=1024,
)
for op in p:
return_counts, rc = op
cluster_health_afterIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
log.info("Execution of Test case CEPH-%s ended" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
if cluster_health_beforeIO == cluster_health_afterIO:
print(result)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=10,
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[0],
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[0],
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile",
)
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"read",
dir_name=dirs[2],
)
p.spawn(
fs_util.read_write_IO,
client2,
client_info["mounting_dir"],
"g",
"read",
dir_name=dirs[0],
)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=10,
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[0],
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[0],
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile",
)
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"read",
dir_name=dirs[0],
)
p.spawn(
fs_util.read_write_IO,
client2,
client_info["mounting_dir"],
"g",
"read",
dir_name=dirs[0],
)
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
if rc == 0:
log.info("Cleaning up successfull")
else:
return 1
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
if rc == 0:
log.info("Cleaning up successfull")
else:
return 1
log.info("Execution of Test case CEPH-%s ended" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
print(result)
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '10625,11225'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get('config')
build = config.get('build', config.get('rhbuild'))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
log.error("Activate multiple mdss failed")
return 1
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client2,
client_info['mounting_dir'], 'g', 'read')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
'',
0,
1,
iotype='crefi',
)
p.spawn(fs_util.read_write_IO, client4,
client_info['mounting_dir'], 'g', 'readwrite')
p.spawn(fs_util.read_write_IO, client3,
client_info['mounting_dir'])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify('', return_counts)
if result == 'Data validation success':
dirs, rc = fs_util.mkdir(client1, 0, 6,
client_info['mounting_dir'], dir_name)
if rc == 0:
log.info("Directories created")
else:
raise CommandFailed("Directory creation failed")
dirs = dirs.split('\n')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_create',
fnum=1000,
fsize=10)
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='smallfile_create',
fnum=1000,
fsize=10)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_create',
fnum=1000,
fsize=10)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_create',
fnum=1000,
fsize=10)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_rename',
fnum=1000,
fsize=10)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_delete-renamed',
fnum=1000,
fsize=10)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_delete',
fnum=1000,
fsize=10)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[3],
0,
1,
iotype='smallfile_create',
fnum=1,
fsize=1000000)
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[4],
0,
1,
iotype='smallfile_create',
fnum=1,
fsize=1000000)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[5],
0,
1,
iotype='smallfile_create',
fnum=1,
fsize=1000000)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[6],
0,
1,
iotype='smallfile_create',
fnum=1,
fsize=1000000)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[3],
0,
1,
iotype='smallfile_rename',
fnum=1,
fsize=1000000)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[3],
0,
1,
iotype='smallfile_delete-renamed',
fnum=1,
fsize=1000000)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[4],
0,
1,
iotype='smallfile_delete',
fnum=1,
fsize=1000000)
dir_name = '!@#$%^&*()-_=+[]{};:,.<>?'
out, rc = client1[0].exec_command(
cmd="sudo mkdir '%s%s'" % (client_info['mounting_dir'], dir_name))
if client1[0].node.exit_status == 0:
log.info("Directory created")
else:
raise CommandFailed("Directory creation failed")
for client in client_info['fuse_clients']:
file_name = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(255))
client.exec_command(
cmd="sudo touch '%s%s/%s'" %
(client_info['mounting_dir'], dir_name, file_name))
for client in client_info['kernel_clients']:
if client.pkg_type == 'rpm':
file_name = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(255))
client.exec_command(
cmd="sudo touch '%s%s/%s'" %
(client_info['mounting_dir'], dir_name, file_name))
for num in range(0, 5):
for client in client_info['fuse_clients']:
client.exec_command(
cmd="sudo crefi %s'%s' --fop create -t %s "
"--multi -b 10 -d 10 -n 10 -T 10 "
"--random --min=1K --max=%dK" %
(client_info['mounting_dir'], dir_name, 'text', 5),
long_running=True)
for i in range(0, 6):
ops = [
'create', 'rename', 'chmod', 'chown', 'chgrp',
'setxattr'
]
rand_ops = random.choice(ops)
ftypes = ['text', 'sparse', 'binary', 'tar']
rand_filetype = random.choice(ftypes)
rand_count = random.randint(2, 10)
client.exec_command(cmd='sudo crefi %s%s --fop %s -t %s '
'--multi -b 10 -d 10 -n 10 -T 10 '
'--random --min=1K --max=%dK' %
(client_info['mounting_dir'], dir_name,
rand_ops, rand_filetype, rand_count),
long_running=True)
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
if rc == 0:
log.info('Cleaning up successfull')
else:
return 1
log.info("Execution of Test cases CEPH-%s ended:" % (tc))
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
if rc == 0:
log.info('Cleaning up successfull')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11231"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get("config")
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = [client_info["fuse_clients"][0]]
client2 = [client_info["fuse_clients"][1]]
client3 = [client_info["kernel_clients"][0]]
client4 = [client_info["kernel_clients"][1]]
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
client1[0].exec_command(sudo=True,
cmd="mkdir %s%s" %
(client_info["mounting_dir"], dir_name))
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(fs_util.read_write_IO, client2,
client_info["mounting_dir"], "g", "read")
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
"",
0,
1,
iotype="smallfile",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
1,
iotype="smallfile",
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"g",
"readwrite",
)
p.spawn(fs_util.read_write_IO, client3,
client_info["mounting_dir"])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
if result == "Data validation success":
print("Data validation success")
fs_util.activate_multiple_mdss(client_info["mds_nodes"])
log.info("Execution of Test case CEPH-%s started:" % (tc))
for client in client1:
client.exec_command(cmd="sudo mkdir %s%s_{1..50}" %
(client_info["mounting_dir"], dir_name))
if client.node.exit_status == 0:
log.info("directories created succcessfully")
else:
raise CommandFailed("directories creation failed")
with parallel() as p:
p.spawn(
fs_util.pinning,
client1,
1,
25,
client_info["mounting_dir"],
dir_name,
0,
)
p.spawn(
fs_util.pinning,
client3,
26,
50,
client_info["mounting_dir"],
dir_name,
1,
)
with parallel() as p:
p.spawn(
fs_util.max_dir_io,
client1,
client_info["mounting_dir"],
dir_name,
1,
25,
1000,
)
p.spawn(
fs_util.max_dir_io,
client3,
client_info["mounting_dir"],
dir_name,
26,
50,
1000,
)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
p.spawn(fs_util.reboot, node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
fs_util.network_disconnect(node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
fs_util.pid_kill(node, "mon")
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["osd_nodes"]:
p.spawn(fs_util.reboot, node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
),
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["osd_nodes"]:
fs_util.network_disconnect(node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["osd_nodes"]:
fs_util.pid_kill(node, "osd")
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
),
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["osd_nodes"]:
fs_util.network_disconnect(node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
p.spawn(fs_util.daemon_systemctl, node, "mon", "restart")
log.info("Execution of Test case CEPH-%s ended:" % (tc))
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc_client = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc_client = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
rc_mds = fs_util.mds_cleanup(client_info["mds_nodes"], None)
if rc_client == 0 and rc_mds == 0:
log.info("Cleaning up successfull")
else:
return 1
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
if rc == 0:
log.info("Cleaning up successfull")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11255_11336-fuse client'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
config = kw.get('config')
num_of_osds = config.get('num_of_osds')
fs_util = FsUtils(ceph_cluster)
build = config.get('build', config.get('rhbuild'))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("creating auth failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(
client3,
client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(
client4,
client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
cluster_health_beforeIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds, len(
client_info['mon_node']), build, None, 300)
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
rc = fs_util.standby_rank(
client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank')
if rc == 0:
log.info("Added standby ranks")
else:
log.error("Failed to add standby ranks")
return 1
client1[0].exec_command(
cmd='sudo mkdir %s%s' %
(client_info['mounting_dir'], dir_name))
if client1[0].node.exit_status == 0:
log.info("Dir created")
else:
raise CommandFailed('Dir creation failed')
rc1 = fs_util.fstab_entry(
client1,
client_info['mounting_dir'],
action='doEntry')
rc2 = fs_util.fstab_entry(
client2,
client_info['mounting_dir'],
action='doEntry')
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
raise CommandFailed("FsEntry failed")
rc1 = fs_util.fstab_entry(
client3,
client_info['mounting_dir'],
action='doEntry', mon_node_ip=client_info['mon_node_ip'])
rc2 = fs_util.fstab_entry(
client4,
client_info['mounting_dir'],
action='doEntry', mon_node_ip=client_info['mon_node_ip'])
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
raise CommandFailed("FsEntry failed")
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client3,
client_info['mounting_dir'], 'g', 'read')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir_name,
0,
50,
iotype='fio')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
1,
iotype='smallfile_create', fnum=1000, fsize=100)
p.spawn(
fs_util.stress_io,
client4,
client_info['mounting_dir'],
dir_name,
0,
1,
iotype='crefi')
p.spawn(fs_util.reboot, client1[0])
res = []
with parallel() as p:
for node in client_info['mds_nodes']:
p.spawn(fs_util.heartbeat_map, node)
for op in p:
res.append(op)
print(res)
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client4,
client_info['mounting_dir'], 'g', 'read')
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir_name,
0,
1,
iotype='fio')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='dd')
p.spawn(
fs_util.stress_io,
client4,
client_info['mounting_dir'],
dir_name,
0,
500,
iotype='touch'),
p.spawn(fs_util.reboot, client2[0])
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds, len(
client_info['mon_node']), build, None, 300)
if cluster_health_afterIO == cluster_health_beforeIO:
log.info('cluster is healthy')
else:
log.error("cluster is not healty")
return 1
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='fio')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='dd')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
500,
iotype='touch')
for node in client_info['osd_nodes']:
p.spawn(fs_util.reboot, node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='fio'),
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='dd')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
500,
iotype='touch')
for node in client_info['osd_nodes']:
fs_util.network_disconnect(node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='fio')
p.spawn(
fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir_name,
0,
10,
iotype='dd')
p.spawn(
fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
500,
iotype='touch')
for node in client_info['osd_nodes']:
fs_util.pid_kill(node, 'osd')
time.sleep(100)
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds, len(
client_info['mon_node']), build, None, 300)
if cluster_health_beforeIO == cluster_health_afterIO:
log.info("Cluster is healthy")
else:
return 1
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(
client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
rc = fs_util.standby_rank(
client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
if rc == 0:
log.info("removed standby ranks")
rc1 = fs_util.fstab_entry(
client1,
client_info['mounting_dir'],
action='revertEntry')
rc2 = fs_util.fstab_entry(
client2,
client_info['mounting_dir'],
action='revertEntry')
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
return 1
else:
rc = fs_util.client_clean_up(
client_info['fuse_clients'],
'',
client_info['mounting_dir'],
'umount')
rc = fs_util.standby_rank(
client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
if rc == 0:
log.info("removed standby ranks")
rc1 = fs_util.fstab_entry(
client1,
client_info['mounting_dir'],
action='revertEntry')
rc2 = fs_util.fstab_entry(
client2,
client_info['mounting_dir'],
action='revertEntry')
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
if rc == 0:
log.info('Cleaning up successfull')
log.info("Execution of Test cases CEPH-%s ended:" % (tc))
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(
client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
else:
rc = fs_util.client_clean_up(
client_info['fuse_clients'],
'',
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11219,11224'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
config = kw.get('config')
num_of_osds = config.get('num_of_osds')
fs_util = FsUtils(ceph_cluster)
build = config.get('build', config.get('rhbuild'))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
cluster_health_beforeIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), build, None, 300)
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
log.error("Activate multiple mdss failed")
return 1
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client2,
client_info['mounting_dir'], 'g', 'read')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
2,
iotype='crefi')
p.spawn(fs_util.read_write_IO, client4,
client_info['mounting_dir'], 'g', 'readwrite')
p.spawn(fs_util.read_write_IO, client3,
client_info['mounting_dir'])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify('', return_counts)
if result == 'Data validation success':
print("Data validation success")
dirs, rc = fs_util.mkdir(client1, 0, 3,
client_info['mounting_dir'], dir_name)
if rc == 0:
log.info("Directories created")
else:
raise CommandFailed("Directory creation failed")
dirs = dirs.split('\n')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
100,
iotype='touch')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='dd')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='crefi')
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_delete',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='smallfile_delete')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_delete',
fnum=10,
fsize=1024)
for op in p:
return_counts, rc = op
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), build, None, 300)
log.info("Execution of Test case CEPH-%s ended" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
if cluster_health_beforeIO == cluster_health_afterIO:
print(result)
print('-----------------------------------------')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_create',
fnum=1000,
fsize=10)
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
5,
iotype='fio')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[0],
0,
10,
iotype='dd')
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
print('-------------------------------------------------------')
with parallel() as p:
p.spawn(fs_util.read_write_IO,
client1,
client_info['mounting_dir'],
'g',
'read',
dir_name=dirs[0])
p.spawn(fs_util.read_write_IO,
client2,
client_info['mounting_dir'],
'g',
'read',
dir_name=dirs[0])
print('-------------------------------------------------------')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='smallfile_create',
fnum=1000,
fsize=10)
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
5,
iotype='fio')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[0],
0,
10,
iotype='dd')
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
print('-------------------------------------------------------')
with parallel() as p:
p.spawn(fs_util.read_write_IO,
client1,
client_info['mounting_dir'],
'g',
'read',
dir_name=dirs[0])
p.spawn(fs_util.read_write_IO,
client2,
client_info['mounting_dir'],
'g',
'read',
dir_name=dirs[0])
print('-------------------------------------------------------')
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
else:
return 1
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
else:
return 1
log.info("Execution of Test case CEPH-%s ended" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
print(result)
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11222"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get("config")
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
cluster_health_beforeIO = check_ceph_healthly(client_info["mon_node"],
12, 1, build, None, 300)
dir1 = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
for client in client_info["clients"]:
log.info("Creating directory:")
client.exec_command(cmd="sudo mkdir %s%s" %
(client_info["mounting_dir"], dir1))
log.info("Creating directories with breadth and depth:")
out, rc = client.exec_command(
cmd="sudo crefi %s%s --fop create --multi -b 10 -d 10 "
"--random --min=1K --max=10K" %
(client_info["mounting_dir"], dir1))
print(out.read().decode())
break
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="crefi",
)
for op in p:
return_counts, rc = op
result1 = fs_util.rc_verify("", return_counts)
print(result1)
for client in client_info["clients"]:
client.exec_command(cmd="sudo rm -rf %s%s" %
(client_info["mounting_dir"], dir1))
break
for client in client_info["clients"]:
log.info("Creating directories with breadth and depth:")
out, rc = client.exec_command(
cmd="sudo crefi %s%s --fop create --multi -b 10 -d 10 "
"--random --min=1K --max=10K" %
(client_info["mounting_dir"], dir1))
print(out.read().decode())
log.info("Renaming the dirs:")
out, rc = client.exec_command(
cmd="sudo crefi "
"%s%s --fop rename --multi -b 10 -d 10 --random "
"--min=1K --max=10K" % (client_info["mounting_dir"], dir1))
print(out.read().decode())
break
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir1,
0,
100,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir1,
0,
5,
iotype="crefi",
)
for op in p:
return_counts, rc = op
result2 = fs_util.rc_verify("", return_counts)
print(result2)
cluster_health_afterIO = check_ceph_healthly(client_info["mon_node"],
12, 1, build, None, 300)
if cluster_health_beforeIO == cluster_health_afterIO:
print("Testcase %s passed" % (tc))
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
if rc == 0:
log.info("Cleaning up successfull")
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11335"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get("config")
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
active_mds_node_1, active_mds_node_2, rc = fs_util.get_active_mdss(
client_info["mds_nodes"])
if rc == 0:
log.info("Got active mdss")
else:
raise CommandFailed("getting active-mdss failed")
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"m",
"write",
)
p.spawn(fs_util.read_write_IO, client2,
client_info["mounting_dir"], "m", "read")
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"m",
"readwrite",
)
p.spawn(
fs_util.read_write_IO,
client3,
client_info["mounting_dir"],
"m",
"readwrite",
)
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
print(result)
log.info("Performing Auto Eviction:")
mds1_before_evict, _, rc = fs_util.get_mds_info(active_mds_node_1,
active_mds_node_2,
info="session ls")
rc = fs_util.auto_evict(active_mds_node_1, client_info["clients"], 0)
if rc == 0:
log.info("client process killed successfully for auto eviction")
else:
raise CommandFailed(
"client process killing failed for auto eviction")
log.info("Waiting 300 seconds for auto eviction---")
time.sleep(300)
mds1_after_evict, _, rc = fs_util.get_mds_info(active_mds_node_1,
active_mds_node_2,
info="session ls")
if mds1_before_evict != mds1_after_evict:
log.info("Auto eviction Passed")
else:
raise CommandFailed("Auto eviction Failed")
print("-------------------------------------------------------")
if client3[0].pkg_type == "deb" and client4[0].pkg_type == "deb":
for client in client_info["fuse_clients"]:
client.exec_command(cmd="sudo fusermount -u %s -z" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
else:
for client in client_info["fuse_clients"]:
client.exec_command(cmd="sudo fusermount -u %s -z" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
for client in client_info["kernel_clients"]:
client.exec_command(cmd="sudo umount %s -l" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"m",
"write",
)
p.spawn(fs_util.read_write_IO, client2,
client_info["mounting_dir"], "m", "read")
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
"",
0,
1,
iotype="smallfile",
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"m",
"readwrite",
)
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
print(result)
log.info("Performing Manual eviction:")
ip_addr = fs_util.manual_evict(active_mds_node_1, 0)
mds1_after_evict, _, rc = fs_util.get_mds_info(active_mds_node_1,
active_mds_node_2,
info="session ls")
print(mds1_before_evict)
print("------------------------")
print(mds1_after_evict)
print("-----------------------")
if mds1_before_evict != mds1_after_evict:
log.info("Manual eviction success")
else:
raise CommandFailed("Manual Eviction Failed")
log.info("Removing client from OSD blacklisting:")
rc = fs_util.osd_blacklist(active_mds_node_1, ip_addr)
if rc == 0:
log.info("Removing client from OSD blacklisting successfull")
else:
raise CommandFailed("Removing client from OSD blacklisting Failed")
print("-" * 10)
if client3[0].pkg_type == "deb" and client4[0].pkg_type == "deb":
for client in client_info["fuse_clients"]:
client.exec_command(cmd="sudo fusermount -u %s -z" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
else:
for client in client_info["fuse_clients"]:
client.exec_command(cmd="sudo fusermount -u %s -z" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
for client in client_info["kernel_clients"]:
client.exec_command(cmd="sudo umount %s -l" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"m",
"write",
)
p.spawn(fs_util.read_write_IO, client2,
client_info["mounting_dir"], "m", "read")
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
"",
0,
1,
iotype="smallfile",
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"m",
"readwrite",
)
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
print(result)
log.info("Performing configuring blacklisting:")
rc = fs_util.config_blacklist_manual_evict(active_mds_node_1, 0)
if rc == 0:
log.info("Configure blacklisting for manual evict success")
rc = fs_util.config_blacklist_manual_evict(active_mds_node_1,
0,
revert=True)
else:
raise CommandFailed(
"Configure blacklisting for manual evict failed")
print("-" * 10)
rc = fs_util.config_blacklist_auto_evict(active_mds_node_1, 0)
if rc == 0:
log.info("Configure blacklisting for auto evict success")
rc = fs_util.config_blacklist_auto_evict(active_mds_node_1,
0,
revert=True)
if rc == 0:
log.info("Reverted successfully")
else:
raise CommandFailed(
"Configure blacklisting for auto evict failed")
else:
raise CommandFailed("Configure blacklisting for auto evict failed")
if client3[0].pkg_type == "deb" and client4[0].pkg_type == "deb":
for client in client_info["fuse_clients"]:
client.exec_command(cmd="sudo fusermount -u %s -z" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
else:
for client in client_info["fuse_clients"]:
client.exec_command(cmd="sudo fusermount -u %s -z" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
for client in client_info["kernel_clients"]:
client.exec_command(cmd="sudo umount %s -l" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
if client3[0].pkg_type == "deb" and client4[0].pkg_type == "deb":
for client in client_info["fuse_clients"]:
client.exec_command(cmd="sudo rm -rf %s*" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo fusermount -u %s -z" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
else:
for client in client_info["fuse_clients"]:
client.exec_command(cmd="sudo rm -rf %s*" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo fusermount -u %s -z" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
for client in client_info["kernel_clients"]:
client.exec_command(cmd="sudo umount %s -l" %
(client_info["mounting_dir"]))
client.exec_command(cmd="sudo rm -rf %s" %
(client_info["mounting_dir"]))
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11254-fuse_clients"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
config = kw.get("config")
num_of_osds = config.get("num_of_osds")
fs_util = FsUtils(ceph_cluster)
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(
client3, client_info["mounting_dir"], client_info["mon_node_ip"]
)
rc4 = fs_util.kernel_mount(
client4, client_info["mounting_dir"], client_info["mon_node_ip"]
)
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
cluster_health_beforeIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
rc = fs_util.standby_rank(
client_info["mds_nodes"], client_info["mon_node"], todo="add_rank"
)
if rc == 0:
log.info("Added standby ranks")
else:
raise Exception("Adding standby ranks failed")
client1[0].exec_command(
cmd="sudo mkdir %s%s" % (client_info["mounting_dir"], dir_name)
)
if client1[0].node.exit_status == 0:
log.info("Dir created")
else:
fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
)
rc1 = fs_util.fstab_entry(
client1, client_info["mounting_dir"], action="doEntry"
)
rc2 = fs_util.fstab_entry(
client2, client_info["mounting_dir"], action="doEntry"
)
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
raise CommandFailed("FsEntry failed")
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(
fs_util.read_write_IO, client3, client_info["mounting_dir"], "g", "read"
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
50,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=100,
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dir_name,
0,
1,
iotype="smallfile",
)
p.spawn(fs_util.reboot, client1[0])
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(
fs_util.read_write_IO, client4, client_info["mounting_dir"], "g", "read"
)
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
1,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
p.spawn(fs_util.reboot, client2[0])
cluster_health_afterIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
if cluster_health_afterIO == cluster_health_beforeIO:
log.info("cluster is healthy")
else:
log.error("cluster is not healty")
return 1
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
p.spawn(fs_util.reboot, node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
fs_util.network_disconnect(node)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="fio",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
0,
10,
iotype="dd",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
500,
iotype="touch",
)
for node in client_info["mon_node"]:
fs_util.pid_kill(node, "mon")
cluster_health_afterIO = check_ceph_healthly(
client_info["mon_node"][0],
num_of_osds,
len(client_info["mon_node"]),
build,
None,
300,
)
if cluster_health_beforeIO == cluster_health_afterIO:
log.info("Cluster is healthy")
else:
return 1
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
rc = fs_util.standby_rank(
client_info["mds_nodes"],
client_info["mon_node"],
todo="add_rank_revert",
)
if rc == 0:
log.info("removed standby ranks")
rc1 = fs_util.fstab_entry(
client1, client_info["mounting_dir"], action="revertEntry"
)
rc2 = fs_util.fstab_entry(
client2, client_info["mounting_dir"], action="revertEntry"
)
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
return 1
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"], "", client_info["mounting_dir"], "umount"
)
rc = fs_util.standby_rank(
client_info["mds_nodes"],
client_info["mon_node"],
todo="add_rank_revert",
)
if rc == 0:
log.info("removed standby ranks")
else:
return 1
rc1 = fs_util.fstab_entry(
client1, client_info["mounting_dir"], action="revertEntry"
)
rc2 = fs_util.fstab_entry(
client2, client_info["mounting_dir"], action="revertEntry"
)
if rc1 == 0 and rc2 == 0:
log.info("FSentry for clients are done")
else:
return 1
if rc == 0:
log.info("Cleaning up successfull")
log.info("Execution of Test cases CEPH-%s ended:" % (tc))
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(
client_info["fuse_clients"], "", client_info["mounting_dir"], "umount"
)
if rc == 0:
log.info("Cleaning up successfull")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11256-fuse'
dir_name = 'dir'
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get('config')
num_of_osds = config.get('num_of_osds')
client_info, rc = fs_util.get_clients()
if rc == 0:
log.info("Got client info")
else:
log.error("fetching client info failed")
return 1
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise Exception("kernel mount failed")
cluster_health_beforeIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), None, 300)
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
rc = fs_util.standby_rank(client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank')
if rc == 0:
log.info("Added standby ranks")
else:
raise Exception("Adding standby ranks failed")
dirs, rc = fs_util.mkdir(client1, 0, 4, client_info['mounting_dir'],
dir_name)
if rc == 0:
log.info("Directories created")
else:
raise CommandFailed("Directory creation failed")
dirs = dirs.split('\n')
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
'',
'',
iotype='smallfile_create',
fnum=1000,
fsize=1024)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[3],
0,
10,
iotype='dd')
for node in client_info['mds_nodes']:
p.spawn(fs_util.reboot, node)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[1],
'',
'',
iotype='smallfile_create',
fnum=1000,
fsize=1024)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[3],
0,
10,
iotype='dd')
for node in client_info['mds_nodes']:
p.spawn(fs_util.daemon_systemctl, node, 'mds',
'active_mds_restart')
with parallel() as p:
for node in client_info['mds_nodes']:
p.spawn(fs_util.heartbeat_map, node)
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), None, 300)
if cluster_health_afterIO == cluster_health_beforeIO:
log.info('cluster is healthy')
else:
log.error("cluster is not healty")
return 1
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[3],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='crefi')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='dd')
for node in client_info['mds_nodes']:
fs_util.network_disconnect(node)
with parallel() as p:
for node in client_info['mds_nodes']:
p.spawn(fs_util.heartbeat_map, node)
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dirs[0],
0,
1,
iotype='fio')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dirs[1],
0,
1,
iotype='crefi')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dirs[3],
0,
1,
iotype='smallfile_create',
fnum=10,
fsize=1024)
p.spawn(fs_util.stress_io,
client4,
client_info['mounting_dir'],
dirs[2],
0,
1,
iotype='dd')
for node in client_info['mds_nodes']:
fs_util.pid_kill(node, 'mds')
with parallel() as p:
for node in client_info['mds_nodes']:
p.spawn(fs_util.heartbeat_map, node)
cluster_health_afterIO = check_ceph_healthly(
client_info['mon_node'][0], num_of_osds,
len(client_info['mon_node']), None, 300)
if cluster_health_beforeIO == cluster_health_afterIO:
log.info("Cluster is healthy")
else:
return 1
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
if rc == 0:
log.info('Cleaning up successfull')
fs_util.standby_rank(client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
log.info("Execution of Test cases CEPH-%s ended:" % (tc))
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
rc = fs_util.standby_rank(client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
rc = fs_util.standby_rank(client_info['mds_nodes'],
client_info['mon_node'],
todo='add_rank_revert')
if rc == 0:
log.info('Cleaning up successfull')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "11232 and 11233"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get("config")
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(fs_util.read_write_IO, client3,
client_info["mounting_dir"], "g", "read")
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
"",
0,
2,
iotype="crefi",
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"g",
"readwrite",
)
p.spawn(fs_util.read_write_IO, client3,
client_info["mounting_dir"])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
if "Data validation success" in result:
print("Data validation success")
tc = "11232 and 11233"
log.info("Execution of Test cases %s started:" % (tc))
fs_util.allow_dir_fragmentation(client_info["mds_nodes"])
log.info("Creating directory:")
for node in client_info["fuse_clients"]:
out, rc = node.exec_command(
cmd="sudo mkdir %s%s" %
(client_info["mounting_dir"], dir_name))
print(out.read().decode())
break
active_mds_node_1, active_mds_node_2, rc = fs_util.get_active_mdss(
client_info["mds_nodes"])
if rc == 0:
log.info("Got active mdss")
else:
raise CommandFailed("getting active-mdss failed")
node1_before_io, _, rc = fs_util.get_mds_info(active_mds_node_1,
active_mds_node_2,
info="get subtrees")
if rc == 0:
log.info("Got mds subtree info")
else:
raise CommandFailed("Mds info command failed")
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dir_name,
0,
1000,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
1000,
2000,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dir_name,
2000,
3000,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dir_name,
3000,
4000,
iotype="touch",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
4000,
5000,
iotype="touch",
)
node1_after_io, _, rc = fs_util.get_mds_info(active_mds_node_1,
active_mds_node_2,
info="get subtrees")
if rc == 0:
log.info("Got mds subtree info")
else:
raise CommandFailed("Mds info command failed")
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
)
if rc == 0:
log.info("Cleaning mount success")
else:
raise CommandFailed("Cleaning mount failed")
node1_after_del, _, rc = fs_util.get_mds_info(active_mds_node_1,
active_mds_node_2,
info="get subtrees")
if rc == 0:
log.info("Got mds subtree info")
else:
raise CommandFailed("Mds info command failed")
log.info("Execution of Test case 11232 and 11233 ended:")
print("Results:")
if node1_before_io != node1_after_io and node1_after_io != node1_after_del:
log.info("Test case %s Passed" % (tc))
else:
return 1
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc_client = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
rc_mds = fs_util.mds_cleanup(client_info["mds_nodes"], None)
else:
rc_client = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
rc_mds = fs_util.mds_cleanup(client_info["mds_nodes"], None)
if rc_client == 0 and rc_mds == 0:
log.info("Cleaning up successfull")
else:
return 1
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
if rc == 0:
log.info("Cleaning up successfull")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = "10625,11225"
dir_name = "dir"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
config = kw.get("config")
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
log.error("auth list failed")
return 1
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
log.error("Fuse mount failed")
return 1
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
log.error("kernel mount failed")
return 1
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
log.error("Activate multiple mdss failed")
return 1
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(fs_util.read_write_IO, client2,
client_info["mounting_dir"], "g", "read")
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
"",
0,
1,
iotype="smallfile",
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"g",
"readwrite",
)
p.spawn(fs_util.read_write_IO, client3,
client_info["mounting_dir"])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
if result == "Data validation success":
dirs, rc = fs_util.mkdir(client1, 0, 6,
client_info["mounting_dir"], dir_name)
if rc == 0:
log.info("Directories created")
else:
raise CommandFailed("Directory creation failed")
dirs = dirs.split("\n")
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=10,
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[1],
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=10,
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=10,
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile_create",
fnum=1000,
fsize=10,
)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_rename",
fnum=1000,
fsize=10,
)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[0],
0,
1,
iotype="smallfile_delete-renamed",
fnum=1000,
fsize=10,
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dirs[2],
0,
1,
iotype="smallfile_delete",
fnum=1000,
fsize=10,
)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[3],
0,
1,
iotype="smallfile_create",
fnum=1,
fsize=1000000,
)
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
dirs[4],
0,
1,
iotype="smallfile_create",
fnum=1,
fsize=1000000,
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dirs[5],
0,
1,
iotype="smallfile_create",
fnum=1,
fsize=1000000,
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dirs[6],
0,
1,
iotype="smallfile_create",
fnum=1,
fsize=1000000,
)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[3],
0,
1,
iotype="smallfile_rename",
fnum=1,
fsize=1000000,
)
with parallel() as p:
p.spawn(
fs_util.stress_io,
client1,
client_info["mounting_dir"],
dirs[3],
0,
1,
iotype="smallfile_delete-renamed",
fnum=1,
fsize=1000000,
)
p.spawn(
fs_util.stress_io,
client4,
client_info["mounting_dir"],
dirs[4],
0,
1,
iotype="smallfile_delete",
fnum=1,
fsize=1000000,
)
dir_name = "!@#$%^&*()-_=+[]{};:,.<>?"
out, rc = client1[0].exec_command(
cmd="sudo mkdir '%s%s'" % (client_info["mounting_dir"], dir_name))
if client1[0].node.exit_status == 0:
log.info("Directory created")
else:
raise CommandFailed("Directory creation failed")
for client in client_info["fuse_clients"]:
file_name = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(255))
client.exec_command(
cmd="sudo touch '%s%s/%s'" %
(client_info["mounting_dir"], dir_name, file_name))
for client in client_info["kernel_clients"]:
if client.pkg_type == "rpm":
file_name = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(255))
client.exec_command(
cmd="sudo touch '%s%s/%s'" %
(client_info["mounting_dir"], dir_name, file_name))
for num in range(0, 5):
for client in client_info["fuse_clients"]:
ops = ["create", "setxattr", "getxattr", "chmod", "rename"]
for op in ops:
client.exec_command(
sudo=True,
cmd=
f"python3 smallfile/smallfile_cli.py --operation {op} --threads 10 --file-size 4 "
f"--files 1000 --files-per-dir 10 --dirs-per-dir 2 --top "
f"{client_info['mounting_dir']}{dir_name}",
long_running=True,
timeout=300,
)
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
if rc == 0:
log.info("Cleaning up successfull")
else:
return 1
log.info("Execution of Test cases CEPH-%s ended:" % (tc))
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
else:
rc = fs_util.client_clean_up(client_info["fuse_clients"], "",
client_info["mounting_dir"], "umount")
if rc == 0:
log.info("Cleaning up successfull")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11222'
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
client_info, rc = fs_util.get_clients()
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
cluster_health_beforeIO = check_ceph_healthly(client_info['mon_node'],
12, 1, None, 300)
dir1 = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
for client in client_info['clients']:
log.info("Creating directory:")
client.exec_command(cmd='sudo mkdir %s%s' %
(client_info['mounting_dir'], dir1))
log.info("Creating directories with breadth and depth:")
out, rc = client.exec_command(
cmd='sudo crefi %s%s --fop create --multi -b 10 -d 10 '
'--random --min=1K --max=10K' %
(client_info['mounting_dir'], dir1))
print(out.read().decode())
break
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='fio')
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir1,
0,
100,
iotype='touch')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='dd')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='crefi')
for op in p:
return_counts, rc = op
result1 = fs_util.rc_verify('', return_counts)
print(result1)
for client in client_info['clients']:
client.exec_command(cmd='sudo rm -rf %s%s' %
(client_info['mounting_dir'], dir1))
break
for client in client_info['clients']:
log.info("Creating directories with breadth and depth:")
out, rc = client.exec_command(
cmd='sudo crefi %s%s --fop create --multi -b 10 -d 10 '
'--random --min=1K --max=10K' %
(client_info['mounting_dir'], dir1))
print(out.read().decode())
log.info("Renaming the dirs:")
out, rc = client.exec_command(
cmd='sudo crefi '
'%s%s --fop rename --multi -b 10 -d 10 --random '
'--min=1K --max=10K' % (client_info['mounting_dir'], dir1))
print(out.read().decode())
break
with parallel() as p:
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='fio')
p.spawn(fs_util.stress_io,
client1,
client_info['mounting_dir'],
dir1,
0,
100,
iotype='touch')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='dd')
p.spawn(fs_util.stress_io,
client2,
client_info['mounting_dir'],
dir1,
0,
5,
iotype='crefi')
for op in p:
return_counts, rc = op
result2 = fs_util.rc_verify('', return_counts)
print(result2)
cluster_health_afterIO = check_ceph_healthly(client_info['mon_node'],
12, 1, None, 300)
if cluster_health_beforeIO == cluster_health_afterIO:
print("Testcase %s passed" % (tc))
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
config = kw.get("config")
num_of_dirs = config.get("num_of_dirs")
num_of_dirs = num_of_dirs / 5
tc = "11228"
dir_name = "dir"
test_dir = "testdir/"
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
build = config.get("build", config.get("rhbuild"))
client_info, rc = fs_util.get_clients(build)
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1 = []
client2 = []
client3 = []
client4 = []
client1.append(client_info["fuse_clients"][0])
client2.append(client_info["fuse_clients"][1])
client3.append(client_info["kernel_clients"][0])
client4.append(client_info["kernel_clients"][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info["mounting_dir"])
rc2 = fs_util.fuse_mount(client2, client_info["mounting_dir"])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info["mounting_dir"],
client_info["mon_node_ip"])
rc4 = fs_util.kernel_mount(client4, client_info["mounting_dir"],
client_info["mon_node_ip"])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info["mds_nodes"])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
with parallel() as p:
p.spawn(
fs_util.read_write_IO,
client1,
client_info["mounting_dir"],
"g",
"write",
)
p.spawn(fs_util.read_write_IO, client2,
client_info["mounting_dir"], "g", "read")
p.spawn(
fs_util.stress_io,
client2,
client_info["mounting_dir"],
"",
0,
1,
iotype="crefi",
)
p.spawn(
fs_util.stress_io,
client3,
client_info["mounting_dir"],
dir_name,
0,
1,
iotype="crefi",
)
p.spawn(
fs_util.read_write_IO,
client4,
client_info["mounting_dir"],
"g",
"readwrite",
)
p.spawn(fs_util.read_write_IO, client3,
client_info["mounting_dir"])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify("", return_counts)
if result == "Data validation success":
print("Data validation success")
fs_util.activate_multiple_mdss(client_info["mds_nodes"])
log.info("Execution of Test case CEPH-%s started:" % (tc))
for client in client1:
client.exec_command(cmd="sudo mkdir %s%s" %
(client_info["mounting_dir"], test_dir))
with parallel() as p:
p.spawn(
fs_util.mkdir_bulk,
client1,
0,
num_of_dirs * 2,
client_info["mounting_dir"] + test_dir,
dir_name,
)
p.spawn(
fs_util.mkdir_bulk,
client2,
num_of_dirs * 2 + 1,
num_of_dirs * 4,
client_info["mounting_dir"] + test_dir,
dir_name,
)
p.spawn(
fs_util.mkdir_bulk,
client2,
num_of_dirs * 4 + 1,
num_of_dirs * 6,
client_info["mounting_dir"] + test_dir,
dir_name,
)
p.spawn(
fs_util.mkdir_bulk,
client2,
num_of_dirs * 6 + 1,
num_of_dirs * 8,
client_info["mounting_dir"] + test_dir,
dir_name,
)
p.spawn(
fs_util.mkdir_bulk,
client2,
num_of_dirs * 8 + 1,
num_of_dirs * 10,
client_info["mounting_dir"] + test_dir,
dir_name,
)
for op in p:
rc = op
if rc == 0:
log.info("Directories created successfully")
else:
raise CommandFailed("Directory creation failed")
with parallel() as p:
p.spawn(
fs_util.pinning,
client2,
0,
num_of_dirs * 1,
client_info["mounting_dir"] + test_dir,
dir_name,
1,
)
p.spawn(
fs_util.pinning,
client3,
num_of_dirs * 1,
num_of_dirs * 2,
client_info["mounting_dir"] + test_dir,
dir_name,
1,
)
p.spawn(
fs_util.pinning,
client4,
num_of_dirs * 2,
num_of_dirs * 3,
client_info["mounting_dir"] + test_dir,
dir_name,
1,
)
p.spawn(
fs_util.pinning,
client1,
num_of_dirs * 3,
num_of_dirs * 4,
client_info["mounting_dir"] + test_dir,
dir_name,
0,
)
p.spawn(
fs_util.pinning,
client3,
num_of_dirs * 4,
num_of_dirs * 5,
client_info["mounting_dir"] + test_dir,
dir_name,
0,
)
with parallel() as p:
p.spawn(
fs_util.pinned_dir_io_mdsfailover,
client1,
client_info["mounting_dir"] + test_dir,
dir_name,
num_of_dirs * 1,
num_of_dirs * 5,
10,
fs_util.mds_fail_over,
client_info["mds_nodes"],
)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(
fs_util.pinned_dir_io_mdsfailover,
client3,
client_info["mounting_dir"] + test_dir,
dir_name,
num_of_dirs * 7,
num_of_dirs * 8,
20,
fs_util.mds_fail_over,
client_info["mds_nodes"],
)
for op in p:
return_counts, rc = op
log.info("Execution of Test case CEPH-%s ended:" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
print(result)
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc_client = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
rc_mds = fs_util.mds_cleanup(client_info["mds_nodes"], None)
else:
rc_client = fs_util.client_clean_up(
client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount",
)
rc_mds = fs_util.mds_cleanup(client_info["mds_nodes"], None)
if rc_client == 0 and rc_mds == 0:
log.info("Cleaning up successfull")
else:
return 1
print("Script execution time:------")
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info("Cleaning up!-----")
if client3[0].pkg_type != "deb" and client4[0].pkg_type != "deb":
rc_client = fs_util.client_clean_up(
client_info["fuse_clients"],
client_info["kernel_clients"],
client_info["mounting_dir"],
"umount",
)
rc_mds = fs_util.mds_cleanup(client_info["mds_nodes"], None)
else:
rc_client = fs_util.client_clean_up(client_info["fuse_clients"],
"",
client_info["mounting_dir"],
"umount")
rc_mds = fs_util.mds_cleanup(client_info["mds_nodes"], None)
if rc_client == 0 and rc_mds == 0:
log.info("Cleaning up successfull")
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
def run(ceph_cluster, **kw):
try:
start = timeit.default_timer()
tc = '11242'
dir_name = 'dir'
config = kw.get('config')
num_of_dirs = config.get('num_of_dirs')
num_of_dirs = num_of_dirs / 5
log.info("Running cephfs %s test case" % (tc))
fs_util = FsUtils(ceph_cluster)
client_info, rc = fs_util.get_clients()
if rc == 0:
log.info("Got client info")
else:
raise CommandFailed("fetching client info failed")
client1, client2, client3, client4 = ([] for _ in range(4))
client1.append(client_info['fuse_clients'][0])
client2.append(client_info['fuse_clients'][1])
client3.append(client_info['kernel_clients'][0])
client4.append(client_info['kernel_clients'][1])
rc1 = fs_util.auth_list(client1)
rc2 = fs_util.auth_list(client2)
rc3 = fs_util.auth_list(client3)
rc4 = fs_util.auth_list(client4)
print(rc1, rc2, rc3, rc4)
if rc1 == 0 and rc2 == 0 and rc3 == 0 and rc4 == 0:
log.info("got auth keys")
else:
raise CommandFailed("auth list failed")
rc1 = fs_util.fuse_mount(client1, client_info['mounting_dir'])
rc2 = fs_util.fuse_mount(client2, client_info['mounting_dir'])
if rc1 == 0 and rc2 == 0:
log.info("Fuse mount passed")
else:
raise CommandFailed("Fuse mount failed")
rc3 = fs_util.kernel_mount(client3, client_info['mounting_dir'],
client_info['mon_node_ip'])
rc4 = fs_util.kernel_mount(client4, client_info['mounting_dir'],
client_info['mon_node_ip'])
if rc3 == 0 and rc4 == 0:
log.info("kernel mount passed")
else:
raise CommandFailed("kernel mount failed")
rc = fs_util.activate_multiple_mdss(client_info['mds_nodes'])
if rc == 0:
log.info("Activate multiple mdss successfully")
else:
raise CommandFailed("Activate multiple mdss failed")
with parallel() as p:
p.spawn(fs_util.read_write_IO, client1,
client_info['mounting_dir'], 'g', 'write')
p.spawn(fs_util.read_write_IO, client2,
client_info['mounting_dir'], 'g', 'read')
p.spawn(fs_util.stress_io,
client3,
client_info['mounting_dir'],
dir_name,
0,
1,
iotype='crefi')
p.spawn(fs_util.read_write_IO, client4,
client_info['mounting_dir'], 'g', 'readwrite')
p.spawn(fs_util.read_write_IO, client3,
client_info['mounting_dir'])
for op in p:
return_counts, rc = op
result = fs_util.rc_verify('', return_counts)
if result == 'Data validation success':
print("Data validation success")
log.info("Execution of Test cases %s started:" % (tc))
with parallel() as p:
p.spawn(fs_util.mkdir, client1, 0, num_of_dirs * 1,
client_info['mounting_dir'], dir_name)
p.spawn(fs_util.mkdir, client2, num_of_dirs * 1,
num_of_dirs * 2, client_info['mounting_dir'], dir_name)
p.spawn(fs_util.mkdir, client1, num_of_dirs * 2,
num_of_dirs * 3, client_info['mounting_dir'], dir_name)
p.spawn(fs_util.mkdir, client2, num_of_dirs * 3,
num_of_dirs * 4, client_info['mounting_dir'], dir_name)
p.spawn(fs_util.mkdir, client1, num_of_dirs * 4,
num_of_dirs * 5, client_info['mounting_dir'], dir_name)
for op in p:
_, rc = op
if rc == 0:
log.info("Dirs created successfully")
else:
raise CommandFailed("Dirs creation failed")
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client1,
client_info['mounting_dir'], dir_name, 0,
num_of_dirs * 1, 1, fs_util.mds_fail_over,
client_info['mds_nodes'])
p.spawn(fs_util.filesystem_utilities, client2,
client_info['mounting_dir'], dir_name, 0,
num_of_dirs * 1)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client3,
client_info['mounting_dir'], dir_name, num_of_dirs * 1,
num_of_dirs * 2, 1, fs_util.mds_fail_over,
client_info['mds_nodes'])
p.spawn(fs_util.filesystem_utilities, client4,
client_info['mounting_dir'], dir_name, num_of_dirs * 1,
num_of_dirs * 2)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client1,
client_info['mounting_dir'], dir_name, num_of_dirs * 2,
num_of_dirs * 3, 1, fs_util.mds_fail_over,
client_info['mds_nodes'])
p.spawn(fs_util.filesystem_utilities, client2,
client_info['mounting_dir'], dir_name, num_of_dirs * 2,
num_of_dirs * 3)
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(fs_util.pinned_dir_io_mdsfailover, client3,
client_info['mounting_dir'], dir_name, num_of_dirs * 3,
num_of_dirs * 4, 1, fs_util.mds_fail_over,
client_info['mds_nodes'])
for op in p:
return_counts, rc = op
with parallel() as p:
p.spawn(fs_util.filesystem_utilities, client3,
client_info['mounting_dir'], dir_name, num_of_dirs * 3,
num_of_dirs * 4)
p.spawn(fs_util.pinned_dir_io_mdsfailover, client4,
client_info['mounting_dir'], dir_name, num_of_dirs * 4,
num_of_dirs * 5, 1, fs_util.mds_fail_over,
client_info['mds_nodes'])
p.spawn(fs_util.filesystem_utilities, client1,
client_info['mounting_dir'], dir_name, num_of_dirs * 4,
num_of_dirs * 5)
for op in p:
return_counts, rc = op
log.info("Execution of Test case CEPH-%s ended:" % (tc))
print("Results:")
result = fs_util.rc_verify(tc, return_counts)
print(result)
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'],
'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'],
'umount')
if rc == 0:
log.info('Cleaning up successfull')
else:
return 1
print('Script execution time:------')
stop = timeit.default_timer()
total_time = stop - start
mins, secs = divmod(total_time, 60)
hours, mins = divmod(mins, 60)
print("Hours:%d Minutes:%d Seconds:%f" % (hours, mins, secs))
return 0
except CommandFailed as e:
log.info(e)
log.info(traceback.format_exc())
log.info('Cleaning up!-----')
if client3[0].pkg_type != 'deb' and client4[0].pkg_type != 'deb':
rc = fs_util.client_clean_up(client_info['fuse_clients'],
client_info['kernel_clients'],
client_info['mounting_dir'], 'umount')
else:
rc = fs_util.client_clean_up(client_info['fuse_clients'], '',
client_info['mounting_dir'], 'umount')
if rc == 0:
log.info('Cleaning up successfull')
return 1
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
return 1
