def create_tenant_user(tenant_name, user_id, cluster_name='ceph'):
# using userid as displayname
admin_ops = UserMgmt()
return admin_ops.create_tenant_user(user_id=user_id,
displayname=user_id,
cluster_name=cluster_name,
tenant_name=tenant_name)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
# preparing data
user_names = ["tuffy", "scooby", "max"]
tenant = "tenant"
tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0])
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0])
cmd = "radosgw-admin quota enable --quota-scope=user --uid={uid} --tenant={tenant}".format(
uid=user_names[0], tenant=tenant)
enable_user_quota = utils.exec_shell_cmd(cmd)
cmd = "radosgw-admin quota set --quota-scope=user --uid={uid} --tenant={tenant} --max_buckets=2000".format(
uid=user_names[0], tenant=tenant)
max_bucket = utils.exec_shell_cmd(cmd)
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=cc)
container = swiftlib.resource_op({
"obj": rgw,
"resource": "put_container",
"args": [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
host, ip = utils.get_hostname_ip()
port = utils.get_radosgw_port_no()
hostname = str(ip) + ":" + str(port)
cmd = "swift -A http://{hostname}/auth/1.0 -U '{uid}' -K '{key}' stat".format(
hostname=hostname, uid=user_info["user_id"], key=user_info["key"])
swift_cmd = utils.exec_shell_cmd(cmd)
swift_cmd = swift_cmd.replace(" ", "")
swift_cmd = swift_cmd.replace("\n", ":")
li = list(swift_cmd.split(":"))
res_dct = {li[i]: li[i + 1] for i in range(0, len(li) - 1, 2)}
if int(res_dct["Containers"]) == config.container_count:
cmd = "radosgw-admin user rm --uid={uid} --tenant={tenant} --purge-data".format(
uid=user_names[0], tenant=tenant)
delete_user_bucket = utils.exec_shell_cmd(cmd)
test_info.success_status("test passed")
sys.exit(0)
else:
cmd = "radosgw-admin user rm --uid={uid} --tenant={tenant} --purge-data".format(
uid=user_names[0], tenant=tenant)
delete_user_bucket = utils.exec_shell_cmd(cmd)
test_info.failed_status("test failed")
sys.exit(1)
def create_users(no_of_users_to_create, user_names=None, cluster_name="ceph"):
"""
This function is to create n users on the cluster
Parameters:
no_of_users_to_create(int): users to create
cluster_name(char): Name of the ceph cluster. defaults to 'ceph'
Returns:
all_users_details
"""
admin_ops = UserMgmt()
all_users_details = []
primary = utils.is_cluster_primary()
user_detail_file = os.path.join(lib_dir, "user_details.json")
if primary:
for i in range(no_of_users_to_create):
if user_names:
user_details = admin_ops.create_admin_user(
user_id=user_names,
displayname=user_names,
cluster_name=cluster_name,
)
all_users_details.append(user_details)
else:
user_details = admin_ops.create_admin_user(
user_id=names.get_first_name().lower()
+ random.choice(string.ascii_lowercase)
+ "."
+ str(random.randint(1, 1000)),
displayname=names.get_full_name().lower(),
cluster_name=cluster_name,
)
all_users_details.append(user_details)
with open(user_detail_file, "w") as fout:
json.dump(all_users_details, fout)
elif not primary:
if not os.path.exists(user_detail_file):
raise FileNotFoundError(
"user_details.json missing, this is needed in multisite setup"
)
with open(user_detail_file, "r") as fout:
all_users_details = json.load(fout)
for each_user_info in all_users_details:
write_user_info = AddUserInfo()
basic_io_structure = BasicIOInfoStructure()
user_info = basic_io_structure.user(
**{
"user_id": each_user_info["user_id"],
"access_key": each_user_info["access_key"],
"secret_key": each_user_info["secret_key"],
}
)
write_user_info.add_user_info(user_info)
return all_users_details
def create_users(no_of_users_to_create, cluster_name='ceph'):
admin_ops = UserMgmt()
all_users_details = []
for i in range(no_of_users_to_create):
user_details = admin_ops.create_admin_user(
user_id=names.get_first_name().lower() + random.choice(string.ascii_lowercase) + "." + str(
random.randint(1, 1000)),
displayname=names.get_full_name().lower(),
cluster_name=cluster_name)
all_users_details.append(user_details)
return all_users_details
def test_exec(config):
"""
Executes test based on configuration passed
Args:
config(object): Test configuration
"""
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
ceph_conf = CephConfOp()
rgw_service = RGWService()
# preparing data
user_name = resource_op.create_users(no_of_users_to_create=1)[0]["user_id"]
tenant = "tenant"
tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_name,
displayname=user_name)
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_name)
hostname = socket.gethostname()
ip = socket.gethostbyname(hostname)
port = utils.get_radosgw_port_no()
ip_and_port = f"{ip}:{port}"
s3_auth.do_auth(tenant_user_info, ip_and_port)
bucket_name = utils.gen_bucket_name_from_userid(user_name, rand_no=0)
# Create a bucket
s3cmd_reusable.create_bucket(bucket_name)
log.info(f"Bucket {bucket_name} created")
# Upload file to bucket
uploaded_file_info = s3cmd_reusable.upload_file(
bucket_name, test_data_path=TEST_DATA_PATH)
uploaded_file = uploaded_file_info["name"]
log.info(f"Uploaded file {uploaded_file} to bucket {bucket_name}")
# Delete file from bucket
s3cmd_reusable.delete_file(bucket_name, uploaded_file)
log.info(f"Deleted file {uploaded_file} from bucket {bucket_name}")
# Delete bucket
s3cmd_reusable.delete_bucket(bucket_name)
log.info(f"Bucket {bucket_name} deleted")
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def create_tenant_users(no_of_users_to_create, tenant_name, cluster_name='ceph'):
"""
This function is to create n users with tenant on the cluster
Parameters:
no_of_users_to_create(int): users to create with tenant
cluster_name(char): Name of the ceph cluster. defaults to 'ceph'
Returns:
all_users_details
"""
admin_ops = UserMgmt()
all_users_details = []
primary = utils.is_cluster_primary()
user_detail_file = os.path.join(lib_dir, 'user_details.json')
if primary:
for i in range(no_of_users_to_create):
user_details = admin_ops.create_tenant_user(
user_id=names.get_first_name().lower() + random.choice(string.ascii_lowercase) + "." + str(
random.randint(1, 1000)),
displayname=names.get_full_name().lower(),
cluster_name=cluster_name,
tenant_name=tenant_name)
all_users_details.append(user_details)
with open(user_detail_file, 'w') as fout:
json.dump(all_users_details, fout)
elif not primary:
if not os.path.exists(user_detail_file):
raise FileNotFoundError("user_details.json missing, this is needed in multisite setup")
with open(user_detail_file, 'r') as fout:
all_users_details = json.load(fout)
log.info('dump user_info into io_info.yaml')
for each_user_info in all_users_details:
write_user_info = AddUserInfo()
basic_io_structure = BasicIOInfoStructure()
tenant_info = TenantInfo()
user_info = basic_io_structure.user(**{'user_id': each_user_info['user_id'],
'access_key': each_user_info['access_key'],
'secret_key': each_user_info['secret_key']})
write_user_info.add_user_info(dict(user_info, **tenant_info.tenant(each_user_info['tenant'])))
return all_users_details
def test_exec(config):
test_info = AddTestInfo('test swift user key gen')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
try:
test_info.started_info()
# preparing data
user_names = ['tuffy', 'scooby', 'max']
tenant = 'tenant'
tenant_user_info = umgmt.create_tenant_user(
tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0],
cluster_name=config.cluster_name)
user_info = umgmt.create_subuser(tenant_name=tenant,
user_id=user_names[0],
cluster_name=config.cluster_name)
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
container_name = utils.gen_bucket_name_from_userid(
user_info['user_id'], rand_no=cc)
container = swiftlib.resource_op({
'obj': rgw,
'resource': 'put_container',
'args': [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
for oc in range(config.objects_count):
swift_object_name = utils.gen_s3_object_name(
'%s.container.%s' % (user_names[0], cc), oc)
log.info('object name: %s' % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info('object path: %s' % object_path)
object_size = utils.get_file_size(
config.objects_size_range['min'],
config.objects_size_range['max'])
data_info = manage_data.io_generator(object_path, object_size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info('uploading object: %s' % object_path)
with open(object_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
# download object
swift_object_download_fname = swift_object_name + ".download"
log.info('download object name: %s' %
swift_object_download_fname)
swift_object_download_path = os.path.join(
TEST_DATA_PATH, swift_object_download_fname)
log.info('download object path: %s' %
swift_object_download_path)
swift_object_downloaded = rgw.get_object(
container_name, swift_object_name)
with open(swift_object_download_path, 'w') as fp:
fp.write(swift_object_downloaded[1])
# modify and re-upload
log.info('appending new message to test_data')
message_to_append = 'adding new msg after download'
fp = open(swift_object_download_path, 'a+')
fp.write(message_to_append)
fp.close()
with open(swift_object_download_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
# delete object
log.info('deleting swift object')
rgw.delete_object(container_name, swift_object_name)
# delete container
log.info('deleting swift container')
rgw.delete_container(container_name)
test_info.success_status('test passed')
sys.exit(0)
except Exception, e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
rgw_service = RGWService()
# create pool
pool_name = '.rgw.buckets.special'
pg_num = '8'
pgp_num = '8'
pool_create = 'sudo ceph osd pool create "%s" %s %s replicated' % (
pool_name, pg_num, pgp_num)
pool_create_exec = utils.exec_shell_cmd(pool_create)
if pool_create_exec is False:
raise TestExecError("Pool creation failed")
# create realm
realm_name = 'buz-tickets'
log.info('creating realm name')
realm_create = 'sudo radosgw-admin realm create --rgw-realm=%s' % realm_name
realm_create_exec = utils.exec_shell_cmd(realm_create)
if realm_create_exec is False:
raise TestExecError("cmd execution failed")
# sample output of create realm
"""
{
"id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62",
"name": "buz-tickets",
"current_period": "1950b710-3e63-4c41-a19e-46a715000980",
"epoch": 1
}
"""
log.info('modify zonegroup ')
modify = 'sudo radosgw-admin zonegroup modify --rgw-zonegroup=default --rgw-realm=%s --master' % realm_name
modify_exec = utils.exec_shell_cmd(modify)
if modify_exec is False:
raise TestExecError("cmd execution failed")
# get the zonegroup
zonegroup_file = 'zonegroup.json'
get_zonegroup = 'sudo radosgw-admin zonegroup --rgw-zonegroup=default get > %s' % zonegroup_file
get_zonegroup_exec = utils.exec_shell_cmd(get_zonegroup)
if get_zonegroup_exec is False:
raise TestExecError("cmd execution failed")
add_to_placement_targets = {"name": "special-placement", "tags": []}
fp = open(zonegroup_file, 'r')
zonegroup_txt = fp.read()
fp.close()
log.info('got zonegroup info: \n%s' % zonegroup_txt)
zonegroup = json.loads(zonegroup_txt)
log.info('adding placement targets')
zonegroup['placement_targets'].append(add_to_placement_targets)
with open(zonegroup_file, 'w') as fp:
json.dump(zonegroup, fp)
zonegroup_set = 'sudo radosgw-admin zonegroup set < %s' % zonegroup_file
zonegroup_set_exec = utils.exec_shell_cmd(zonegroup_set)
if zonegroup_set_exec is False:
raise TestExecError("cmd execution failed")
log.info('zone group update completed')
log.info('getting zone file')
# get zone
log.info('getting zone info')
zone_file = 'zone.json'
get_zone = 'sudo radosgw-admin zone --rgw-zone=default get > zone.json'
get_zone_exec = utils.exec_shell_cmd(get_zone)
if get_zone_exec is False:
raise TestExecError("cmd execution failed")
fp = open(zone_file, 'r')
zone_info = fp.read()
fp.close()
log.info('zone_info :\n%s' % zone_info)
zone_info_cleaned = json.loads(zone_info)
special_placement_info = {
"key": "special-placement",
"val": {
"index_pool": ".rgw.buckets.index",
"data_pool": ".rgw.buckets.special",
"data_extra_pool": ".rgw.buckets.extra"
}
}
log.info('adding special placement info')
zone_info_cleaned['placement_pools'].append(special_placement_info)
with open(zone_file, 'w+') as fp:
json.dump(zone_info_cleaned, fp)
zone_file_set = 'sudo radosgw-admin zone set < %s' % zone_file
zone_file_set_exec = utils.exec_shell_cmd(zone_file_set)
if zone_file_set_exec is False:
raise TestExecError("cmd execution failed")
log.info('zone info updated ')
zone_group_update_set = 'radosgw-admin period update --commit'
zone_group_update_set_exec = utils.exec_shell_cmd(zone_group_update_set)
log.info(zone_group_update_set_exec)
restarted = rgw_service.restart()
if restarted is False:
raise TestExecError("service restart failed")
if config.rgw_client == 'rgw':
log.info('client type is rgw')
rgw_user_info = s3_swift_lib.create_users(1)
auth = Auth(rgw_user_info)
rgw_conn = auth.do_auth()
# create bucket
bucket_name = utils.gen_bucket_name_from_userid(
rgw_user_info['user_id'], 0)
bucket = resuables.create_bucket(bucket_name, rgw_conn, rgw_user_info)
# create object
s3_object_name = utils.gen_s3_object_name(bucket_name, 0)
resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config,
rgw_user_info)
if config.rgw_client == 'swift':
log.info('client type is swift')
user_names = ['tuffy', 'scooby', 'max']
tenant = 'tenant'
umgmt = UserMgmt()
umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0])
user_info = umgmt.create_subuser(tenant_name=tenant,
user_id=user_names[0])
auth = Auth(user_info)
rgw = auth.do_auth()
container_name = utils.gen_bucket_name_from_userid(
user_info['user_id'], rand_no=0)
container = s3_swift_lib.resource_op({
'obj': rgw,
'resource': 'put_container',
'args': [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
swift_object_name = utils.gen_s3_object_name(
'%s.container.%s' % (user_names[0], 0), 0)
log.info('object name: %s' % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info('object path: %s' % object_path)
object_size = utils.get_file_size(config.objects_size_range['min'],
config.objects_size_range['max'])
data_info = manage_data.io_generator(object_path, object_size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info('uploading object: %s' % object_path)
with open(object_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
def test_exec(config):
test_info = AddTestInfo("test swift user key gen")
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
try:
test_info.started_info()
# preparing data
user_names = ["tuffy", "scooby", "max"]
tenant = "tenant"
tenant_user_info = umgmt.create_tenant_user(
tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0],
cluster_name=config.cluster_name,
)
user_info = umgmt.create_subuser(tenant_name=tenant,
user_id=user_names[0],
cluster_name=config.cluster_name)
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=cc)
container = swiftlib.resource_op({
"obj": rgw,
"resource": "put_container",
"args": [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
for oc in range(config.objects_count):
swift_object_name = utils.gen_s3_object_name(
"%s.container.%s" % (user_names[0], cc), oc)
log.info("object name: %s" % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info("object path: %s" % object_path)
object_size = utils.get_file_size(
config.objects_size_range["min"],
config.objects_size_range["max"])
data_info = manage_data.io_generator(object_path, object_size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info("uploading object: %s" % object_path)
with open(object_path, "r") as fp:
rgw.put_object(
container_name,
swift_object_name,
contents=fp.read(),
content_type="text/plain",
)
# download object
swift_object_download_fname = swift_object_name + ".download"
log.info("download object name: %s" %
swift_object_download_fname)
swift_object_download_path = os.path.join(
TEST_DATA_PATH, swift_object_download_fname)
log.info("download object path: %s" %
swift_object_download_path)
swift_object_downloaded = rgw.get_object(
container_name, swift_object_name)
with open(swift_object_download_path, "w") as fp:
fp.write(swift_object_downloaded[1])
# modify and re-upload
log.info("appending new message to test_data")
message_to_append = "adding new msg after download"
fp = open(swift_object_download_path, "a+")
fp.write(message_to_append)
fp.close()
with open(swift_object_download_path, "r") as fp:
rgw.put_object(
container_name,
swift_object_name,
contents=fp.read(),
content_type="text/plain",
)
# delete object
log.info("deleting swift object")
rgw.delete_object(container_name, swift_object_name)
# delete container
log.info("deleting swift container")
rgw.delete_container(container_name)
test_info.success_status("test passed")
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
def test_exec(config):
"""
Executes test based on configuration passed
Args:
config(object): Test configuration
"""
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
ceph_conf = CephConfOp()
rgw_service = RGWService()
# preparing data
user_name = resource_op.create_users(no_of_users_to_create=1)[0]["user_id"]
tenant = "tenant"
tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_name,
displayname=user_name)
umgmt.create_subuser(tenant_name=tenant, user_id=user_name)
ip_and_port = s3cmd_reusable.get_rgw_ip_and_port()
s3_auth.do_auth(tenant_user_info, ip_and_port)
bucket_name = utils.gen_bucket_name_from_userid(user_name, rand_no=0)
# Create a bucket
s3cmd_reusable.create_bucket(bucket_name)
log.info(f"Bucket {bucket_name} created")
# Upload a 2GB file to bucket
uploaded_file_info = s3cmd_reusable.upload_file(
bucket_name, file_size=2147483648, test_data_path=TEST_DATA_PATH)
uploaded_file = uploaded_file_info["name"]
uploaded_file_md5 = uploaded_file_info["md5"]
log.info(f"Uploaded file {uploaded_file} to bucket {bucket_name}")
if config.gc_verification is True:
log.info("making changes to ceph.conf")
config.rgw_gc_obj_min_wait = 5
ceph_conf.set_to_ceph_conf(
"global",
ConfigOpts.rgw_gc_obj_min_wait,
str(config.rgw_gc_obj_min_wait),
)
log.info("trying to restart services")
srv_restarted = rgw_service.restart()
time.sleep(30)
if srv_restarted is False:
raise TestExecError("RGW service restart failed")
else:
log.info("RGW service restarted")
log.info(
"download large object again to make gc list with shadow entries")
downloaded_file1 = s3cmd_reusable.download_file(
bucket_name,
uploaded_file,
local_file_name="download1.img",
test_data_path=TEST_DATA_PATH,
)
time.sleep(5)
downloaded_file1_md5 = utils.get_md5(downloaded_file1)
assert uploaded_file_md5 == downloaded_file1_md5
gc_list_output = json.loads(
utils.exec_shell_cmd("radosgw-admin gc list --include-all"))
log.info(gc_list_output)
if gc_list_output:
log.info(
"Shadow obj found after setting rgw_gc_obj_min_wait to 5 sec")
utils.exec_shell_cmd("radosgw-admin gc process --include-all")
log.info(
"Object download should not error out in 404 NoSuchKey error")
downloaded_file2 = s3cmd_reusable.download_file(
bucket_name,
uploaded_file,
local_file_name="download2.img",
test_data_path=TEST_DATA_PATH,
)
downloaded_file2_md5 = utils.get_md5(downloaded_file2)
assert uploaded_file_md5 == downloaded_file2_md5
# Delete file from bucket
s3cmd_reusable.delete_file(bucket_name, uploaded_file)
log.info(f"Deleted file {uploaded_file} from bucket {bucket_name}")
# Delete bucket
s3cmd_reusable.delete_bucket(bucket_name)
log.info(f"Bucket {bucket_name} deleted")
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
test_info = AddTestInfo("storage_policy for %s" % config.rgw_client)
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
rgw_service = RGWService()
try:
# create pool
pool_name = ".rgw.buckets.special"
pg_num = "8"
pgp_num = "8"
pool_create = 'sudo ceph osd pool create "%s" %s %s replicated' % (
pool_name,
pg_num,
pgp_num,
)
pool_create_exec = utils.exec_shell_cmd(pool_create)
if pool_create_exec is False:
raise TestExecError("Pool creation failed")
# create realm
realm_name = "buz-tickets"
log.info("creating realm name")
realm_create = (
"sudo radosgw-admin realm create --rgw-realm=%s --default" % realm_name
)
realm_create_exec = utils.exec_shell_cmd(realm_create)
if realm_create_exec is False:
raise TestExecError("cmd execution failed")
# sample output of create realm
"""
{
"id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62",
"name": "buz-tickets",
"current_period": "1950b710-3e63-4c41-a19e-46a715000980",
"epoch": 1
}
"""
log.info("modify zonegroup ")
modify = (
"sudo radosgw-admin zonegroup modify --rgw-zonegroup=default --rgw-realm=%s --master --default"
% realm_name
)
modify_exec = utils.exec_shell_cmd(modify)
if modify_exec is False:
raise TestExecError("cmd execution failed")
# get the zonegroup
zonegroup_file = "zonegroup.json"
get_zonegroup = (
"sudo radosgw-admin zonegroup --rgw-zonegroup=default get > %s"
% zonegroup_file
)
get_zonegroup_exec = utils.exec_shell_cmd(get_zonegroup)
if get_zonegroup_exec is False:
raise TestExecError("cmd execution failed")
add_to_placement_targets = {"name": "special-placement", "tags": []}
fp = open(zonegroup_file, "r")
zonegroup_txt = fp.read()
fp.close()
log.info("got zonegroup info: \n%s" % zonegroup_txt)
zonegroup = json.loads(zonegroup_txt)
log.info("adding placement targets")
zonegroup["placement_targets"].append(add_to_placement_targets)
with open(zonegroup_file, "w") as fp:
json.dump(zonegroup, fp)
zonegroup_set = "sudo radosgw-admin zonegroup set < %s" % zonegroup_file
zonegroup_set_exec = utils.exec_shell_cmd(zonegroup_set)
if zonegroup_set_exec is False:
raise TestExecError("cmd execution failed")
log.info("zone group update completed")
log.info("getting zone file")
# get zone
log.info("getting zone info")
zone_file = "zone.json"
get_zone = "sudo radosgw-admin zone --rgw-zone=default get > zone.json"
get_zone_exec = utils.exec_shell_cmd(get_zone)
if get_zone_exec is False:
raise TestExecError("cmd execution failed")
fp = open(zone_file, "r")
zone_info = fp.read()
fp.close()
log.info("zone_info :\n%s" % zone_info)
zone_info_cleaned = json.loads(zone_info)
special_placement_info = {
"key": "special-placement",
"val": {
"index_pool": ".rgw.buckets.index",
"data_pool": ".rgw.buckets.special",
"data_extra_pool": ".rgw.buckets.extra",
},
}
log.info("adding special placement info")
zone_info_cleaned["placement_pools"].append(special_placement_info)
print(zone_info_cleaned)
with open(zone_file, "w+") as fp:
json.dump(zone_info_cleaned, fp)
zone_file_set = "sudo radosgw-admin zone set < %s" % zone_file
zone_file_set_exec = utils.exec_shell_cmd(zone_file_set)
if zone_file_set_exec is False:
raise TestExecError("cmd execution failed")
log.info("zone info updated ")
restarted = rgw_service.restart()
if restarted is False:
raise TestExecError("service restart failed")
if config.rgw_client == "rgw":
log.info("client type is rgw")
rgw_user_info = s3_swift_lib.create_users(1)
auth = Auth(rgw_user_info)
rgw_conn = auth.do_auth()
# create bucket
bucket_name = utils.gen_bucket_name_from_userid(rgw_user_info["user_id"], 0)
bucket = resuables.create_bucket(bucket_name, rgw_conn, rgw_user_info)
# create object
s3_object_name = utils.gen_s3_object_name(bucket_name, 0)
resuables.upload_object(
s3_object_name, bucket, TEST_DATA_PATH, config, rgw_user_info
)
if config.rgw_client == "swift":
log.info("client type is swift")
user_names = ["tuffy", "scooby", "max"]
tenant = "tenant"
umgmt = UserMgmt()
umgmt.create_tenant_user(
tenant_name=tenant, user_id=user_names[0], displayname=user_names[0]
)
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0])
auth = Auth(user_info)
rgw = auth.do_auth()
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=0
)
container = s3_swift_lib.resource_op(
{"obj": rgw, "resource": "put_container", "args": [container_name]}
)
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield"
)
swift_object_name = utils.gen_s3_object_name(
"%s.container.%s" % (user_names[0], 0), 0
)
log.info("object name: %s" % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info("object path: %s" % object_path)
object_size = utils.get_file_size(
config.objects_size_range["min"], config.objects_size_range["max"]
)
data_info = manage_data.io_generator(object_path, object_size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info("uploading object: %s" % object_path)
with open(object_path, "r") as fp:
rgw.put_object(
container_name,
swift_object_name,
contents=fp.read(),
content_type="text/plain",
)
test_info.success_status("test passed")
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
host, ip = utils.get_hostname_ip()
port = utils.get_radosgw_port_no()
hostname = str(ip) + ":" + str(port)
log.info(hostname)
# preparing data
admin_api_user = "******" + randomString()
log.info(admin_api_user)
user_info = umgmt.create_rest_admin_user(user_id=admin_api_user,
displayname=admin_api_user)
rgw = RGWAdmin(access_key=user_info['access_key'],
secret_key=user_info['secret_key'],
server=hostname,
secure=False,
verify=False)
api_user = "******" + randomString()
log.info(api_user)
for uc in range(config.user_count):
#Create User
data = rgw.create_user(uid=api_user,
display_name=api_user,
email=api_user + '@abc.xyz')
log.info("User created successfully")
log.info(data)
log.info('verification starts')
op = utils.exec_shell_cmd("radosgw-admin user info --uid %s" %
api_user)
json_doc = json.loads(op)
log.info(json_doc)
v = verify_user(data, json_doc)
if v is False:
test_info.failed_status('test failed')
sys.exit(1)
log.info("Verification for create operation completed")
#Update User
data = rgw.modify_user(uid=api_user,
display_name=api_user + "_11",
email=api_user + '*****@*****.**')
log.info("User Updated successfully")
log.info(data)
log.info('verification starts')
op = utils.exec_shell_cmd("radosgw-admin user info --uid %s" %
api_user)
json_doc = json.loads(op)
log.info(json_doc)
v = verify_user(data, json_doc)
if v is False:
test_info.failed_status('test failed')
sys.exit(1)
log.info("Verification for Update operation completed")
#delete User
data = rgw.remove_user(uid=api_user, purge_data=False)
log.info(data)
log.info("User removed")
op = utils.exec_shell_cmd("radosgw-admin user list")
json_doc = json.loads(op)
if api_user in json_doc:
test_info.failed_status('test failed')
sys.exit(1)
log.info("Verification for Delete operation completed")
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
"""
Executes test based on configuration passed
Args:
config(object): Test configuration
"""
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
ceph_conf = CephConfOp()
rgw_service = RGWService()
# preparing data
user_names = ["tom", "ram", "sam"]
tenant = "tenant"
tenant_user_info = umgmt.create_tenant_user(
tenant_name=tenant, user_id=user_names[1], displayname=user_names[1]
)
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[1])
auth = Auth(user_info, config.ssl)
rgw = auth.do_auth()
container_name = utils.gen_bucket_name_from_userid(user_info["user_id"], rand_no=0)
container = swiftlib.resource_op(
{"obj": rgw, "resource": "put_container", "args": [container_name]}
)
if container is False:
raise TestExecError("Resource execution failed: container creation faield")
for oc, size in list(config.mapped_sizes.items()):
# upload objects to the container
swift_object_name = fill_container(
rgw, container_name, user_names[1], oc, 0, size
)
# delete all uploaded objects
log.info("deleting all swift objects")
auth_response = rgw.get_auth()
token = auth_response[1]
# test.txt file should contain container_name
with open("test.txt", "w") as f:
f.write(container_name)
ip_and_port = rgw.authurl.split("/")[2]
proto = "https" if config.ssl else "http"
url = f"{proto}://{ip_and_port}/swift/v1/?bulk-delete"
test_file = open("test.txt", "r")
headers = {
"Accept": "application/json",
"Content-Type": "text/plain",
"X-Auth-Token": token,
}
response = requests.delete(
url, headers=headers, verify=False, files={"form_field_name": test_file}
)
if response.status_code == 200:
log.info("Bulk delete succeeded")
else:
raise TestExecError(
"Bulk delete failed with status code: %d" % response.status_code
)
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
ceph_conf = CephConfOp()
log.info(type(ceph_conf))
rgw_service = RGWService()
# preparing data
user_names = ["tuffy", "scooby", "max"]
tenant = "tenant"
tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0])
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0])
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
if config.version_enable is True:
log.info("making changes to ceph.conf")
ceph_conf.set_to_ceph_conf("global",
ConfigOpts.rgw_swift_versioning_enabled,
"True")
log.info("trying to restart services ")
srv_restarted = rgw_service.restart()
time.sleep(30)
if srv_restarted is False:
raise TestExecError("RGW service restart failed")
else:
log.info("RGW service restarted")
container_name_old = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=str(cc) + "old")
log.info(container_name_old)
container = swiftlib.resource_op({
"obj":
rgw,
"resource":
"put_container",
"kwargs":
dict(container=container_name_old),
})
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=str(cc) + "new")
log.info(container_name)
container = swiftlib.resource_op({
"obj":
rgw,
"resource":
"put_container",
"args": [
container_name,
{
"X-Versions-Location": container_name_old
},
],
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation failed")
ls = []
swift_object_name = ""
for version_count in range(config.version_count):
for oc, size in list(config.mapped_sizes.items()):
swift_object_name = fill_container(rgw, container_name,
user_names[0], oc, cc,
size)
ls = rgw.get_container(container_name_old)
ls = list(ls)
if config.copy_version_object is True:
old_obj_name = ls[1][config.version_count - 2]["name"]
log.info(old_obj_name)
container = swiftlib.resource_op({
"obj":
rgw,
"resource":
"copy_object",
"kwargs":
dict(
container=container_name_old,
obj=old_obj_name,
destination=container_name + "/" + swift_object_name,
),
})
if container is False:
raise TestExecError("Resource execution failed")
log.info("Successfully copied item")
else:
current_count = "radosgw-admin bucket stats --uid={uid} --tenant={tenant} --bucket='{bucket}' ".format(
uid=user_names[0], tenant=tenant, bucket=container_name)
num_obj_current = utils.exec_shell_cmd(current_count)
num_obj_current = json.loads(num_obj_current)
num_obj_current = (num_obj_current[0].get("usage").get(
"rgw.main").get("num_objects"))
old_count = "radosgw-admin bucket stats --uid={uid} --tenant={tenant} --bucket='{bucket}' ".format(
uid=user_names[0],
tenant=tenant,
bucket=container_name_old)
num_obj_old = utils.exec_shell_cmd(old_count)
num_obj_old = json.loads(num_obj_old)
num_obj_old = (num_obj_old[0].get("usage").get("rgw.main").get(
"num_objects"))
version_count_from_config = (
config.objects_count *
config.version_count) - config.objects_count
if (num_obj_current == config.objects_count) and (
num_obj_old == version_count_from_config):
log.info("objects and versioned obbjects are correct")
else:
test_info.failed_status("test failed")
elif config.object_expire is True:
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=cc)
container = swiftlib.resource_op({
"obj": rgw,
"resource": "put_container",
"args": [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation failed")
for oc, size in list(config.mapped_sizes.items()):
swift_object_name = fill_container(
rgw,
container_name,
user_names[0],
oc,
cc,
size,
header={"X-Delete-After": 5},
)
time.sleep(7)
container_exists = swiftlib.resource_op({
"obj":
rgw,
"resource":
"get_object",
"args": [container_name, swift_object_name],
})
log.info(container_exists)
if container_exists:
msg = "test failed as the objects are still present"
test_info.failed_status(msg)
raise TestExecError(msg)
elif config.large_object_upload is True:
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=cc)
container = swiftlib.resource_op({
"obj": rgw,
"resource": "put_container",
"args": [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation failed")
for oc, size in list(config.mapped_sizes.items()):
swift_object_name = fill_container(
rgw,
container_name,
user_names[0],
oc,
cc,
size,
multipart=True,
split_size=config.split_size,
)
container_name_new = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=str(cc) + "New")
container = swiftlib.resource_op({
"obj":
rgw,
"resource":
"put_container",
"kwargs":
dict(container=container_name_new),
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation failed")
container = swiftlib.resource_op({
"obj":
rgw,
"resource":
"put_object",
"kwargs":
dict(
container=container_name_new,
obj=swift_object_name,
contents=None,
headers={
"X-Object-Manifest":
container_name + "/" + swift_object_name + "/"
},
),
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation failed")
if config.large_object_download is True:
swift_old_object_path = os.path.join(
TEST_DATA_PATH, swift_object_name)
swift_object_download_fname = swift_object_name + ".download"
log.info("download object name: %s" %
swift_object_download_fname)
swift_object_download_path = os.path.join(
TEST_DATA_PATH, swift_object_download_fname)
log.info("download object path: %s" %
swift_object_download_path)
swift_object_downloaded = rgw.get_object(
container_name_new, swift_object_name)
with open(swift_object_download_path, "wb") as fp:
fp.write(swift_object_downloaded[1])
old_object = utils.get_md5(swift_old_object_path)
downloaded_obj = utils.get_md5(swift_object_download_path)
log.info("s3_object_downloaded_md5: %s" % old_object)
log.info("s3_object_uploaded_md5: %s" % downloaded_obj)
if str(old_object) == str(downloaded_obj):
log.info("md5 match")
utils.exec_shell_cmd("rm -rf %s" %
swift_object_download_path)
else:
raise TestExecError("md5 mismatch")
else:
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=cc)
container = swiftlib.resource_op({
"obj": rgw,
"resource": "put_container",
"args": [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation failed")
for oc, size in list(config.mapped_sizes.items()):
swift_object_name = fill_container(rgw, container_name,
user_names[0], oc, cc, size)
# download object
swift_object_download_fname = swift_object_name + ".download"
log.info("download object name: %s" %
swift_object_download_fname)
swift_object_download_path = os.path.join(
TEST_DATA_PATH, swift_object_download_fname)
log.info("download object path: %s" %
swift_object_download_path)
swift_object_downloaded = rgw.get_object(
container_name, swift_object_name)
with open(swift_object_download_path, "w") as fp:
fp.write(str(swift_object_downloaded[1]))
# modify and re-upload
log.info("appending new message to test_data")
message_to_append = "adding new msg after download"
fp = open(swift_object_download_path, "a+")
fp.write(message_to_append)
fp.close()
with open(swift_object_download_path, "r") as fp:
rgw.put_object(
container_name,
swift_object_name,
contents=fp.read(),
content_type="text/plain",
)
# delete object
log.info("deleting swift object")
rgw.delete_object(container_name, swift_object_name)
# delete container
log.info("deleting swift container")
rgw.delete_container(container_name)
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
reusable.remove_user(tenant_user_info, tenant=tenant)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
# preparing data
user_names = ['tuffy', 'scooby', 'max']
tenant = 'tenant'
tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0])
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0])
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
container_name = utils.gen_bucket_name_from_userid(
user_info['user_id'], rand_no=cc)
container = swiftlib.resource_op({
'obj': rgw,
'resource': 'put_container',
'args': [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
for oc, size in list(config.mapped_sizes.items()):
swift_object_name = utils.gen_s3_object_name(
'%s.container.%s' % (user_names[0], cc), oc)
log.info('object name: %s' % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info('object path: %s' % object_path)
data_info = manage_data.io_generator(object_path, size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info('uploading object: %s' % object_path)
with open(object_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
# download object
swift_object_download_fname = swift_object_name + ".download"
log.info('download object name: %s' % swift_object_download_fname)
swift_object_download_path = os.path.join(
TEST_DATA_PATH, swift_object_download_fname)
log.info('download object path: %s' % swift_object_download_path)
swift_object_downloaded = rgw.get_object(container_name,
swift_object_name)
with open(swift_object_download_path, 'w') as fp:
fp.write(str(swift_object_downloaded[1]))
# modify and re-upload
log.info('appending new message to test_data')
message_to_append = 'adding new msg after download'
fp = open(swift_object_download_path, 'a+')
fp.write(message_to_append)
fp.close()
with open(swift_object_download_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
# delete object
log.info('deleting swift object')
rgw.delete_object(container_name, swift_object_name)
# delete container
log.info('deleting swift container')
rgw.delete_container(container_name)
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")