def test_exec(config):
test_info = AddTestInfo('create users')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
user_detail_file = os.path.join(lib_dir, 'user_details.json')
try:
test_info.started_info()
# create a non-tenanted user
if config.user_type == 'non-tenanted':
all_users_info = s3lib.create_users(config.user_count)
with open(user_detail_file, 'w') as fout:
json.dump(all_users_info, fout)
test_info.success_status('non-tenanted users creation completed')
else:
log.info('create tenanted users')
for i in range(config.user_count):
tenant_name = 'tenant' + str(i)
all_users_info = s3lib.create_tenant_users(
config.user_count, tenant_name)
with open(user_detail_file, 'w') as fout:
json.dump(all_users_info, fout)
test_info.success_status('tenanted users creation completed')
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('user creation failed')
sys.exit(1)
except (RGWBaseException, Exception) as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('user creation failed')
sys.exit(1)
def test_exec(config):
test_info = AddTestInfo('test frontends configuration')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
bucket_name_to_create2 = utils.gen_bucket_name_from_userid(
each_user['user_id'])
log.info('creating bucket with name: %s' % bucket_name_to_create2)
bucket = resuables.create_bucket(bucket_name_to_create2, rgw_conn,
each_user)
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config, requester):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
log.info('requester type: %s' % requester)
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
# create buckets
log.info('no of buckets to create: %s' % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(
each_user['user_id'], rand_no=bc)
log.info('creating bucket with name: %s' % bucket_name_to_create)
# bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create)
bucket = resuables.create_bucket(bucket_name=bucket_name_to_create,
rgw=rgw_conn,
user_info=each_user)
bucket_request_payer = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'BucketRequestPayment',
'args': [bucket.name]
})
# change the bucket request payer to 'requester'
payer = {'Payer': requester}
response = s3lib.resource_op({
'obj':
bucket_request_payer,
'resource':
'put',
'kwargs':
dict(RequestPaymentConfiguration=payer)
})
log.info(response)
if response is not None:
response = HttpResponseParser(response)
if response.status_code == 200:
log.info('bucket created')
else:
raise TestExecError(
"bucket request payer modification failed")
else:
raise TestExecError("bucket request payer modification failed")
payer = bucket_request_payer.payer
log.info('bucket request payer: %s' % payer)
if payer != 'Requester':
TestExecError('Request payer is not set or changed properly ')
log.info('s3 objects to create: %s' % config.objects_count)
if config.objects_count is not None:
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(bucket.name, oc)
resuables.upload_object(s3_object_name, bucket,
TEST_DATA_PATH, config, each_user)
def create_rgw_user(self):
log.info('creating rgw user')
rgw_user = s3lib.create_users(1)[0]
self.rgw_user_info['user_id'] = rgw_user['user_id']
self.rgw_user_info['access_key'] = rgw_user['access_key']
self.rgw_user_info['secret_key'] = rgw_user['secret_key']
self.rgw_user_info['rgw_hostname'] = socket.gethostname()
self.rgw_user_info['ganesha_config_exists'] = False
self.rgw_user_info['already_mounted'] = False
def create_rgw_user(self):
log.info("creating rgw user")
rgw_user = s3lib.create_users(1)[0]
self.rgw_user_info["user_id"] = rgw_user["user_id"]
self.rgw_user_info["access_key"] = rgw_user["access_key"]
self.rgw_user_info["secret_key"] = rgw_user["secret_key"]
self.rgw_user_info["rgw_hostname"] = socket.gethostname()
self.rgw_user_info["ganesha_config_exists"] = False
self.rgw_user_info["already_mounted"] = False
def test_exec(config):
test_info = AddTestInfo('Bucket Request Payer')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
# create user
all_users_info = s3lib.create_users(config.user_count,
config.cluster_name)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user)
rgw_conn = auth.do_auth()
s3_object_names = []
# create buckets
log.info('no of buckets to create: %s' % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(
each_user['user_id'], rand_no=bc)
log.info('creating bucket with name: %s' %
bucket_name_to_create)
# bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create)
bucket = resuables.create_bucket(
bucket_name=bucket_name_to_create,
rgw=rgw_conn,
user_info=each_user)
bucket_request_payer = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'BucketRequestPayment',
'args': [bucket.name]
})
# change the bucket request payer to 'requester'
payer = {'Payer': 'Requester'}
response = s3lib.resource_op({
'obj':
bucket_request_payer,
'resource':
'put',
'kwargs':
dict(RequestPaymentConfiguration=payer)
})
log.info(response)
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
test_info = AddTestInfo('Test Byte range')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
# create buckets
log.info('no of buckets to create: %s' % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=1)
bucket = resuables.create_bucket(bucket_name, rgw_conn, each_user)
# uploading data
log.info('s3 objects to create: %s' % config.objects_count)
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(bucket.name, oc)
resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user)
log.info('testing for negative range')
response = rgw_conn2.get_object(Bucket=bucket.name, Key=s3_object_name, Range='-2--1')
log.info('response: %s\n' % response)
log.info('Content-Lenght: %s' % response['ContentLength'])
log.info('s3_object_size: %s' % (config.obj_size * 1024 * 1024))
if response['ContentLength'] != config.obj_size * 1024 * 1024:
TestExecError("Content Lenght not matched")
log.info('testing for one positive and one negative range')
response = rgw_conn2.get_object(Bucket=bucket.name, Key=s3_object_name, Range='-1-3')
log.info('response: %s\n' % response)
log.info('Content-Length: %s' % response['ContentLength'])
log.info('s3_object_size: %s' % (config.obj_size * 1024 * 1024))
if response['ContentLength'] != config.obj_size * 1024 * 1024:
TestExecError("Content Lenght not matched")
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
bucket_name_to_create2 = utils.gen_bucket_name_from_userid(each_user['user_id'])
log.info('creating bucket with name: %s' % bucket_name_to_create2)
bucket = resuables.create_bucket(bucket_name_to_create2, rgw_conn, each_user)
def test_exec(config):
"""
Executes test based on configuration passed
Args:
config(object): Test configuration
"""
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
user_info = resource_op.create_users(
no_of_users_to_create=config.user_count)[0]
user_name = user_info["user_id"]
ip_and_port = s3cmd_reusable.get_rgw_ip_and_port()
s3_auth.do_auth(user_info, ip_and_port)
if config.bucket_stats:
bucket_name = utils.gen_bucket_name_from_userid(user_name, rand_no=0)
s3cmd_reusable.create_bucket(bucket_name)
log.info(f"Bucket {bucket_name} created")
utils.exec_shell_cmd(f"fallocate -l 25m obj25m")
object_name = f"s3://{bucket_name}/encyclopedia/space & universe/.bkp/journal$i"
range_val = f"1..{config.objects_count}"
cmd = ("for i in {" + range_val +
"}; do /home/cephuser/venv/bin/s3cmd put obj25m " +
object_name + ";done;")
rc = utils.exec_shell_cmd(cmd)
if rc:
raise AssertionError("expected scenario is not achieved!!!")
bucket_stats = utils.exec_shell_cmd(
f"radosgw-admin bucket stats --bucket {bucket_name}")
log.info(f" bucket stats are :{bucket_stats}")
data = json.loads(bucket_stats)
num_objects = data["usage"]["rgw.main"]["num_objects"]
log.info(f"num objects :{num_objects}")
object_count = utils.exec_shell_cmd(
f"/home/cephuser/venv/bin/s3cmd ls s3://{bucket_name} --recursive | wc -l"
)
log.info(f"object_count :{object_count}")
if int(num_objects) != int(object_count):
raise AssertionError("Inconsistency found in number of objects")
if "rgw.none" in data["usage"].keys():
raise AssertionError("inconsistency issue observed")
def test_exec(config):
"""
Executes test based on configuration passed
Args:
config(object): Test configuration
"""
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
ceph_conf = CephConfOp()
rgw_service = RGWService()
# preparing data
user_name = resource_op.create_users(no_of_users_to_create=1)[0]["user_id"]
tenant = "tenant"
tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_name,
displayname=user_name)
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_name)
hostname = socket.gethostname()
ip = socket.gethostbyname(hostname)
port = utils.get_radosgw_port_no()
ip_and_port = f"{ip}:{port}"
s3_auth.do_auth(tenant_user_info, ip_and_port)
bucket_name = utils.gen_bucket_name_from_userid(user_name, rand_no=0)
# Create a bucket
s3cmd_reusable.create_bucket(bucket_name)
log.info(f"Bucket {bucket_name} created")
# Upload file to bucket
uploaded_file_info = s3cmd_reusable.upload_file(
bucket_name, test_data_path=TEST_DATA_PATH)
uploaded_file = uploaded_file_info["name"]
log.info(f"Uploaded file {uploaded_file} to bucket {bucket_name}")
# Delete file from bucket
s3cmd_reusable.delete_file(bucket_name, uploaded_file)
log.info(f"Deleted file {uploaded_file} from bucket {bucket_name}")
# Delete bucket
s3cmd_reusable.delete_bucket(bucket_name)
log.info(f"Bucket {bucket_name} deleted")
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
test_info = AddTestInfo("Test Byte range")
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
test_info.started_info()
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
# create buckets
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=1
)
bucket = reusable.create_bucket(bucket_name, rgw_conn, each_user)
# uploading data
log.info("s3 objects to create: %s" % config.objects_count)
for oc, size in config.mapped_sizes.items():
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(bucket.name, oc)
reusable.upload_object(
s3_object_name, bucket, TEST_DATA_PATH, config, each_user
)
log.info("testing for negative range")
response = rgw_conn2.get_object(
Bucket=bucket.name, Key=s3_object_name, Range="-2--1"
)
log.info("response: %s\n" % response)
log.info("Content-Lenght: %s" % response["ContentLength"])
log.info("s3_object_size: %s" % (config.obj_size * 1024 * 1024))
if response["ContentLength"] != config.obj_size * 1024 * 1024:
TestExecError("Content Lenght not matched")
log.info("testing for one positive and one negative range")
response = rgw_conn2.get_object(
Bucket=bucket.name, Key=s3_object_name, Range="-1-3"
)
log.info("response: %s\n" % response)
log.info("Content-Length: %s" % response["ContentLength"])
log.info("s3_object_size: %s" % (config.obj_size * 1024 * 1024))
if response["ContentLength"] != config.obj_size * 1024 * 1024:
TestExecError("Content Lenght not matched")
def test_exec(config):
test_info = AddTestInfo('test with acls')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
s3_ops = ResourceOps()
# create user
config.user_count = 2
all_users_info = s3lib.create_users(config.user_count,
config.cluster_name)
u1 = all_users_info[0]
u2 = all_users_info[1]
# authenticate
u1_auth = Auth(u1)
u1_rgw_conn = u1_auth.do_auth()
u2_auth = Auth(u2)
u2_rgw_conn = u2_auth.do_auth()
no_of_buckets_to_create = 3
u1_buckets = []
u2_buckets = []
for i in range(no_of_buckets_to_create):
u1_bucket = create_bucket(u1_rgw_conn, u1, rand_no=i)
log.info('u1_bucket_name: %s' % u1_bucket.name)
u1_buckets.append(u1_bucket)
u2_bucket = create_bucket(u2_rgw_conn, u2, rand_no=i)
log.info('u2_bucket_name: %s' % u2_bucket.name)
u2_buckets.append(u2_bucket)
# test_acls_private(u1_rgw_conn, u1, u2, u1_buckets[0], u2_buckets[0])
test_acls_public_write(u1_rgw_conn, u1, u2, u1_buckets[1],
u2_buckets[1])
# test_acls_public_read(u1_rgw_conn, u1, u2, u1_buckets[2], u2_buckets[2])
# print u1_bucket_info.delete()
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
test_info = AddTestInfo('create users')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
all_users_info = s3lib.create_users(config.user_count, config.cluster_name)
with open('user_details', 'w') as fout:
json.dump(all_users_info, fout)
test_info.success_status('user creation completed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('user creation failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('user creation failed')
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
write_bucket_io_info = BucketIoInfo()
write_key_io_info = KeyIoInfo()
io_info_initialize.initialize(basic_io_structure.initial())
# create user
all_users_info = s3lib.create_users(config.user_count)
extra_user = s3lib.create_users(1)[0]
extra_user_auth = Auth(extra_user, ssl=config.ssl)
extra_user_conn = extra_user_auth.do_auth()
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
s3_object_names = []
# create buckets
log.info('no of buckets to create: %s' % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc)
log.info('creating bucket with name: %s' % bucket_name_to_create)
# bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create)
bucket = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Bucket',
'args': [bucket_name_to_create]})
# created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']})
created = s3lib.resource_op({'obj': bucket,
'resource': 'create',
'args': None,
'extra_info': {'access_key': each_user['access_key']}})
if created is False:
raise TestExecError("Resource execution failed: bucket creation faield")
if created is not None:
response = HttpResponseParser(created)
if response.status_code == 200:
log.info('bucket created')
else:
raise TestExecError("bucket creation failed")
else:
raise TestExecError("bucket creation failed")
# getting bucket version object
if config.test_ops['enable_version'] is True:
log.info('bucket versionig test on bucket: %s' % bucket.name)
# bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name)
bucket_versioning = s3lib.resource_op({'obj': rgw_conn,
'resource': 'BucketVersioning',
'args': [bucket.name]})
# checking the versioning status
# version_status = s3_ops.resource_op(bucket_versioning, 'status')
version_status = s3lib.resource_op({'obj': bucket_versioning,
'resource': 'status',
'args': None
})
if version_status is None:
log.info('bucket versioning still not enabled')
# enabling bucket versioning
# version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable')
version_enable_status = s3lib.resource_op({'obj': bucket_versioning,
'resource': 'enable',
'args': None,
})
response = HttpResponseParser(version_enable_status)
if response.status_code == 200:
log.info('version enabled')
write_bucket_io_info.add_versioning_status(each_user['access_key'],bucket.name,
VERSIONING_STATUS['ENABLED'])
else:
raise TestExecError("version enable failed")
if config.objects_count > 0:
log.info('s3 objects to create: %s' % config.objects_count)
for oc, s3_object_size in list(config.mapped_sizes.items()):
# versioning upload
s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc))
s3_object_names.append(s3_object_name)
log.info('s3 object name: %s' % s3_object_name)
log.info('versioning count: %s' % config.version_count)
s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc))
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
original_data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if original_data_info is False:
TestExecError("data creation failed")
created_versions_count = 0
for vc in range(config.version_count):
log.info('version count for %s is %s' % (s3_object_name, str(vc)))
log.info('modifying data: %s' % s3_object_name)
modified_data_info = manage_data.io_generator(s3_object_path, s3_object_size,
op='append',
**{'message': '\nhello for version: %s\n'
% str(vc)})
if modified_data_info is False:
TestExecError("data modification failed")
log.info('uploading s3 object: %s' % s3_object_path)
upload_info = dict({'access_key': each_user['access_key'],
'versioning_status': VERSIONING_STATUS['ENABLED'],
'version_count_no': vc}, **modified_data_info)
s3_obj = s3lib.resource_op({'obj': bucket,
'resource': 'Object',
'args': [s3_object_name],
'extra_info': upload_info, })
object_uploaded_status = s3lib.resource_op({'obj': s3_obj,
'resource': 'upload_file',
'args': [modified_data_info['name']],
'extra_info': upload_info})
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info('object uploaded')
s3_obj = rgw_conn.Object(bucket.name, s3_object_name)
log.info('current_version_id: %s' % s3_obj.version_id)
key_version_info = basic_io_structure.version_info(
**{'version_id': s3_obj.version_id,
'md5_local': upload_info['md5'],
'count_no': vc,
'size': upload_info['size']})
log.info('key_version_info: %s' % key_version_info)
write_key_io_info.add_versioning_info(each_user['access_key'], bucket.name,
s3_object_path, key_version_info)
created_versions_count += 1
log.info('created_versions_count: %s' % created_versions_count)
log.info('adding metadata')
metadata1 = {"m_data1": "this is the meta1 for this obj"}
s3_obj.metadata.update(metadata1)
metadata2 = {"m_data2": "this is the meta2 for this obj"}
s3_obj.metadata.update(metadata2)
log.info('metadata for this object: %s' % s3_obj.metadata)
log.info('metadata count for object: %s' % (len(s3_obj.metadata)))
if not s3_obj.metadata:
raise TestExecError('metadata not created even adding metadata')
versions = bucket.object_versions.filter(Prefix=s3_object_name)
created_versions_count_from_s3 = len([v.version_id for v in versions])
log.info('created versions count on s3: %s' % created_versions_count_from_s3)
if created_versions_count is created_versions_count_from_s3:
log.info('no new versions are created when added metdata')
else:
raise TestExecError("version count missmatch, "
"possible creation of version on adding metadata")
s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download")
object_downloaded_status = s3lib.resource_op({'obj': bucket,
'resource': 'download_file',
'args': [s3_object_name,
s3_object_download_path],
})
if object_downloaded_status is False:
raise TestExecError("Resource execution failed: object download failed")
if object_downloaded_status is None:
log.info('object downloaded')
# checking md5 of the downloaded file
s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path)
log.info('downloaded_md5: %s' % s3_object_downloaded_md5)
log.info('uploaded_md5: %s' % modified_data_info['md5'])
# tail_op = utils.exec_shell_cmd('tail -l %s' % s3_object_download_path)
log.info('all versions for the object: %s\n' % s3_object_name)
versions = bucket.object_versions.filter(Prefix=s3_object_name)
for version in versions:
log.info('key_name: %s --> version_id: %s' % (version.object_key, version.version_id))
if config.test_ops.get('set_acl', None) is True:
s3_obj_acl = s3lib.resource_op({'obj': rgw_conn,
'resource': 'ObjectAcl',
'args': [bucket.name, s3_object_name]})
# setting acl to private, just need to set to any acl and
# check if its set - check by response code
acls_set_status = s3_obj_acl.put(ACL='private')
response = HttpResponseParser(acls_set_status)
if response.status_code == 200:
log.info('ACLs set')
else:
raise TestExecError("Acls not Set")
# get obj details based on version id
for version in versions:
log.info('getting info for version id: %s' % version.version_id)
obj = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('obj get detils :%s\n' % (obj.get(VersionId=version.version_id)))
if config.test_ops['copy_to_version'] is True:
# reverting object to one of the versions ( randomly chosen )
version_id_to_copy = random.choice([v.version_id for v in versions])
log.info('version_id_to_copy: %s' % version_id_to_copy)
s3_obj = rgw_conn.Object(bucket.name, s3_object_name)
log.info('current version_id: %s' % s3_obj.version_id)
copy_response = s3_obj.copy_from(CopySource={'Bucket': bucket.name,
'Key': s3_object_name,
'VersionId': version_id_to_copy})
log.info('copy_response: %s' % copy_response)
if copy_response is None:
raise TestExecError("copy object from version id failed")
# current_version_id = copy_response['VersionID']
log.info('current_version_id: %s' % s3_obj.version_id)
# delete the version_id_to_copy object
s3_obj.delete(VersionId=version_id_to_copy)
log.info('all versions for the object after the copy operation: %s\n' % s3_object_name)
for version in versions:
log.info(
'key_name: %s --> version_id: %s' % (version.object_key, version.version_id))
# log.info('downloading current s3object: %s' % s3_object_name)
# s3_obj.download_file(s3_object_name + ".download")
if config.test_ops['delete_object_versions'] is True:
log.info('deleting s3_obj keys and its versions')
s3_obj = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('deleting versions for s3 obj: %s' % s3_object_name)
for version in versions:
log.info('trying to delete obj version: %s' % version.version_id)
del_obj_version = s3lib.resource_op({'obj': s3_obj,
'resource': 'delete',
'kwargs': dict(VersionId=version.version_id)})
log.info('response:\n%s' % del_obj_version)
if del_obj_version is not None:
response = HttpResponseParser(del_obj_version)
if response.status_code == 204:
log.info('version deleted ')
write_key_io_info.delete_version_info(each_user['access_key'], bucket.name,
s3_object_path, version.version_id)
else:
raise TestExecError("version deletion failed")
else:
raise TestExecError("version deletion failed")
log.info('available versions for the object')
versions = bucket.object_versions.filter(Prefix=s3_object_name)
for version in versions:
log.info('key_name: %s --> version_id: %s' % (
version.object_key, version.version_id))
if config.test_ops.get('delete_from_extra_user') is True:
log.info('trying to delete objects from extra user')
s3_obj = s3lib.resource_op({'obj': extra_user_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('deleting versions for s3 obj: %s' % s3_object_name)
for version in versions:
log.info('trying to delete obj version: %s' % version.version_id)
del_obj_version = s3lib.resource_op({'obj': s3_obj,
'resource': 'delete',
'kwargs': dict(
VersionId=version.version_id)})
log.info('response:\n%s' % del_obj_version)
if del_obj_version is not False:
response = HttpResponseParser(del_obj_version)
if response.status_code == 204:
log.info('version deleted ')
write_key_io_info.delete_version_info(each_user['access_key'],
bucket.name,
s3_object_path,
version.version_id)
raise TestExecError("version and deleted, this should not happen")
else:
log.info('version did not delete, expected behaviour')
else:
log.info('version did not delete, expected behaviour')
if config.local_file_delete is True:
log.info('deleting local file')
utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_path)
if config.test_ops['suspend_version'] is True:
log.info('suspending versioning')
# suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend')
suspend_version_status = s3lib.resource_op({'obj': bucket_versioning,
'resource': 'suspend',
'args': None})
response = HttpResponseParser(suspend_version_status)
if response.status_code == 200:
log.info('versioning suspended')
write_bucket_io_info.add_versioning_status(each_user['access_key'], bucket.name,
VERSIONING_STATUS['SUSPENDED'])
else:
raise TestExecError("version suspend failed")
# getting all objects in the bucket
log.info('getting all objects in the bucket')
objects = s3lib.resource_op({'obj': bucket,
'resource': 'objects',
'args': None})
log.info('objects :%s' % objects)
all_objects = s3lib.resource_op({'obj': objects,
'resource': 'all',
'args': None})
log.info('all objects: %s' % all_objects)
log.info('all objects2 :%s ' % bucket.objects.all())
for obj in all_objects:
log.info('object_name: %s' % obj.key)
versions = bucket.object_versions.filter(Prefix=obj.key)
log.info('displaying all versions of the object')
for version in versions:
log.info(
'key_name: %s --> version_id: %s' % (version.object_key, version.version_id))
if config.test_ops.get('suspend_from_extra_user') is True:
log.info('suspending versioning from extra user')
# suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend')
bucket_versioning = s3lib.resource_op({'obj': extra_user_conn,
'resource': 'BucketVersioning',
'args': [bucket.name]})
suspend_version_status = s3lib.resource_op({'obj': bucket_versioning,
'resource': 'suspend',
'args': None})
if suspend_version_status is not False:
response = HttpResponseParser(suspend_version_status)
if response.status_code == 200:
log.info('versioning suspended')
write_bucket_io_info.add_versioning_status(each_user['access_key'], bucket.name,
VERSIONING_STATUS['SUSPENDED'])
raise TestExecError('version suspended, this should not happen')
else:
log.info('versioning not suspended, expected behaviour')
if config.test_ops.get('upload_after_suspend') is True:
log.info('trying to upload after suspending versioning on bucket')
for oc, s3_object_size in list(config.mapped_sizes.items()):
# non versioning upload
s3_object_name = s3_object_names[oc] + ".after_version_suspending"
log.info('s3 object name: %s' % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
non_version_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op="append",
**{
'message': '\nhello for non version\n'})
if non_version_data_info is False:
TestExecError("data creation failed")
log.info('uploading s3 object: %s' % s3_object_path)
upload_info = dict({'access_key': each_user['access_key'],
'versioning_status': 'suspended'},**non_version_data_info)
s3_obj = s3lib.resource_op({'obj': bucket,
'resource': 'Object',
'args': [s3_object_name],
'extra_info': upload_info})
object_uploaded_status = s3lib.resource_op({'obj': s3_obj,
'resource': 'upload_file',
'args': [non_version_data_info['name']],
'extra_info': upload_info})
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info('object uploaded')
s3_obj = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('version_id: %s' % s3_obj.version_id)
if s3_obj.version_id is None:
log.info('Versions are not created after suspending')
else:
raise TestExecError('Versions are created even after suspending')
s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download")
object_downloaded_status = s3lib.resource_op({'obj': bucket,
'resource': 'download_file',
'args': [s3_object_name,
s3_object_download_path],
})
if object_downloaded_status is False:
raise TestExecError("Resource execution failed: object download failed")
if object_downloaded_status is None:
log.info('object downloaded')
# checking md5 of the downloaded file
s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path)
log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5)
log.info('s3_object_uploaded_md5: %s' % non_version_data_info['md5'])
if config.local_file_delete is True:
utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_path)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
ceph_config_set = CephConfOp()
rgw_service = RGWService()
if config.sts is None:
raise TestExecError("sts policies are missing in yaml config")
# create users
config.user_count = 2
users_info = s3lib.create_users(config.user_count)
# user1 is the owner
user1, user2 = users_info[0], users_info[1]
log.info("adding sts config to ceph.conf")
sesison_encryption_token = "abcdefghijklmnoq"
ceph_config_set.set_to_ceph_conf(
"global", ConfigOpts.rgw_sts_key, sesison_encryption_token
)
ceph_config_set.set_to_ceph_conf("global", ConfigOpts.rgw_s3_auth_use_sts, "True")
srv_restarted = rgw_service.restart()
time.sleep(30)
if srv_restarted is False:
raise TestExecError("RGW service restart failed")
else:
log.info("RGW service restarted")
auth = Auth(user1, ssl=config.ssl)
iam_client = auth.do_auth_iam_client()
policy_document = json.dumps(config.sts["policy_document"]).replace(" ", "")
policy_document = policy_document.replace("<user_name>", user2["user_id"])
role_policy = json.dumps(config.sts["role_policy"]).replace(" ", "")
add_caps_cmd = (
'sudo radosgw-admin caps add --uid="{user_id}" --caps="roles=*"'.format(
user_id=user1["user_id"]
)
)
utils.exec_shell_cmd(add_caps_cmd)
role_name = f"S3RoleOf.{user1['user_id']}"
log.info(f"role_name: {role_name}")
log.info("creating role")
create_role_response = iam_client.create_role(
AssumeRolePolicyDocument=policy_document,
Path="/",
RoleName=role_name,
)
log.info("create_role_response")
log.info(create_role_response)
policy_name = f"policy.{user1['user_id']}"
log.info(f"policy_name: {policy_name}")
log.info("putting role policy")
put_policy_response = iam_client.put_role_policy(
RoleName=role_name, PolicyName=policy_name, PolicyDocument=role_policy
)
log.info("put_policy_response")
log.info(put_policy_response)
auth = Auth(user2, ssl=config.ssl)
sts_client = auth.do_auth_sts_client()
log.info("assuming role")
assume_role_response = sts_client.assume_role(
RoleArn=create_role_response["Role"]["Arn"],
RoleSessionName=user1["user_id"],
DurationSeconds=3600,
)
log.info(assume_role_response)
assumed_role_user_info = {
"access_key": assume_role_response["Credentials"]["AccessKeyId"],
"secret_key": assume_role_response["Credentials"]["SecretAccessKey"],
"session_token": assume_role_response["Credentials"]["SessionToken"],
"user_id": user2["user_id"],
}
log.info("got the credentials after assume role")
s3client = Auth(assumed_role_user_info, ssl=config.ssl)
s3_client_rgw = s3client.do_auth()
io_info_initialize.initialize(basic_io_structure.initial())
write_user_info = AddUserInfo()
basic_io_structure = BasicIOInfoStructure()
user_info = basic_io_structure.user(
**{
"user_id": assumed_role_user_info["user_id"],
"access_key": assumed_role_user_info["access_key"],
"secret_key": assumed_role_user_info["secret_key"],
}
)
write_user_info.add_user_info(user_info)
buckets_created = []
if config.test_ops["create_bucket"] is True:
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(
assumed_role_user_info["user_id"], rand_no=bc
)
log.info("creating bucket with name: %s" % bucket_name)
bucket = reusable.create_bucket(
bucket_name, s3_client_rgw, assumed_role_user_info
)
buckets_created.append(bucket)
if config.test_ops["create_object"] is True:
for bucket in buckets_created:
# uploading data
log.info("s3 objects to create: %s" % config.objects_count)
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(bucket.name, oc)
log.info("s3 object name: %s" % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info("s3 object path: %s" % s3_object_path)
if config.test_ops.get("upload_type") == "multipart":
log.info("upload type: multipart")
reusable.upload_mutipart_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
assumed_role_user_info,
)
else:
log.info("upload type: normal")
reusable.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
assumed_role_user_info,
)
if config.test_ops["server_side_copy"] is True:
bucket1, bucket2 = buckets_created
# copy object1 from bucket1 to bucket2 with the same name as in bucket1
log.info("copying first object from bucket1 to bucket2")
all_keys_in_buck1 = []
for obj in bucket1.objects.all():
all_keys_in_buck1.append(obj.key)
copy_source = {"Bucket": bucket1.name, "Key": all_keys_in_buck1[0]}
copy_object_name = all_keys_in_buck1[0] + "_copied_obj"
log.info(f"copy object name: {copy_object_name}")
bucket2.copy(copy_source, copy_object_name)
# list the objects in bucket2
log.info("listing all objects im bucket2 after copy")
all_bucket2_objs = []
for obj in bucket2.objects.all():
log.info(obj.key)
all_bucket2_objs.append(obj.key)
# check for object existence in bucket2
if copy_object_name in all_bucket2_objs:
log.info("server side copy successful")
else:
raise TestExecError("server side copy operation was not successful")
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
test_info = AddTestInfo("create m buckets with n objects with bucket life cycle")
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
# create user
all_users_info = s3lib.create_users(config.user_count, config.cluster_name)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
# create buckets
if config.test_ops["create_bucket"] is True:
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=1
)
bucket = resuables.create_bucket(bucket_name, rgw_conn, each_user)
if config.test_ops["create_object"] is True:
# uploading data
log.info("s3 objects to create: %s" % config.objects_count)
for oc in range(config.objects_count):
s3_object_name = utils.gen_s3_object_name(bucket.name, oc)
resuables.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
)
bucket_life_cycle = s3lib.resource_op(
{
"obj": rgw_conn,
"resource": "BucketLifecycleConfiguration",
"args": [bucket.name],
}
)
life_cycle = basic_lifecycle_config(
prefix="key", days=20, id="rul1"
)
put_bucket_life_cycle = s3lib.resource_op(
{
"obj": bucket_life_cycle,
"resource": "put",
"kwargs": dict(LifecycleConfiguration=life_cycle),
}
)
log.info("put bucket life cycle:\n%s" % put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError("bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op(
{
"obj": rgw_conn2,
"resource": "get_bucket_lifecycle_configuration",
"kwargs": dict(Bucket=bucket.name),
}
)
if get_bucket_life_cycle_config is False:
raise TestExecError("bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(get_bucket_life_cycle_config)
if response.status_code == 200:
log.info("bucket life cycle retrieved")
else:
raise TestExecError(
"bucket lifecycle config retrieval failed"
)
else:
raise TestExecError("bucket life cycle retrieved")
test_info.success_status("test passed")
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
non_ten_buckets = {}
ten_buckets = {}
user_names = ['bill', 'newbill', 'joe', 'newjoe']
tenant1 = 'tenant'
non_ten_users = s3lib.create_users(config.user_count)
ten_users = s3lib.create_tenant_users(config.user_count, tenant1)
# Rename users
if config.test_ops['rename_users'] is True:
for user in non_ten_users:
new_non_ten_name = 'new' + user['user_id']
out = reusable.rename_user(user['user_id'], new_non_ten_name)
if out is False:
raise TestExecError("RGW User rename error")
log.info('output :%s' % out)
user['user_id'] = new_non_ten_name
for ten_user in ten_users:
new_ten_name = 'new' + ten_user['user_id']
out1 = reusable.rename_user(ten_user['user_id'], new_ten_name,
tenant1)
if out1 is False:
raise TestExecError("RGW User rename error")
log.info('output :%s' % out1)
ten_user['user_id'] = new_ten_name
# create buckets and test rename
for user in non_ten_users:
auth = Auth(user, ssl=config.ssl)
rgw_conn = auth.do_auth()
bucket_name_to_create1 = utils.gen_bucket_name_from_userid(
user['user_id'])
log.info('creating bucket with name: %s' % bucket_name_to_create1)
bucket = reusable.create_bucket(bucket_name_to_create1, rgw_conn, user)
non_ten_buckets[user['user_id']] = bucket_name_to_create1
if config.test_ops['rename_buckets'] is True:
bucket_new_name1 = 'new' + bucket_name_to_create1
non_ten_buckets[user['user_id']] = bucket_new_name1
out2 = reusable.rename_bucket(bucket.name, bucket_new_name1,
user['user_id'])
if out2 is False:
raise TestExecError("RGW Bucket rename error")
log.info('output :%s' % out2)
for ten_user in ten_users:
auth = Auth(ten_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
bucket_name_to_create2 = utils.gen_bucket_name_from_userid(
ten_user['user_id'])
log.info('creating bucket with name: %s' % bucket_name_to_create2)
bucket = reusable.create_bucket(bucket_name_to_create2, rgw_conn,
ten_user)
ten_buckets[ten_user['user_id']] = bucket_name_to_create2
if config.test_ops['rename_buckets'] is True:
bucket_new_name2 = 'new' + bucket_name_to_create2
ten_buckets[ten_user['user_id']] = bucket_new_name2
out3 = reusable.rename_bucket(bucket.name, bucket_new_name2,
ten_user['user_id'], tenant1)
if out3 is False:
raise TestExecError("RGW Bucket rename error")
log.info('output :%s' % out3)
if config.test_ops['bucket_link_unlink'] is True:
# Bucket unlink and link from non tenanted to tenanted users
out4 = reusable.unlink_bucket(
non_ten_users[0]['user_id'],
non_ten_buckets[non_ten_users[0]['user_id']])
if out4 is False:
raise TestExecError("RGW Bucket unlink error")
log.info('output :%s' % out4)
reusable.link_chown_to_tenanted(
ten_users[0]['user_id'],
non_ten_buckets[non_ten_users[0]['user_id']], tenant1)
# Bucket unlink and link from tenanted to non tenanted users
out5 = reusable.unlink_bucket(ten_users[0]['user_id'],
ten_buckets[ten_users[0]['user_id']],
tenant1)
if out5 is False:
raise TestExecError("RGW Bucket unlink error")
log.info('output :%s' % out5)
reusable.link_chown_to_nontenanted(
non_ten_users[0]['user_id'], ten_buckets[ten_users[0]['user_id']],
tenant1)
def test_exec(config, requester):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
log.info("requester type: %s" % requester)
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
# create buckets
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=bc
)
log.info("creating bucket with name: %s" % bucket_name_to_create)
# bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create)
bucket = reusable.create_bucket(
bucket_name=bucket_name_to_create, rgw=rgw_conn, user_info=each_user
)
bucket_request_payer = s3lib.resource_op(
{
"obj": rgw_conn,
"resource": "BucketRequestPayment",
"args": [bucket.name],
}
)
# change the bucket request payer to 'requester'
payer = {"Payer": requester}
response = s3lib.resource_op(
{
"obj": bucket_request_payer,
"resource": "put",
"kwargs": dict(RequestPaymentConfiguration=payer),
}
)
log.info(response)
if response is not None:
response = HttpResponseParser(response)
if response.status_code == 200:
log.info("bucket created")
else:
raise TestExecError("bucket request payer modification failed")
else:
raise TestExecError("bucket request payer modification failed")
payer = bucket_request_payer.payer
log.info("bucket request payer: %s" % payer)
if payer != "Requester":
TestExecError("Request payer is not set or changed properly ")
log.info("s3 objects to create: %s" % config.objects_count)
if config.objects_count is not None:
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(bucket.name, oc)
reusable.upload_object(
s3_object_name, bucket, TEST_DATA_PATH, config, each_user
)
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
# create buckets
if config.test_ops["create_bucket"] is True:
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=1)
bucket = reusable.create_bucket(bucket_name, rgw_conn,
each_user)
if config.test_ops["enable_versioning"] is True:
log.info("bucket versionig test on bucket: %s" %
bucket.name)
# bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name)
bucket_versioning = s3lib.resource_op({
"obj":
rgw_conn,
"resource":
"BucketVersioning",
"args": [bucket.name],
})
version_status = s3lib.resource_op({
"obj": bucket_versioning,
"resource": "status",
"args": None
})
if version_status is None:
log.info("bucket versioning still not enabled")
# enabling bucket versioning
version_enable_status = s3lib.resource_op({
"obj": bucket_versioning,
"resource": "enable",
"args": None
})
response = HttpResponseParser(version_enable_status)
if response.status_code == 200:
log.info("version enabled")
else:
raise TestExecError("version enable failed")
if config.test_ops["create_object"] is True:
# upload data
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(
bucket.name, oc)
if config.test_ops["version_count"] > 0:
for vc in range(config.test_ops["version_count"]):
log.info("version count for %s is %s" %
(s3_object_name, str(vc)))
log.info("modifying data: %s" % s3_object_name)
reusable.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
append_data=True,
append_msg="hello object for version: %s\n"
% str(vc),
)
else:
log.info("s3 objects to create: %s" %
config.objects_count)
reusable.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
)
bucket_life_cycle = s3lib.resource_op({
"obj": rgw_conn,
"resource": "BucketLifecycleConfiguration",
"args": [bucket.name],
})
life_cycle = basic_lifecycle_config(prefix="key",
days=20,
id="rul1")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(LifecycleConfiguration=life_cycle),
})
log.info("put bucket life cycle:\n%s" % put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield")
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError("bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
"get_bucket_lifecycle_configuration",
"kwargs":
dict(Bucket=bucket.name),
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(get_bucket_life_cycle_config)
if response.status_code == 200:
log.info("bucket life cycle retrieved")
else:
raise TestExecError(
"bucket lifecycle config retrieval failed")
else:
raise TestExecError("bucket life cycle retrieved")
if config.test_ops["create_object"] is True:
for oc in range(config.objects_count):
s3_object_name = utils.gen_s3_object_name(
bucket.name, oc)
if config.test_ops["version_count"] > 0:
if (config.test_ops.get("delete_versioned_object",
None) is True):
log.info(
"list all the versions of the object and delete the "
"current version of the object")
log.info("all versions for the object: %s\n" %
s3_object_name)
versions = bucket.object_versions.filter(
Prefix=s3_object_name)
t1 = []
for version in versions:
log.info(
"key_name: %s --> version_id: %s" %
(version.object_key,
version.version_id))
t1.append(version.version_id)
s3_object = s3lib.resource_op({
"obj":
rgw_conn,
"resource":
"Object",
"args": [bucket.name, s3_object_name],
})
# log.info('object version to delete: %s -> %s' % (versions[0].object_key,
# versions[0].version_id))
delete_response = s3_object.delete()
log.info("delete response: %s" %
delete_response)
if delete_response["DeleteMarker"] is True:
log.info(
"object delete marker is set to true")
else:
raise TestExecError(
"'object delete marker is set to false"
)
log.info(
"available versions for the object after delete marker is set"
)
t2 = []
versions_after_delete_marker_is_set = (
bucket.object_versions.filter(
Prefix=s3_object_name))
for version in versions_after_delete_marker_is_set:
log.info(
"key_name: %s --> version_id: %s" %
(version.object_key,
version.version_id))
t2.append(version.version_id)
t2.pop()
if t1 == t2:
log.info("versions remained intact")
else:
raise TestExecError(
"versions are not intact after delete marker is set"
)
# modify bucket lifecycle configuration, modify expiration days here for the test case.
if config.test_ops.get("modify_lifecycle", False) is True:
log.info("modifying lifecycle configuration")
life_cycle_modifed = basic_lifecycle_config(
prefix="key", days=15, id="rul1", status="Disabled")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(LifecycleConfiguration=life_cycle_modifed),
})
log.info("put bucket life cycle:\n%s" %
put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError(
"bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
"get_bucket_lifecycle_configuration",
"kwargs":
dict(Bucket=bucket.name),
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(
get_bucket_life_cycle_config)
modified_expiration_days = get_bucket_life_cycle_config[
"Rules"][0]["Expiration"]["Days"]
log.info("modified expiration days: %s" %
modified_expiration_days)
if (response.status_code == 200
and modified_expiration_days == 15):
log.info(
"bucket life cycle retrieved after modifying")
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after modifying"
)
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after modifying"
)
# disable bucket lifecycle configuration
if config.test_ops.get("disable_lifecycle", False) is True:
log.info("disabling lifecycle configuration")
life_cycle_disabled_config = basic_lifecycle_config(
prefix="key", days=20, id="rul1", status="Disabled")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(
LifecycleConfiguration=life_cycle_disabled_config),
})
log.info("put bucket life cycle:\n%s" %
put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError(
"bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
"get_bucket_lifecycle_configuration",
"kwargs":
dict(Bucket=bucket.name),
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(
get_bucket_life_cycle_config)
if (response.status_code == 200
and get_bucket_life_cycle_config["Rules"][0]
["Status"] == "Disabled"):
log.info("disabled_status: %s" %
get_bucket_life_cycle_config["Rules"][0]
["Status"])
log.info(
"bucket life cycle retrieved after disabled")
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after disabled"
)
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after disabled"
)
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
write_bucket_io_info = BucketIoInfo()
io_info_initialize.initialize(basic_io_structure.initial())
ceph_conf = CephConfOp()
rgw_service = RGWService()
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
# authenticate sns client.
rgw_sns_conn = auth.do_auth_sns_client()
# authenticate with s3 client
rgw_s3_client = auth.do_auth_using_client()
# get ceph version
ceph_version_id, ceph_version_name = utils.get_ceph_version()
objects_created_list = []
if config.test_ops["create_bucket"] is True:
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=bc)
bucket = reusable.create_bucket(bucket_name_to_create,
rgw_conn, each_user)
if config.test_ops.get("enable_version", False):
log.info("enable bucket version")
reusable.enable_versioning(bucket, rgw_conn, each_user,
write_bucket_io_info)
# create topic with endpoint
if config.test_ops["create_topic"] is True:
endpoint = config.test_ops.get("endpoint")
ack_type = config.test_ops.get("ack_type")
topic_id = str(uuid.uuid4().hex[:16])
persistent = False
topic_name = "cephci-kafka-" + ack_type + "-ack-type-" + topic_id
log.info(
f"creating a topic with {endpoint} endpoint with ack type {ack_type}"
)
if config.test_ops.get("persistent_flag", False):
log.info("topic with peristent flag enabled")
persistent = config.test_ops.get("persistent_flag")
topic = notification.create_topic(rgw_sns_conn, endpoint,
ack_type, topic_name,
persistent)
# get topic attributes
if config.test_ops.get("get_topic_info", False):
log.info("get topic attributes")
get_topic_info = notification.get_topic(
rgw_sns_conn, topic, ceph_version_name)
# put bucket notification with topic configured for event
if config.test_ops["put_get_bucket_notification"] is True:
event = config.test_ops.get("event_type")
notification_name = "notification-" + str(event)
notification.put_bucket_notification(
rgw_s3_client,
bucket_name_to_create,
notification_name,
topic,
event,
)
# get bucket notification
log.info(
f"get bucket notification for bucket : {bucket_name_to_create}"
)
notification.get_bucket_notification(
rgw_s3_client, bucket_name_to_create)
# create objects
if config.test_ops["create_object"] is True:
# uploading data
log.info("s3 objects to create: %s" % config.objects_count)
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(
bucket_name_to_create, oc)
log.info("s3 object name: %s" % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH,
s3_object_name)
log.info("s3 object path: %s" % s3_object_path)
if config.test_ops.get("upload_type") == "multipart":
log.info("upload type: multipart")
reusable.upload_mutipart_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
)
else:
log.info("upload type: normal")
reusable.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
)
# copy objects
if config.test_ops.get("copy_object", False):
log.info("copy object")
status = rgw_s3_client.copy_object(
Bucket=bucket_name_to_create,
Key="copy_of_object" + s3_object_name,
CopySource={
"Bucket": bucket_name_to_create,
"Key": s3_object_name,
},
)
if status is None:
raise TestExecError("copy object failed")
# delete objects
if config.test_ops.get("delete_bucket_object", False):
if config.test_ops.get("enable_version", False):
for name, path in objects_created_list:
reusable.delete_version_object(bucket, name, path,
rgw_conn, each_user)
else:
reusable.delete_objects(bucket)
# start kafka broker and consumer
event_record_path = "/home/cephuser/event_record"
start_consumer = notification.start_kafka_broker_consumer(
topic_name, event_record_path)
if start_consumer is False:
raise TestExecError("Kafka consumer not running")
# verify all the attributes of the event record. if event not received abort testcase
log.info("verify event record attributes")
verify = notification.verify_event_record(event,
bucket_name_to_create,
event_record_path,
ceph_version_name)
if verify is False:
raise EventRecordDataError(
"Event record is empty! notification is not seen")
# delete topic logs on kafka broker
notification.del_topic_from_kafka_broker(topic_name)
# check sync status if a multisite cluster
reusable.check_sync_status()
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
test_info = AddTestInfo('test versioning with objects')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
write_bucket_io_info = BucketIoInfo()
write_key_io_info = KeyIoInfo()
try:
test_info.started_info()
version_count = 3
# create user
s3_user = s3lib.create_users(1)[0]
# authenticate
auth = Auth(s3_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
b1_name = 'bucky.1e' # bucket 1
b1_k1_name = b1_name + ".key.1" # key1
b1_k2_name = b1_name + ".key.2" # key2
b2_name = 'bucky.2e' # bucket 2
b2_k1_name = b2_name + ".key.1" # key1
b2_k2_name = b2_name + ".key.2" # key2
b1 = resuables.create_bucket(b1_name, rgw_conn, s3_user)
b2 = resuables.create_bucket(b2_name, rgw_conn, s3_user)
# enable versioning on b1
resuables.enable_versioning(b1, rgw_conn, s3_user,
write_bucket_io_info)
# upload object to version enabled bucket b1
obj_sizes = list(config.mapped_sizes.values())
config.obj_size = obj_sizes[0]
for vc in range(version_count):
resuables.upload_object(b1_k1_name,
b1,
TEST_DATA_PATH,
config,
s3_user,
append_data=True,
append_msg='hello vc count: %s' % str(vc))
# upload object to non version bucket b2
config.obj_size = obj_sizes[1]
resuables.upload_object(b2_k1_name, b2, TEST_DATA_PATH, config,
s3_user)
# copy b2_k1 to b1 and check if version id is created, expectation: version id should be created
# copy b1_k1 to b2 and check if version id is created, expectation: version id should not be present
b1_k2 = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'Object',
'args': [b1.name, b1_k2_name]
})
b2_k2 = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'Object',
'args': [b2.name, b2_k2_name]
})
log.info(
'copy from b2_k1 key to b1_k2 key to bucket 1 -> version enabled bucket'
)
copy_response = b1_k2.copy_from(CopySource={
'Bucket': b2.name,
'Key': b2_k1_name,
})
log.info('copy_response: %s' % copy_response)
if copy_response is None:
raise TestExecError("copy object failed")
log.info('checking if copies object has version id created')
b1_k2_version_id = b1_k2.version_id
log.info('version id: %s' % b1_k2_version_id)
if b1_k2_version_id is None:
raise TestExecError(
'Version ID not created for the copied object on to the versioned enabled bucket'
)
else:
log.info(
'Version ID created for the copied object on to the versioned bucket'
)
all_objects_in_b1 = b1.objects.all()
log.info('all objects in bucket 1')
for obj in all_objects_in_b1:
log.info('object_name: %s' % obj.key)
versions = b1.object_versions.filter(Prefix=obj.key)
log.info('displaying all versions of the object')
for version in versions:
log.info('key_name: %s --> version_id: %s' %
(version.object_key, version.version_id))
log.info('-------------------------------------------')
log.info(
'copy from b1_k1 key to b2_k2 to bucket 2 -> non version bucket')
copy_response = b2_k2.copy_from(CopySource={
'Bucket': b1.name,
'Key': b1_k1_name,
})
log.info('copy_response: %s' % copy_response)
if copy_response is None:
raise TestExecError("copy object failed")
log.info('checking if copies object has version id created')
b2_k2_version_id = b2_k2.version_id
log.info('version id: %s' % b2_k2_version_id)
if b2_k2_version_id is None:
log.info(
'Version ID not created for the copied object on to the non versioned bucket'
)
else:
raise TestExecError(
'Version ID created for the copied object on to the non versioned bucket'
)
all_objects_in_b2 = b2.objects.all()
log.info('all objects in bucket 2')
for obj in all_objects_in_b2:
log.info('object_name: %s' % obj.key)
versions = b2.object_versions.filter(Prefix=obj.key)
log.info('displaying all versions of the object')
for version in versions:
log.info('key_name: %s --> version_id: %s' %
(version.object_key, version.version_id))
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
write_bucket_io_info = BucketIoInfo()
io_info_initialize.initialize(basic_io_structure.initial())
ceph_conf = CephConfOp()
rgw_service = RGWService()
# check the default data log backing
default_data_log = reusable.get_default_datalog_type()
log.info(f"{default_data_log} is the default data log backing")
# check sync status if a multisite cluster
reusable.check_sync_status()
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
objects_created_list = []
# change the default datalog backing to FIFO
if config.test_ops.get("change_datalog_backing", False):
logtype = config.test_ops["change_datalog_backing"]
log.info(f"change default datalog backing to {logtype}")
cmd = f"radosgw-admin datalog type --log-type={logtype}"
change_datalog_type = utils.exec_shell_cmd(cmd)
if change_datalog_type is False:
raise TestExecError("Failed to change the datalog type to fifo")
log.info(
"restart the rgw daemons and sleep of 30secs for rgw daemon to be up "
)
srv_restarted = rgw_service.restart()
time.sleep(30)
if srv_restarted is False:
raise TestExecError("RGW service restart failed")
else:
log.info("RGW service restarted")
if config.test_ops["create_bucket"] is True:
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=bc
)
log.info("creating bucket with name: %s" % bucket_name_to_create)
bucket = reusable.create_bucket(
bucket_name_to_create, rgw_conn, each_user
)
if config.test_ops.get("enable_version", False):
log.info("enable bucket version")
reusable.enable_versioning(
bucket, rgw_conn, each_user, write_bucket_io_info
)
if config.test_ops["create_object"] is True:
# uploading data
log.info(
"top level s3 objects to create: %s" % config.objects_count
)
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(
bucket_name_to_create, oc
)
log.info("s3 object name: %s" % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info("s3 object path: %s" % s3_object_path)
if config.test_ops.get("upload_type") == "multipart":
log.info("upload type: multipart")
reusable.upload_mutipart_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
)
else:
log.info("upload type: normal")
reusable.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
)
objects_created_list.append((s3_object_name, s3_object_path))
# deleting the local file created after upload
if config.local_file_delete is True:
log.info("deleting local file created after the upload")
utils.exec_shell_cmd("rm -rf %s" % s3_object_path)
# delete object and bucket
if config.test_ops.get("delete_bucket_object", False):
if config.test_ops.get("enable_version", False):
for name, path in objects_created_list:
reusable.delete_version_object(
bucket, name, path, rgw_conn, each_user
)
else:
reusable.delete_objects(bucket)
time.sleep(30)
reusable.delete_bucket(bucket)
# check for any ERRORs in datalog list. ref- https://bugzilla.redhat.com/show_bug.cgi?id=1917687
error_in_data_log_list = reusable.check_datalog_list()
if error_in_data_log_list:
raise TestExecError("Error in datalog list")
# check for data log markers. ref: https://bugzilla.redhat.com/show_bug.cgi?id=1831798#c22
data_log_marker = reusable.check_datalog_marker()
log.info(f"The data_log_marker is: {data_log_marker}")
# check sync status if a multisite cluster
reusable.check_sync_status()
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
non_ten_buckets = {}
ten_buckets = {}
user_names = ["bill", "newbill", "joe", "newjoe"]
tenant1 = "tenant"
non_ten_users = s3lib.create_users(config.user_count)
ten_users = s3lib.create_tenant_users(config.user_count, tenant1)
# Rename users
if config.test_ops["rename_users"] is True:
for user in non_ten_users:
new_non_ten_name = "new" + user["user_id"]
out = reusable.rename_user(user["user_id"], new_non_ten_name)
if out is False:
raise TestExecError("RGW User rename error")
log.info("output :%s" % out)
user["user_id"] = new_non_ten_name
for ten_user in ten_users:
new_ten_name = "new" + ten_user["user_id"]
out1 = reusable.rename_user(ten_user["user_id"], new_ten_name,
tenant1)
if out1 is False:
raise TestExecError("RGW User rename error")
log.info("output :%s" % out1)
ten_user["user_id"] = new_ten_name
# create buckets and test rename
for user in non_ten_users:
auth = Auth(user, ssl=config.ssl)
rgw_conn = auth.do_auth()
bucket_name_to_create1 = utils.gen_bucket_name_from_userid(
user["user_id"])
log.info("creating bucket with name: %s" % bucket_name_to_create1)
bucket = reusable.create_bucket(bucket_name_to_create1, rgw_conn, user)
non_ten_buckets[user["user_id"]] = bucket_name_to_create1
if config.test_ops["rename_buckets"] is True:
bucket_new_name1 = "new" + bucket_name_to_create1
non_ten_buckets[user["user_id"]] = bucket_new_name1
out2 = reusable.rename_bucket(bucket.name, bucket_new_name1,
user["user_id"])
if out2 is False:
raise TestExecError("RGW Bucket rename error")
log.info("output :%s" % out2)
for ten_user in ten_users:
auth = Auth(ten_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
bucket_name_to_create2 = utils.gen_bucket_name_from_userid(
ten_user["user_id"])
log.info("creating bucket with name: %s" % bucket_name_to_create2)
bucket = reusable.create_bucket(bucket_name_to_create2, rgw_conn,
ten_user)
ten_buckets[ten_user["user_id"]] = bucket_name_to_create2
if config.test_ops["rename_buckets"] is True:
bucket_new_name2 = "new" + bucket_name_to_create2
ten_buckets[ten_user["user_id"]] = bucket_new_name2
out3 = reusable.rename_bucket(bucket.name, bucket_new_name2,
ten_user["user_id"], tenant1)
if out3 is False:
raise TestExecError("RGW Bucket rename error")
log.info("output :%s" % out3)
if config.test_ops["bucket_link_unlink"] is True:
# Bucket unlink and link from non tenanted to tenanted users
out4 = reusable.unlink_bucket(
non_ten_users[0]["user_id"],
non_ten_buckets[non_ten_users[0]["user_id"]])
if out4 is False:
raise TestExecError("RGW Bucket unlink error")
log.info("output :%s" % out4)
reusable.link_chown_to_tenanted(
ten_users[0]["user_id"],
non_ten_buckets[non_ten_users[0]["user_id"]],
tenant1,
)
# Bucket unlink and link from tenanted to non tenanted users
out5 = reusable.unlink_bucket(ten_users[0]["user_id"],
ten_buckets[ten_users[0]["user_id"]],
tenant1)
if out5 is False:
raise TestExecError("RGW Bucket unlink error")
log.info("output :%s" % out5)
reusable.link_chown_to_nontenanted(
non_ten_users[0]["user_id"], ten_buckets[ten_users[0]["user_id"]],
tenant1)
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
test_info = AddTestInfo(
'create m buckets with n objects with bucket life cycle')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
# create buckets
if config.test_ops['create_bucket'] is True:
log.info('no of buckets to create: %s' % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(
each_user['user_id'], rand_no=1)
bucket = resuables.create_bucket(bucket_name, rgw_conn,
each_user)
if config.test_ops['enable_versioning'] is True:
log.info('bucket versionig test on bucket: %s' %
bucket.name)
# bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name)
bucket_versioning = s3lib.resource_op({
'obj':
rgw_conn,
'resource':
'BucketVersioning',
'args': [bucket.name]
})
version_status = s3lib.resource_op({
'obj': bucket_versioning,
'resource': 'status',
'args': None
})
if version_status is None:
log.info('bucket versioning still not enabled')
# enabling bucket versioning
version_enable_status = s3lib.resource_op({
'obj': bucket_versioning,
'resource': 'enable',
'args': None
})
response = HttpResponseParser(version_enable_status)
if response.status_code == 200:
log.info('version enabled')
else:
raise TestExecError("version enable failed")
if config.test_ops['create_object'] is True:
# upload data
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(
bucket.name, oc)
if config.test_ops['version_count'] > 0:
for vc in range(
config.test_ops['version_count']):
log.info('version count for %s is %s' %
(s3_object_name, str(vc)))
log.info('modifying data: %s' %
s3_object_name)
resuables.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
append_data=True,
append_msg=
'hello object for version: %s\n' %
str(vc))
else:
log.info('s3 objects to create: %s' %
config.objects_count)
resuables.upload_object(
s3_object_name, bucket, TEST_DATA_PATH,
config, each_user)
bucket_life_cycle = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'BucketLifecycleConfiguration',
'args': [bucket.name]
})
life_cycle = basic_lifecycle_config(prefix="key",
days=20,
id="rul1")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(LifecycleConfiguration=life_cycle)
})
log.info('put bucket life cycle:\n%s' %
put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info('bucket life cycle added')
else:
raise TestExecError(
"bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info('trying to retrieve bucket lifecycle config')
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
'get_bucket_lifecycle_configuration',
"kwargs":
dict(Bucket=bucket.name)
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(
get_bucket_life_cycle_config)
if response.status_code == 200:
log.info('bucket life cycle retrieved')
else:
raise TestExecError(
"bucket lifecycle config retrieval failed")
else:
raise TestExecError("bucket life cycle retrieved")
if config.test_ops['create_object'] is True:
for oc in range(config.objects_count):
s3_object_name = utils.gen_s3_object_name(
bucket.name, oc)
if config.test_ops['version_count'] > 0:
if config.test_ops.get(
'delete_versioned_object',
None) is True:
log.info(
'list all the versions of the object and delete the '
'current version of the object')
log.info(
'all versions for the object: %s\n' %
s3_object_name)
versions = bucket.object_versions.filter(
Prefix=s3_object_name)
t1 = []
for version in versions:
log.info(
'key_name: %s --> version_id: %s' %
(version.object_key,
version.version_id))
t1.append(version.version_id)
s3_object = s3lib.resource_op({
'obj':
rgw_conn,
'resource':
'Object',
'args': [bucket.name, s3_object_name]
})
# log.info('object version to delete: %s -> %s' % (versions[0].object_key,
# versions[0].version_id))
delete_response = s3_object.delete()
log.info('delete response: %s' %
delete_response)
if delete_response['DeleteMarker'] is True:
log.info(
'object delete marker is set to true'
)
else:
raise TestExecError(
"'object delete marker is set to false"
)
log.info(
'available versions for the object after delete marker is set'
)
t2 = []
versions_after_delete_marker_is_set = bucket.object_versions.filter(
Prefix=s3_object_name)
for version in versions_after_delete_marker_is_set:
log.info(
'key_name: %s --> version_id: %s' %
(version.object_key,
version.version_id))
t2.append(version.version_id)
t2.pop()
if t1 == t2:
log.info('versions remained intact')
else:
raise TestExecError(
'versions are not intact after delete marker is set'
)
# modify bucket lifecycle configuration, modify expiration days here for the test case.
if config.test_ops.get('modify_lifecycle', False) is True:
log.info('modifying lifecycle configuration')
life_cycle_modifed = basic_lifecycle_config(
prefix="key",
days=15,
id="rul1",
status="Disabled")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(LifecycleConfiguration=life_cycle_modifed)
})
log.info('put bucket life cycle:\n%s' %
put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(
put_bucket_life_cycle)
if response.status_code == 200:
log.info('bucket life cycle added')
else:
raise TestExecError(
"bucket lifecycle addition failed")
else:
raise TestExecError(
"bucket lifecycle addition failed")
log.info('trying to retrieve bucket lifecycle config')
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
'get_bucket_lifecycle_configuration',
"kwargs":
dict(Bucket=bucket.name)
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(
get_bucket_life_cycle_config)
modified_expiration_days = get_bucket_life_cycle_config[
'Rules'][0]['Expiration']['Days']
log.info('modified expiration days: %s' %
modified_expiration_days)
if response.status_code == 200 and modified_expiration_days == 15:
log.info(
'bucket life cycle retrieved after modifying'
)
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after modifying"
)
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after modifying"
)
# disable bucket lifecycle configuration
if config.test_ops.get('disable_lifecycle', False) is True:
log.info('disabling lifecycle configuration')
life_cycle_disabled_config = basic_lifecycle_config(
prefix="key",
days=20,
id="rul1",
status="Disabled")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(LifecycleConfiguration=
life_cycle_disabled_config)
})
log.info('put bucket life cycle:\n%s' %
put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(
put_bucket_life_cycle)
if response.status_code == 200:
log.info('bucket life cycle added')
else:
raise TestExecError(
"bucket lifecycle addition failed")
else:
raise TestExecError(
"bucket lifecycle addition failed")
log.info('trying to retrieve bucket lifecycle config')
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
'get_bucket_lifecycle_configuration',
"kwargs":
dict(Bucket=bucket.name)
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(
get_bucket_life_cycle_config)
if response.status_code == 200 and get_bucket_life_cycle_config[
'Rules'][0]['Status'] == 'Disabled':
log.info('disabled_status: %s' %
get_bucket_life_cycle_config['Rules']
[0]['Status'])
log.info(
'bucket life cycle retrieved after disabled'
)
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after disabled"
)
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after disabled"
)
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
"""
Executes test based on configuration passed
Args:
config(object): Test configuration
"""
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
ceph_conf = CephConfOp()
rgw_service = RGWService()
# preparing data
user_name = resource_op.create_users(no_of_users_to_create=1)[0]["user_id"]
tenant = "tenant"
tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_name,
displayname=user_name)
umgmt.create_subuser(tenant_name=tenant, user_id=user_name)
ip_and_port = s3cmd_reusable.get_rgw_ip_and_port()
s3_auth.do_auth(tenant_user_info, ip_and_port)
bucket_name = utils.gen_bucket_name_from_userid(user_name, rand_no=0)
# Create a bucket
s3cmd_reusable.create_bucket(bucket_name)
log.info(f"Bucket {bucket_name} created")
# Upload a 2GB file to bucket
uploaded_file_info = s3cmd_reusable.upload_file(
bucket_name, file_size=2147483648, test_data_path=TEST_DATA_PATH)
uploaded_file = uploaded_file_info["name"]
uploaded_file_md5 = uploaded_file_info["md5"]
log.info(f"Uploaded file {uploaded_file} to bucket {bucket_name}")
if config.gc_verification is True:
log.info("making changes to ceph.conf")
config.rgw_gc_obj_min_wait = 5
ceph_conf.set_to_ceph_conf(
"global",
ConfigOpts.rgw_gc_obj_min_wait,
str(config.rgw_gc_obj_min_wait),
)
log.info("trying to restart services")
srv_restarted = rgw_service.restart()
time.sleep(30)
if srv_restarted is False:
raise TestExecError("RGW service restart failed")
else:
log.info("RGW service restarted")
log.info(
"download large object again to make gc list with shadow entries")
downloaded_file1 = s3cmd_reusable.download_file(
bucket_name,
uploaded_file,
local_file_name="download1.img",
test_data_path=TEST_DATA_PATH,
)
time.sleep(5)
downloaded_file1_md5 = utils.get_md5(downloaded_file1)
assert uploaded_file_md5 == downloaded_file1_md5
gc_list_output = json.loads(
utils.exec_shell_cmd("radosgw-admin gc list --include-all"))
log.info(gc_list_output)
if gc_list_output:
log.info(
"Shadow obj found after setting rgw_gc_obj_min_wait to 5 sec")
utils.exec_shell_cmd("radosgw-admin gc process --include-all")
log.info(
"Object download should not error out in 404 NoSuchKey error")
downloaded_file2 = s3cmd_reusable.download_file(
bucket_name,
uploaded_file,
local_file_name="download2.img",
test_data_path=TEST_DATA_PATH,
)
downloaded_file2_md5 = utils.get_md5(downloaded_file2)
assert uploaded_file_md5 == downloaded_file2_md5
# Delete file from bucket
s3cmd_reusable.delete_file(bucket_name, uploaded_file)
log.info(f"Deleted file {uploaded_file} from bucket {bucket_name}")
# Delete bucket
s3cmd_reusable.delete_bucket(bucket_name)
log.info(f"Bucket {bucket_name} deleted")
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
write_bucket_io_info = BucketIoInfo()
io_info_initialize.initialize(basic_io_structure.initial())
ceph_conf = CephConfOp()
rgw_service = RGWService()
log.info('starting IO')
config.user_count = 1
user_info = s3lib.create_users(config.user_count)
user_info = user_info[0]
auth = Auth(user_info, ssl=config.ssl)
rgw_conn = auth.do_auth()
log.info('sharding configuration will be added now.')
if config.sharding_type == 'dynamic':
log.info('sharding type is dynamic')
# for dynamic,
# the number of shards should be greater than [ (no of objects)/(max objects per shard) ]
# example: objects = 500 ; max object per shard = 10
# then no of shards should be at least 50 or more
time.sleep(15)
log.info('making changes to ceph.conf')
ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_max_objs_per_shard,
str(config.max_objects_per_shard))
ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_dynamic_resharding,
'True')
num_shards_expected = config.objects_count / config.max_objects_per_shard
log.info('num_shards_expected: %s' % num_shards_expected)
log.info('trying to restart services ')
srv_restarted = rgw_service.restart()
time.sleep(30)
if srv_restarted is False:
raise TestExecError("RGW service restart failed")
else:
log.info('RGW service restarted')
config.bucket_count = 1
objects_created_list = []
log.info('no of buckets to create: %s' % config.bucket_count)
bucket_name = utils.gen_bucket_name_from_userid(user_info['user_id'],
rand_no=1)
bucket = reusable.create_bucket(bucket_name, rgw_conn, user_info)
if config.test_ops.get('enable_version', False):
log.info('enable bucket version')
reusable.enable_versioning(bucket, rgw_conn, user_info,
write_bucket_io_info)
log.info('s3 objects to create: %s' % config.objects_count)
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(bucket.name, oc)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
if config.test_ops.get('enable_version', False):
reusable.upload_version_object(config, user_info, rgw_conn,
s3_object_name, config.obj_size,
bucket, TEST_DATA_PATH)
else:
reusable.upload_object(s3_object_name, bucket, TEST_DATA_PATH,
config, user_info)
objects_created_list.append((s3_object_name, s3_object_path))
if config.sharding_type == 'manual':
log.info('sharding type is manual')
# for manual.
# the number of shards will be the value set in the command.
time.sleep(15)
log.info('in manual sharding')
cmd_exec = utils.exec_shell_cmd(
'radosgw-admin bucket reshard --bucket=%s --num-shards=%s '
'--yes-i-really-mean-it' % (bucket.name, config.shards))
if cmd_exec is False:
raise TestExecError("manual resharding command execution failed")
sleep_time = 600
log.info(f'verification starts after waiting for {sleep_time} seconds')
time.sleep(sleep_time)
op = utils.exec_shell_cmd("radosgw-admin metadata get bucket:%s" %
bucket.name)
json_doc = json.loads(op)
bucket_id = json_doc['data']['bucket']['bucket_id']
op2 = utils.exec_shell_cmd(
"radosgw-admin metadata get bucket.instance:%s:%s" %
(bucket.name, bucket_id))
json_doc2 = json.loads((op2))
num_shards_created = json_doc2['data']['bucket_info']['num_shards']
log.info('no_of_shards_created: %s' % num_shards_created)
if config.sharding_type == 'manual':
if config.shards != num_shards_created:
raise TestExecError("expected number of shards not created")
log.info('Expected number of shards created')
if config.sharding_type == 'dynamic':
log.info('Verify if resharding list is empty')
reshard_list_op = json.loads(
utils.exec_shell_cmd("radosgw-admin reshard list"))
if not reshard_list_op:
log.info(
'for dynamic number of shards created should be greater than or equal to number of expected shards'
)
log.info('no_of_shards_expected: %s' % num_shards_expected)
if int(num_shards_created) >= int(num_shards_expected):
log.info('Expected number of shards created')
else:
raise TestExecError('Expected number of shards not created')
if config.test_ops.get('delete_bucket_object', False):
if config.test_ops.get('enable_version', False):
for name, path in objects_created_list:
reusable.delete_version_object(bucket, name, path, rgw_conn,
user_info)
else:
reusable.delete_objects(bucket)
reusable.delete_bucket(bucket)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
# create user
user_info = s3lib.create_users(config.user_count)
for each_user in user_info:
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
buckets = []
buckets_meta = []
if config.test_ops["create_bucket"]:
log.info("no of buckets to create: %s" % config.bucket_count)
# create bucket
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=bc
)
bucket = reusable.create_bucket(bucket_name, rgw_conn, each_user)
buckets.append(bucket_name)
buckets_meta.append(bucket)
if config.test_ops["create_object"]:
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
log.info(f"s3 objects to create of size {config.obj_size}")
s3_object_name = config.lifecycle_conf[0]["Filter"][
"Prefix"
] + str(oc)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info(
f"s3 object path: {s3_object_path}, name: {s3_object_name}"
)
reusable.upload_object(
s3_object_name, bucket, TEST_DATA_PATH, config, each_user
)
life_cycle_rule = {"Rules": config.lifecycle_conf}
reusable.put_bucket_lifecycle(
bucket, rgw_conn, rgw_conn2, life_cycle_rule
)
log.info(f"buckets are {buckets}")
for bkt in buckets:
bucket_details = json.loads(
utils.exec_shell_cmd(f"radosgw-admin bucket stats --bucket={bkt}")
)
num_objects = bucket_details["usage"]["rgw.main"]["num_objects"]
log.info(f"objects count in bucket {bkt} is {num_objects}")
lc_list_op_before = json.loads(utils.exec_shell_cmd("radosgw-admin lc list"))
log.info(f"lc lists before lc process is {lc_list_op_before}")
utils.exec_shell_cmd(f"radosgw-admin lc process --bucket {buckets[0]}")
time.sleep(60)
lc_list_op_after = json.loads(utils.exec_shell_cmd("radosgw-admin lc list"))
log.info(f"lc lists after lc process is {lc_list_op_after}")
completed_bucket = 0
completed_bkt_name = ""
for data in lc_list_op_after:
if data["status"] == "COMPLETE":
completed_bucket += 1
completed_bkt_name = data["bucket"]
bucket_details = json.loads(
utils.exec_shell_cmd(f"radosgw-admin bucket stats --bucket={buckets[0]}")
)
num_objects_after = bucket_details["usage"]["rgw.main"]["num_objects"]
if config.object_expire:
if (
completed_bucket == 1
and (buckets[0] in completed_bkt_name)
and num_objects_after == 0
):
log.info(f"processing of single bucket:{buckets[0]} succeeded")
else:
raise TestExecError("LC Processing of a single bucket failed")
else:
if (
completed_bucket == 1
and (buckets[0] in completed_bkt_name)
and num_objects_after == config.objects_count
):
log.info(f"Successfully completed, non-expired objects did not deleted")
else:
raise TestExecError(
"Failed! removed non-expired objects from the bucket"
)
delete_conf = config.lifecycle_conf[0]
delete_conf["Status"] = "Disabled"
for bkt in buckets_meta:
life_cycle_rule_delete = {"Rules": [delete_conf]}
reusable.put_bucket_lifecycle(
bkt, rgw_conn, rgw_conn2, life_cycle_rule_delete
)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
write_bucket_io_info = BucketIoInfo()
io_info_initialize.initialize(basic_io_structure.initial())
ceph_conf = CephConfOp()
rgw_service = RGWService()
config.rgw_lc_debug_interval = 30
config.rgw_lc_max_worker = 10
log.info("making changes to ceph.conf")
ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_lc_debug_interval,
str(config.rgw_lc_debug_interval))
_, version_name = utils.get_ceph_version()
if "nautilus" in version_name:
ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_lc_max_worker,
str(config.rgw_lc_max_worker))
else:
ceph_conf.set_to_ceph_conf(
section=None,
option=ConfigOpts.rgw_lc_max_worker,
value=str(config.rgw_lc_max_worker),
)
ceph_conf.set_to_ceph_conf(section=None,
option=ConfigOpts.rgw_lc_debug_interval,
value="30")
log.info("trying to restart services")
srv_restarted = rgw_service.restart()
time.sleep(30)
if srv_restarted is False:
raise TestExecError("RGW service restart failed")
else:
log.info("RGW service restarted")
config.user_count = 1
config.bucket_count = 1
# create user
user_info = s3lib.create_users(config.user_count)
user_info = user_info[0]
auth = Auth(user_info, ssl=config.ssl)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
log.info("no of buckets to create: %s" % config.bucket_count)
bucket_name = utils.gen_bucket_name_from_userid(user_info["user_id"],
rand_no=1)
obj_list = []
obj_tag = "suffix1=WMV1"
bucket = reusable.create_bucket(bucket_name, rgw_conn, user_info)
prefix = list(
map(
lambda x: x,
[
rule["Filter"].get("Prefix")
or rule["Filter"]["And"].get("Prefix")
for rule in config.lifecycle_conf
],
))
prefix = prefix if prefix else ["dummy1"]
if config.test_ops["enable_versioning"] is True:
reusable.enable_versioning(bucket, rgw_conn, user_info,
write_bucket_io_info)
if config.test_ops["create_object"] is True:
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
key = prefix.pop()
prefix.insert(0, key)
s3_object_name = key + "." + bucket.name + "." + str(oc)
obj_list.append(s3_object_name)
if config.test_ops["version_count"] > 0:
for vc in range(config.test_ops["version_count"]):
log.info("version count for %s is %s" %
(s3_object_name, str(vc)))
log.info("modifying data: %s" % s3_object_name)
reusable.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
user_info,
append_data=True,
append_msg="hello object for version: %s\n" %
str(vc),
)
else:
log.info("s3 objects to create: %s" % config.objects_count)
reusable.upload_object(s3_object_name, bucket,
TEST_DATA_PATH, config, user_info)
life_cycle_rule = {"Rules": config.lifecycle_conf}
reusable.put_get_bucket_lifecycle_test(bucket, rgw_conn, rgw_conn2,
life_cycle_rule, config)
lc_ops.validate_prefix_rule(bucket, config)
if config.test_ops["delete_marker"] is True:
life_cycle_rule_new = {"Rules": config.delete_marker_ops}
reusable.put_get_bucket_lifecycle_test(bucket, rgw_conn, rgw_conn2,
life_cycle_rule_new, config)
if config.test_ops["enable_versioning"] is False:
if config.test_ops["create_object"] is True:
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
key = prefix.pop()
prefix.insert(0, key)
s3_object_name = key + "." + bucket.name + "." + str(oc)
obj_list.append(s3_object_name)
reusable.upload_object_with_tagging(s3_object_name, bucket,
TEST_DATA_PATH, config,
user_info, obj_tag)
life_cycle_rule = {"Rules": config.lifecycle_conf}
reusable.put_get_bucket_lifecycle_test(bucket, rgw_conn, rgw_conn2,
life_cycle_rule, config)
lc_ops.validate_and_rule(bucket, config)
reusable.remove_user(user_info)
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
rgw_service = RGWService()
# create pool
pool_name = '.rgw.buckets.special'
pg_num = '8'
pgp_num = '8'
pool_create = 'sudo ceph osd pool create "%s" %s %s replicated' % (
pool_name, pg_num, pgp_num)
pool_create_exec = utils.exec_shell_cmd(pool_create)
if pool_create_exec is False:
raise TestExecError("Pool creation failed")
# create realm
realm_name = 'buz-tickets'
log.info('creating realm name')
realm_create = 'sudo radosgw-admin realm create --rgw-realm=%s' % realm_name
realm_create_exec = utils.exec_shell_cmd(realm_create)
if realm_create_exec is False:
raise TestExecError("cmd execution failed")
# sample output of create realm
"""
{
"id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62",
"name": "buz-tickets",
"current_period": "1950b710-3e63-4c41-a19e-46a715000980",
"epoch": 1
}
"""
log.info('modify zonegroup ')
modify = 'sudo radosgw-admin zonegroup modify --rgw-zonegroup=default --rgw-realm=%s --master' % realm_name
modify_exec = utils.exec_shell_cmd(modify)
if modify_exec is False:
raise TestExecError("cmd execution failed")
# get the zonegroup
zonegroup_file = 'zonegroup.json'
get_zonegroup = 'sudo radosgw-admin zonegroup --rgw-zonegroup=default get > %s' % zonegroup_file
get_zonegroup_exec = utils.exec_shell_cmd(get_zonegroup)
if get_zonegroup_exec is False:
raise TestExecError("cmd execution failed")
add_to_placement_targets = {"name": "special-placement", "tags": []}
fp = open(zonegroup_file, 'r')
zonegroup_txt = fp.read()
fp.close()
log.info('got zonegroup info: \n%s' % zonegroup_txt)
zonegroup = json.loads(zonegroup_txt)
log.info('adding placement targets')
zonegroup['placement_targets'].append(add_to_placement_targets)
with open(zonegroup_file, 'w') as fp:
json.dump(zonegroup, fp)
zonegroup_set = 'sudo radosgw-admin zonegroup set < %s' % zonegroup_file
zonegroup_set_exec = utils.exec_shell_cmd(zonegroup_set)
if zonegroup_set_exec is False:
raise TestExecError("cmd execution failed")
log.info('zone group update completed')
log.info('getting zone file')
# get zone
log.info('getting zone info')
zone_file = 'zone.json'
get_zone = 'sudo radosgw-admin zone --rgw-zone=default get > zone.json'
get_zone_exec = utils.exec_shell_cmd(get_zone)
if get_zone_exec is False:
raise TestExecError("cmd execution failed")
fp = open(zone_file, 'r')
zone_info = fp.read()
fp.close()
log.info('zone_info :\n%s' % zone_info)
zone_info_cleaned = json.loads(zone_info)
special_placement_info = {
"key": "special-placement",
"val": {
"index_pool": ".rgw.buckets.index",
"data_pool": ".rgw.buckets.special",
"data_extra_pool": ".rgw.buckets.extra"
}
}
log.info('adding special placement info')
zone_info_cleaned['placement_pools'].append(special_placement_info)
with open(zone_file, 'w+') as fp:
json.dump(zone_info_cleaned, fp)
zone_file_set = 'sudo radosgw-admin zone set < %s' % zone_file
zone_file_set_exec = utils.exec_shell_cmd(zone_file_set)
if zone_file_set_exec is False:
raise TestExecError("cmd execution failed")
log.info('zone info updated ')
zone_group_update_set = 'radosgw-admin period update --commit'
zone_group_update_set_exec = utils.exec_shell_cmd(zone_group_update_set)
log.info(zone_group_update_set_exec)
restarted = rgw_service.restart()
if restarted is False:
raise TestExecError("service restart failed")
if config.rgw_client == 'rgw':
log.info('client type is rgw')
rgw_user_info = s3_swift_lib.create_users(1)
auth = Auth(rgw_user_info)
rgw_conn = auth.do_auth()
# create bucket
bucket_name = utils.gen_bucket_name_from_userid(
rgw_user_info['user_id'], 0)
bucket = resuables.create_bucket(bucket_name, rgw_conn, rgw_user_info)
# create object
s3_object_name = utils.gen_s3_object_name(bucket_name, 0)
resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config,
rgw_user_info)
if config.rgw_client == 'swift':
log.info('client type is swift')
user_names = ['tuffy', 'scooby', 'max']
tenant = 'tenant'
umgmt = UserMgmt()
umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0])
user_info = umgmt.create_subuser(tenant_name=tenant,
user_id=user_names[0])
auth = Auth(user_info)
rgw = auth.do_auth()
container_name = utils.gen_bucket_name_from_userid(
user_info['user_id'], rand_no=0)
container = s3_swift_lib.resource_op({
'obj': rgw,
'resource': 'put_container',
'args': [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
swift_object_name = utils.gen_s3_object_name(
'%s.container.%s' % (user_names[0], 0), 0)
log.info('object name: %s' % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info('object path: %s' % object_path)
object_size = utils.get_file_size(config.objects_size_range['min'],
config.objects_size_range['max'])
data_info = manage_data.io_generator(object_path, object_size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info('uploading object: %s' % object_path)
with open(object_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
def test_exec(config):
test_info = AddTestInfo("storage_policy for %s" % config.rgw_client)
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
rgw_service = RGWService()
try:
# create pool
pool_name = ".rgw.buckets.special"
pg_num = "8"
pgp_num = "8"
pool_create = 'sudo ceph osd pool create "%s" %s %s replicated' % (
pool_name,
pg_num,
pgp_num,
)
pool_create_exec = utils.exec_shell_cmd(pool_create)
if pool_create_exec is False:
raise TestExecError("Pool creation failed")
# create realm
realm_name = "buz-tickets"
log.info("creating realm name")
realm_create = (
"sudo radosgw-admin realm create --rgw-realm=%s --default" % realm_name
)
realm_create_exec = utils.exec_shell_cmd(realm_create)
if realm_create_exec is False:
raise TestExecError("cmd execution failed")
# sample output of create realm
"""
{
"id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62",
"name": "buz-tickets",
"current_period": "1950b710-3e63-4c41-a19e-46a715000980",
"epoch": 1
}
"""
log.info("modify zonegroup ")
modify = (
"sudo radosgw-admin zonegroup modify --rgw-zonegroup=default --rgw-realm=%s --master --default"
% realm_name
)
modify_exec = utils.exec_shell_cmd(modify)
if modify_exec is False:
raise TestExecError("cmd execution failed")
# get the zonegroup
zonegroup_file = "zonegroup.json"
get_zonegroup = (
"sudo radosgw-admin zonegroup --rgw-zonegroup=default get > %s"
% zonegroup_file
)
get_zonegroup_exec = utils.exec_shell_cmd(get_zonegroup)
if get_zonegroup_exec is False:
raise TestExecError("cmd execution failed")
add_to_placement_targets = {"name": "special-placement", "tags": []}
fp = open(zonegroup_file, "r")
zonegroup_txt = fp.read()
fp.close()
log.info("got zonegroup info: \n%s" % zonegroup_txt)
zonegroup = json.loads(zonegroup_txt)
log.info("adding placement targets")
zonegroup["placement_targets"].append(add_to_placement_targets)
with open(zonegroup_file, "w") as fp:
json.dump(zonegroup, fp)
zonegroup_set = "sudo radosgw-admin zonegroup set < %s" % zonegroup_file
zonegroup_set_exec = utils.exec_shell_cmd(zonegroup_set)
if zonegroup_set_exec is False:
raise TestExecError("cmd execution failed")
log.info("zone group update completed")
log.info("getting zone file")
# get zone
log.info("getting zone info")
zone_file = "zone.json"
get_zone = "sudo radosgw-admin zone --rgw-zone=default get > zone.json"
get_zone_exec = utils.exec_shell_cmd(get_zone)
if get_zone_exec is False:
raise TestExecError("cmd execution failed")
fp = open(zone_file, "r")
zone_info = fp.read()
fp.close()
log.info("zone_info :\n%s" % zone_info)
zone_info_cleaned = json.loads(zone_info)
special_placement_info = {
"key": "special-placement",
"val": {
"index_pool": ".rgw.buckets.index",
"data_pool": ".rgw.buckets.special",
"data_extra_pool": ".rgw.buckets.extra",
},
}
log.info("adding special placement info")
zone_info_cleaned["placement_pools"].append(special_placement_info)
print(zone_info_cleaned)
with open(zone_file, "w+") as fp:
json.dump(zone_info_cleaned, fp)
zone_file_set = "sudo radosgw-admin zone set < %s" % zone_file
zone_file_set_exec = utils.exec_shell_cmd(zone_file_set)
if zone_file_set_exec is False:
raise TestExecError("cmd execution failed")
log.info("zone info updated ")
restarted = rgw_service.restart()
if restarted is False:
raise TestExecError("service restart failed")
if config.rgw_client == "rgw":
log.info("client type is rgw")
rgw_user_info = s3_swift_lib.create_users(1)
auth = Auth(rgw_user_info)
rgw_conn = auth.do_auth()
# create bucket
bucket_name = utils.gen_bucket_name_from_userid(rgw_user_info["user_id"], 0)
bucket = resuables.create_bucket(bucket_name, rgw_conn, rgw_user_info)
# create object
s3_object_name = utils.gen_s3_object_name(bucket_name, 0)
resuables.upload_object(
s3_object_name, bucket, TEST_DATA_PATH, config, rgw_user_info
)
if config.rgw_client == "swift":
log.info("client type is swift")
user_names = ["tuffy", "scooby", "max"]
tenant = "tenant"
umgmt = UserMgmt()
umgmt.create_tenant_user(
tenant_name=tenant, user_id=user_names[0], displayname=user_names[0]
)
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0])
auth = Auth(user_info)
rgw = auth.do_auth()
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=0
)
container = s3_swift_lib.resource_op(
{"obj": rgw, "resource": "put_container", "args": [container_name]}
)
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield"
)
swift_object_name = utils.gen_s3_object_name(
"%s.container.%s" % (user_names[0], 0), 0
)
log.info("object name: %s" % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info("object path: %s" % object_path)
object_size = utils.get_file_size(
config.objects_size_range["min"], config.objects_size_range["max"]
)
data_info = manage_data.io_generator(object_path, object_size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info("uploading object: %s" % object_path)
with open(object_path, "r") as fp:
rgw.put_object(
container_name,
swift_object_name,
contents=fp.read(),
content_type="text/plain",
)
test_info.success_status("test passed")
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)