def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
# preparing data
user_names = ['tuffy', 'scooby', 'max']
tenant = 'tenant'
tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0])
user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0])
cmd = 'radosgw-admin quota enable --quota-scope=user --uid={uid} --tenant={tenant}'.format(
uid=user_names[0], tenant=tenant)
enable_user_quota = utils.exec_shell_cmd(cmd)
cmd = 'radosgw-admin quota set --quota-scope=user --uid={uid} --tenant={tenant} --max_buckets=2000'.format(
uid=user_names[0], tenant=tenant)
max_bucket = utils.exec_shell_cmd(cmd)
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
container_name = utils.gen_bucket_name_from_userid(
user_info['user_id'], rand_no=cc)
container = swiftlib.resource_op({
'obj': rgw,
'resource': 'put_container',
'args': [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
host, ip = utils.get_hostname_ip()
port = utils.get_radosgw_port_no()
hostname = str(ip) + ":" + str(port)
cmd = 'swift -A http://{hostname}/auth/1.0 -U \'{uid}\' -K \'{key}\' stat'.format(
hostname=hostname, uid=user_info['user_id'], key=user_info['key'])
swift_cmd = utils.exec_shell_cmd(cmd)
swift_cmd = swift_cmd.replace(" ", "")
swift_cmd = swift_cmd.replace("\n", ":")
li = list(swift_cmd.split(":"))
res_dct = {li[i]: li[i + 1] for i in range(0, len(li) - 1, 2)}
if (int(res_dct['Containers']) == config.container_count):
cmd = 'radosgw-admin user rm --uid={uid} --tenant={tenant} --purge-data'.format(
uid=user_names[0], tenant=tenant)
delete_user_bucket = utils.exec_shell_cmd(cmd)
test_info.success_status('test passed')
sys.exit(0)
else:
cmd = 'radosgw-admin user rm --uid={uid} --tenant={tenant} --purge-data'.format(
uid=user_names[0], tenant=tenant)
delete_user_bucket = utils.exec_shell_cmd(cmd)
test_info.failed_status('test failed')
sys.exit(1)
def time_to_list_via_boto(bucket_name, rgw):
"""
Time taken to list via boto
:param bucket: s3Bucket object
"""
bucket = s3lib.resource_op({
"obj": rgw,
"resource": "Bucket",
"args": [bucket_name]
})
log.info("listing all objects in bucket: %s" % bucket)
objects = s3lib.resource_op({
"obj": bucket,
"resource": "objects",
"args": None
})
time_taken = timeit.timeit(lambda: bucket.objects.all(), globals=globals())
return time_taken
def list_objects(bucket):
"""
list the objects in a given bucket
:param bucket: S3Bucket object
"""
log.info("listing all objects in bucket: %s" % bucket.name)
objects = s3lib.resource_op({
"obj": bucket,
"resource": "objects",
"args": None
})
log.info("objects :%s" % objects)
all_objects = s3lib.resource_op({
"obj": objects,
"resource": "all",
"args": None
})
log.info("all objects: %s" % all_objects)
for obj in all_objects:
log.info("object_name: %s" % obj.key)
def upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
user_info,
append_data=False,
append_msg=None,
):
log.info("s3 object name: %s" % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info("s3 object path: %s" % s3_object_path)
s3_object_size = config.obj_size
if append_data is True:
data_info = manage_data.io_generator(
s3_object_path,
s3_object_size,
op="append",
**{"message": "\n%s" % append_msg},
)
else:
data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if data_info is False:
TestExecError("data creation failed")
log.info("uploading s3 object: %s" % s3_object_path)
upload_info = dict({"access_key": user_info["access_key"]}, **data_info)
s3_obj = s3lib.resource_op({
"obj": bucket,
"resource": "Object",
"args": [s3_object_name],
})
object_uploaded_status = s3lib.resource_op({
"obj": s3_obj,
"resource": "upload_file",
"args": [s3_object_path],
"extra_info": upload_info,
})
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info("object uploaded")
def create_bucket(bucket_name, rgw, user_info):
log.info('creating bucket with name: %s' % bucket_name)
# bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create)
bucket = s3lib.resource_op({'obj': rgw,
'resource': 'Bucket',
'args': [bucket_name]})
created = s3lib.resource_op({'obj': bucket,
'resource': 'create',
'args': None,
'extra_info': {'access_key': user_info['access_key']}})
if created is False:
raise TestExecError("Resource execution failed: bucket creation failed")
if created is not None:
response = HttpResponseParser(created)
if response.status_code == 200:
log.info('bucket created')
else:
raise TestExecError("bucket creation failed")
else:
raise TestExecError("bucket creation failed")
return bucket
def create_bucket_with_versioning(rgw_conn, user_info, bucket_name):
# create buckets
bucket = resuables.create_bucket(bucket_name, rgw_conn, user_info)
bucket_versioning = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'BucketVersioning',
'args': [bucket.name]
})
# checking the versioning status
version_status = s3lib.resource_op({
'obj': bucket_versioning,
'resource': 'status',
'args': None
})
if version_status is None:
log.info('bucket versioning still not enabled')
# enabling bucket versioning
version_enable_status = s3lib.resource_op({
'obj': bucket_versioning,
'resource': 'enable',
'args': None
})
response = HttpResponseParser(version_enable_status)
if response.status_code == 200:
log.info('version enabled')
else:
raise TestExecError("version enable failed")
return bucket
def enable_mfa_versioning(
bucket, rgw_conn, SEED, serial, user_info, write_bucket_io_info
):
log.info("bucket MFA and versioning test on bucket: %s" % bucket.name)
bucket_versioning = s3lib.resource_op(
{"obj": rgw_conn, "resource": "BucketVersioning", "args": [bucket.name]}
)
# checking the versioning status
version_status = s3lib.resource_op(
{"obj": bucket_versioning, "resource": "status", "args": None}
)
if version_status is None:
log.info("bucket mfa and versioning still not enabled")
# generate MFA token to authenticate
token = generate_totp(SEED)
mfa_token = serial + " " + token
# put mfa and bucket versioning
mfa_version_put = s3lib.resource_op(
{
"obj": bucket_versioning,
"resource": "put",
"kwargs": dict(
MFA=(mfa_token),
VersioningConfiguration={"MFADelete": "Enabled", "Status": "Enabled"},
ExpectedBucketOwner=user_info["user_id"],
),
}
)
if mfa_version_put is False:
return token, mfa_version_put
response = HttpResponseParser(mfa_version_put)
if response.status_code == 200:
log.info("MFA and version enabled")
else:
raise MFAVersionError("bucket mfa and versioning enable failed")
def delete_objects(bucket):
"""
deletes the objects in a given bucket
:param bucket: S3Bucket object
"""
log.info('listing all objects in bucket: %s' % bucket.name)
objects = s3lib.resource_op({
'obj': bucket,
'resource': 'objects',
'args': None
})
log.info('objects :%s' % objects)
all_objects = s3lib.resource_op({
'obj': objects,
'resource': 'all',
'args': None
})
log.info('all objects: %s' % all_objects)
for obj in all_objects:
log.info('object_name: %s' % obj.key)
log.info('deleting all objects in bucket')
objects_deleted = s3lib.resource_op({
'obj': objects,
'resource': 'delete',
'args': None
})
log.info('objects_deleted: %s' % objects_deleted)
if objects_deleted is False:
raise TestExecError(
'Resource execution failed: Object deletion failed')
if objects_deleted is not None:
response = HttpResponseParser(objects_deleted[0])
if response.status_code == 200:
log.info('objects deleted ')
else:
raise TestExecError("objects deletion failed")
else:
raise TestExecError("objects deletion failed")
def upload_object(s3_object_name,
bucket,
TEST_DATA_PATH,
config,
user_info,
append_data=False,
append_msg=None):
log.info('s3 object name: %s' % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info('s3 object path: %s' % s3_object_path)
s3_object_size = config.obj_size
if append_data is True:
data_info = manage_data.io_generator(
s3_object_path,
s3_object_size,
op='append',
**{'message': '\n%s' % append_msg})
else:
data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if data_info is False:
TestExecError("data creation failed")
log.info('uploading s3 object: %s' % s3_object_path)
upload_info = dict({'access_key': user_info['access_key']}, **data_info)
s3_obj = s3lib.resource_op({
'obj': bucket,
'resource': 'Object',
'args': [s3_object_name],
})
object_uploaded_status = s3lib.resource_op({
'obj': s3_obj,
'resource': 'upload_file',
'args': [s3_object_path],
'extra_info': upload_info
})
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info('object uploaded')
def delete_objects(bucket):
"""
deletes the objects in a given bucket
:param bucket: S3Bucket object
"""
log.info("listing all objects in bucket: %s" % bucket.name)
objects = s3lib.resource_op({
"obj": bucket,
"resource": "objects",
"args": None
})
log.info("objects :%s" % objects)
all_objects = s3lib.resource_op({
"obj": objects,
"resource": "all",
"args": None
})
log.info("all objects: %s" % all_objects)
for obj in all_objects:
log.info("object_name: %s" % obj.key)
log.info("deleting all objects in bucket")
objects_deleted = s3lib.resource_op({
"obj": objects,
"resource": "delete",
"args": None
})
log.info("objects_deleted: %s" % objects_deleted)
if objects_deleted is False:
raise TestExecError(
"Resource execution failed: Object deletion failed")
if objects_deleted is not None:
response = HttpResponseParser(objects_deleted[0])
if response.status_code == 200:
log.info("objects deleted ")
else:
raise TestExecError("objects deletion failed")
else:
raise TestExecError("objects deletion failed")
def delete_version_object(bucket, s3_object_name, s3_object_path, rgw_conn, user_info, ):
"""
deletes single object and its versions
:param bucket: S3bucket object
:param s3_object_name: s3 object name
:param s3_object_path: path of the object created in the client
:param rgw_conn: rgw connection
:param user_info: user info dict containing access_key, secret_key and user_id
"""
versions = bucket.object_versions.filter(Prefix=s3_object_name)
log.info('deleting s3_obj keys and its versions')
s3_obj = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('deleting versions for s3 obj: %s' % s3_object_name)
for version in versions:
log.info('trying to delete obj version: %s' % version.version_id)
del_obj_version = s3lib.resource_op({'obj': s3_obj,
'resource': 'delete',
'kwargs': dict(VersionId=version.version_id)})
log.info('response:\n%s' % del_obj_version)
if del_obj_version is not None:
response = HttpResponseParser(del_obj_version)
if response.status_code == 204:
log.info('version deleted ')
write_key_io_info.delete_version_info(user_info['access_key'], bucket.name,
s3_object_path, version.version_id)
else:
raise TestExecError("version deletion failed")
else:
raise TestExecError("version deletion failed")
log.info('available versions for the object')
versions = bucket.object_versions.filter(Prefix=s3_object_name)
for version in versions:
log.info('key_name: %s --> version_id: %s' % (
version.object_key, version.version_id))
def upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, user_info):
log.info('s3 object name: %s' % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info('s3 object path: %s' % s3_object_path)
s3_object_size = utils.get_file_size(config.objects_size_range['min'],
config.objects_size_range['max'])
data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if data_info is False:
TestExecError("data creation failed")
log.info('uploading s3 object: %s' % s3_object_path)
upload_info = dict({'access_key': user_info['access_key']}, **data_info)
object_uploaded_status = s3lib.resource_op({
'obj':
bucket,
'resource':
'upload_file',
'args': [s3_object_path, s3_object_name],
'extra_info':
upload_info
})
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info('object uploaded')
def upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, user_info):
log.info("s3 object name: %s" % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info("s3 object path: %s" % s3_object_path)
s3_object_size = utils.get_file_size(
config.objects_size_range["min"], config.objects_size_range["max"]
)
data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if data_info is False:
TestExecError("data creation failed")
log.info("uploading s3 object: %s" % s3_object_path)
upload_info = dict({"access_key": user_info["access_key"]}, **data_info)
object_uploaded_status = s3lib.resource_op(
{
"obj": bucket,
"resource": "upload_file",
"args": [s3_object_path, s3_object_name],
"extra_info": upload_info,
}
)
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info("object uploaded")
def put_get_bucket_lifecycle_test(bucket, rgw_conn, rgw_conn2, life_cycle_rule,
config):
bucket_life_cycle = s3lib.resource_op({
"obj": rgw_conn,
"resource": "BucketLifecycleConfiguration",
"args": [bucket.name],
})
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(LifecycleConfiguration=life_cycle_rule),
})
log.info("put bucket life cycle:\n%s" % put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: put bucket lifecycle failed")
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
"get_bucket_lifecycle_configuration",
"kwargs":
dict(Bucket=bucket.name),
})
if get_bucket_life_cycle_config is False:
raise TestExecError("bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(get_bucket_life_cycle_config)
if response.status_code == 200:
log.info("bucket life cycle retrieved")
else:
raise TestExecError("bucket lifecycle config retrieval failed")
else:
raise TestExecError("bucket life cycle retrieved")
objs_total = (config.test_ops["version_count"]) * (config.objects_count)
for rule in config.lifecycle_conf:
if rule.get("Expiration", {}).get("Date", False):
# todo: need to get the interval value from yaml file
log.info("wait for 60 seconds")
time.sleep(60)
else:
for time_interval in range(19):
bucket_stats_op = utils.exec_shell_cmd(
"radosgw-admin bucket stats --bucket=%s" % bucket.name)
json_doc1 = json.loads(bucket_stats_op)
obj_pre_lc = json_doc1["usage"]["rgw.main"]["num_objects"]
if obj_pre_lc == objs_total:
time.sleep(30)
else:
raise TestExecError(
"Objects expired before the expected days")
time.sleep(60)
log.info("testing if lc is applied via the radosgw-admin cli")
op = utils.exec_shell_cmd("radosgw-admin lc list")
json_doc = json.loads(op)
for i, entry in enumerate(json_doc):
print(i)
print(entry["status"])
if entry["status"] == "COMPLETE" or entry["status"] == "PROCESSING":
log.info("LC is applied on the bucket")
else:
raise TestExecError("LC is not applied")
def upload_mutipart_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
user_info,
append_data=False,
append_msg=None,
):
log.info("s3 object name: %s" % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info("s3 object path: %s" % s3_object_path)
s3_object_size = config.obj_size
split_size = config.split_size if hasattr(config, "split_size") else 5
log.info("split size: %s" % split_size)
if append_data is True:
data_info = manage_data.io_generator(
s3_object_path,
s3_object_size,
op="append",
**{"message": "\n%s" % append_msg},
)
else:
data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if data_info is False:
TestExecError("data creation failed")
mp_dir = os.path.join(TEST_DATA_PATH, s3_object_name + ".mp.parts")
log.info("mp part dir: %s" % mp_dir)
log.info("making multipart object part dir")
mkdir = utils.exec_shell_cmd("sudo mkdir %s" % mp_dir)
if mkdir is False:
raise TestExecError("mkdir failed creating mp_dir_name")
utils.split_file(s3_object_path, split_size, mp_dir + "/")
parts_list = sorted(glob.glob(mp_dir + "/" + "*"))
log.info("parts_list: %s" % parts_list)
log.info("uploading s3 object: %s" % s3_object_path)
upload_info = dict(
{
"access_key": user_info["access_key"],
"upload_type": "multipart"
}, **data_info)
s3_obj = s3lib.resource_op({
"obj": bucket,
"resource": "Object",
"args": [s3_object_name],
})
log.info("initiating multipart upload")
mpu = s3lib.resource_op({
"obj": s3_obj,
"resource": "initiate_multipart_upload",
"args": None,
"extra_info": upload_info,
})
part_number = 1
parts_info = {"Parts": []}
log.info("no of parts: %s" % len(parts_list))
for each_part in parts_list:
log.info("trying to upload part: %s" % each_part)
part = mpu.Part(part_number)
# part_upload_response = part.upload(Body=open(each_part))
part_upload_response = s3lib.resource_op({
"obj":
part,
"resource":
"upload",
"kwargs":
dict(Body=open(each_part, mode="rb")),
})
if part_upload_response is not False:
response = HttpResponseParser(part_upload_response)
if response.status_code == 200:
log.info("part uploaded")
if config.local_file_delete is True:
log.info("deleting local file part")
utils.exec_shell_cmd("sudo rm -rf %s" % each_part)
else:
raise TestExecError("part uploading failed")
part_info = {
"PartNumber": part_number,
"ETag": part_upload_response["ETag"]
}
parts_info["Parts"].append(part_info)
if each_part != parts_list[-1]:
# increase the part number only if the current part is not the last part
part_number += 1
log.info("curr part_number: %s" % part_number)
# log.info('parts_info so far: %s'% parts_info)
if len(parts_list) == part_number:
log.info("all parts upload completed")
mpu.complete(MultipartUpload=parts_info)
log.info("multipart upload complete for key: %s" % s3_object_name)
def upload_version_object(config, user_info, rgw_conn, s3_object_name,
object_size, bucket, TEST_DATA_PATH):
# versioning upload
log.info("versioning count: %s" % config.version_count)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
original_data_info = manage_data.io_generator(s3_object_path, object_size)
if original_data_info is False:
TestExecError("data creation failed")
created_versions_count = 0
for vc in range(config.version_count):
log.info("version count for %s is %s" % (s3_object_name, str(vc)))
log.info("modifying data: %s" % s3_object_name)
modified_data_info = manage_data.io_generator(
s3_object_path,
object_size,
op="append",
**{"message": "\nhello for version: %s\n" % str(vc)},
)
if modified_data_info is False:
TestExecError("data modification failed")
log.info("uploading s3 object: %s" % s3_object_path)
upload_info = dict(
{
"access_key": user_info["access_key"],
"versioning_status": "enabled",
"version_count_no": vc,
},
**modified_data_info,
)
s3_obj = s3lib.resource_op({
"obj": bucket,
"resource": "Object",
"args": [s3_object_name],
"extra_info": upload_info,
})
object_uploaded_status = s3lib.resource_op({
"obj":
s3_obj,
"resource":
"upload_file",
"args": [modified_data_info["name"]],
"extra_info":
upload_info,
})
if object_uploaded_status is False:
raise TestExecError(
"Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info("object uploaded")
s3_obj = rgw_conn.Object(bucket.name, s3_object_name)
log.info("current_version_id: %s" % s3_obj.version_id)
key_version_info = basic_io_structure.version_info(
**{
"version_id": s3_obj.version_id,
"md5_local": upload_info["md5"],
"count_no": vc,
"size": upload_info["size"],
})
log.info("key_version_info: %s" % key_version_info)
write_key_io_info.add_versioning_info(user_info["access_key"],
bucket.name, s3_object_path,
key_version_info)
created_versions_count += 1
log.info("created_versions_count: %s" % created_versions_count)
log.info("adding metadata")
metadata1 = {"m_data1": "this is the meta1 for this obj"}
s3_obj.metadata.update(metadata1)
metadata2 = {"m_data2": "this is the meta2 for this obj"}
s3_obj.metadata.update(metadata2)
log.info("metadata for this object: %s" % s3_obj.metadata)
log.info("metadata count for object: %s" % (len(s3_obj.metadata)))
if not s3_obj.metadata:
raise TestExecError(
"metadata not created even adding metadata")
versions = bucket.object_versions.filter(Prefix=s3_object_name)
created_versions_count_from_s3 = len(
[v.version_id for v in versions])
log.info("created versions count on s3: %s" %
created_versions_count_from_s3)
if created_versions_count is created_versions_count_from_s3:
log.info("no new versions are created when added metadata")
else:
raise TestExecError(
"version count mismatch, "
"possible creation of version on adding metadata")
s3_object_download_path = os.path.join(TEST_DATA_PATH,
s3_object_name + ".download")
object_downloaded_status = s3lib.resource_op({
"obj":
bucket,
"resource":
"download_file",
"args": [s3_object_name, s3_object_download_path],
})
if object_downloaded_status is False:
raise TestExecError(
"Resource execution failed: object download failed")
if object_downloaded_status is None:
log.info("object downloaded")
# checking md5 of the downloaded file
s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path)
log.info("downloaded_md5: %s" % s3_object_downloaded_md5)
log.info("uploaded_md5: %s" % modified_data_info["md5"])
log.info("deleting downloaded version file")
utils.exec_shell_cmd("sudo rm -rf %s" % s3_object_download_path)
log.info("all versions for the object: %s\n" % s3_object_name)
def test_exec(config):
test_info = AddTestInfo('test swift user key gen')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
# preparing data
user_names = ['tuffy', 'scooby', 'max']
tenant1 = 'tenant'
cmd = 'radosgw-admin user create --uid=%s --display-name="%s" --tenant=%s' % (
user_names[0], user_names[0], tenant1)
out = utils.exec_shell_cmd(cmd)
if out is False:
raise TestExecError("RGW User creation error")
log.info('output :%s' % out)
v1_as_json = json.loads(out)
log.info('creted user_id: %s' % v1_as_json['user_id'])
cmd2 = 'radosgw-admin subuser create --uid=%s$%s --subuser=%s:swift --tenant=%s --access=full' % (
tenant1, user_names[0], user_names[0], tenant1)
out2 = utils.exec_shell_cmd(cmd2)
if out2 is False:
raise TestExecError("sub-user creation error")
v2_as_json = json.loads(out2)
log.info('created subuser: %s' % v2_as_json['subusers'][0]['id'])
cmd3 = 'radosgw-admin key create --subuser=%s:swift --uid=%s$%s --tenant=%s --key-type=swift --gen-secret' % (
user_names[0], user_names[0], tenant1, tenant1)
out3 = utils.exec_shell_cmd(cmd3)
if out3 is False:
raise TestExecError("secret_key gen error")
v3_as_json = json.loads(out3)
log.info('created subuser: %s\nsecret_key generated: %s' %
(v3_as_json['swift_keys'][0]['user'],
v3_as_json['swift_keys'][0]['secret_key']))
user_info = {
'user_id': v3_as_json['swift_keys'][0]['user'],
'key': v3_as_json['swift_keys'][0]['secret_key']
}
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
container_name = utils.gen_bucket_name_from_userid(
user_info['user_id'], rand_no=cc)
container = swiftlib.resource_op({
'obj': rgw,
'resource': 'put_container',
'args': [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
for oc, size in config.mapped_sizes.items():
swift_object_name = utils.gen_s3_object_name(
'%s.container.%s' % (user_names[0], cc), oc)
log.info('object name: %s' % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info('object path: %s' % object_path)
data_info = manage_data.io_generator(object_path, size)
if data_info is False:
TestExecError("data creation failed")
log.info('uploading object: %s' % object_path)
with open(object_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
test_info.success_status('test passed')
sys.exit(0)
except Exception, e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
write_user_info = AddUserInfo()
test_info = AddTestInfo('create m buckets')
conf_path = '/etc/ceph/%s.conf' % config.cluster_name
ceph_conf = CephConfOp(conf_path)
rgw_service = RGWService()
try:
test_info.started_info()
# get user
with open('user_details') as fout:
all_users_info = simplejson.load(fout)
for each_user in all_users_info:
user_info = basic_io_structure.user(
**{
'user_id': each_user['user_id'],
'access_key': each_user['access_key'],
'secret_key': each_user['secret_key']
})
write_user_info.add_user_info(user_info)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user)
rgw_conn = auth.do_auth()
# enabling sharding
if config.test_ops['sharding']['enable'] is True:
log.info('enabling sharding on buckets')
max_shards = config.test_ops['sharding']['max_shards']
log.info('making changes to ceph.conf')
ceph_conf.set_to_ceph_conf(
'global', ConfigOpts.rgw_override_bucket_index_max_shards,
max_shards)
log.info('trying to restart services ')
srv_restarted = rgw_service.restart()
time.sleep(10)
if srv_restarted is False:
raise TestExecError("RGW service restart failed")
else:
log.info('RGW service restarted')
# create buckets
if config.test_ops['create_bucket'] is True:
log.info('no of buckets to create: %s' % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(
each_user['user_id'], rand_no=bc)
log.info('creating bucket with name: %s' %
bucket_name_to_create)
# bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create)
bucket = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'Bucket',
'args': [bucket_name_to_create]
})
created = s3lib.resource_op({
'obj': bucket,
'resource': 'create',
'args': None,
'extra_info': {
'access_key': each_user['access_key']
}
})
if created is False:
raise TestExecError(
"Resource execution failed: bucket creation failed"
)
if created is not None:
response = HttpResponseParser(created)
if response.status_code == 200:
log.info('bucket created')
else:
raise TestExecError("bucket creation failed")
else:
raise TestExecError("bucket creation failed")
if config.test_ops['sharding']['enable'] is True:
cmd = 'radosgw-admin metadata get bucket:%s --cluster %s | grep bucket_id' \
% (bucket.name, config.cluster_name)
out = utils.exec_shell_cmd(cmd)
b_id = out.replace(
'"',
'').strip().split(":")[1].strip().replace(',', '')
cmd2 = 'rados -p default.rgw.buckets.index ls --cluster %s | grep %s' \
% (config.cluster_name, b_id)
out = utils.exec_shell_cmd(cmd2)
log.info(
'got output from sharing verification.--------')
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
test_info = AddTestInfo('create m buckets with n objects')
try:
test_info.started_info()
# get user
with open('user_details') as fout:
all_users_info = simplejson.load(fout)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user)
rgw_conn = auth.do_auth_using_client()
rgw = auth.do_auth()
bucket_list = []
buckets = rgw_conn.list_buckets()
log.info('buckets are %s' % buckets)
for each_bucket in buckets['Buckets']:
bucket_list.append(each_bucket['Name'])
for bucket_name in bucket_list:
# create 'bucket' resource object
bucket = rgw.Bucket(bucket_name)
log.info('In bucket: %s' % bucket_name)
if config.test_ops['create_object'] is True:
# uploading data
log.info('s3 objects to create: %s' % config.objects_count)
for oc in range(config.objects_count):
s3_object_name = utils.gen_s3_object_name(bucket_name, oc)
log.info('s3 object name: %s' % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info('s3 object path: %s' % s3_object_path)
s3_object_size = utils.get_file_size(config.objects_size_range['min'],
config.objects_size_range['max'])
data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if data_info is False:
TestExecError("data creation failed")
log.info('uploading s3 object: %s' % s3_object_path)
upload_info = dict({'access_key': each_user['access_key']}, **data_info)
# object_uploaded_status = bucket.upload_file(s3_object_path, s3_object_name)
object_uploaded_status = s3lib.resource_op({'obj': bucket,
'resource': 'upload_file',
'args': [s3_object_path, s3_object_name],
'extra_info': upload_info})
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info('object uploaded')
if config.test_ops['download_object'] is True:
log.info('trying to download object: %s' % s3_object_name)
s3_object_download_name = s3_object_name + "." + "download"
s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_download_name)
log.info('s3_object_download_path: %s' % s3_object_download_path)
log.info('downloading to filename: %s' % s3_object_download_name)
# object_downloaded_status = bucket.download_file(s3_object_path, s3_object_name)
object_downloaded_status = s3lib.resource_op({'obj': bucket,
'resource': 'download_file',
'args': [s3_object_name,
s3_object_download_path],
})
if object_downloaded_status is False:
raise TestExecError("Resource execution failed: object download failed")
if object_downloaded_status is None:
log.info('object downloaded')
if config.test_ops['delete_bucket_object'] is True:
log.info('listing all objects in bucket: %s' % bucket.name)
# objects = s3_ops.resource_op(bucket, 'objects', None)
objects = s3lib.resource_op({'obj': bucket,
'resource': 'objects',
'args': None})
log.info('objects :%s' % objects)
# all_objects = s3_ops.resource_op(objects, 'all')
all_objects = s3lib.resource_op({'obj': objects,
'resource': 'all',
'args': None})
log.info('all objects: %s' % all_objects)
for obj in all_objects:
log.info('object_name: %s' % obj.key)
log.info('deleting all objects in bucket')
# objects_deleted = s3_ops.resource_op(objects, 'delete')
objects_deleted = s3lib.resource_op({'obj': objects,
'resource': 'delete',
'args': None})
log.info('objects_deleted: %s' % objects_deleted)
if objects_deleted is False:
raise TestExecError('Resource execution failed: Object deletion failed')
if objects_deleted is not None:
response = HttpResponseParser(objects_deleted[0])
if response.status_code == 200:
log.info('objects deleted ')
else:
raise TestExecError("objects deletion failed")
else:
raise TestExecError("objects deletion failed")
# wait for object delete info to sync
time.sleep(60)
log.info('deleting bucket: %s' % bucket.name)
# bucket_deleted_status = s3_ops.resource_op(bucket, 'delete')
bucket_deleted_status = s3lib.resource_op({'obj': bucket,
'resource': 'delete',
'args': None})
log.info('bucket_deleted_status: %s' % bucket_deleted_status)
if bucket_deleted_status is not None:
response = HttpResponseParser(bucket_deleted_status)
if response.status_code == 204:
log.info('bucket deleted ')
else:
raise TestExecError("bucket deletion failed")
else:
raise TestExecError("bucket deletion failed")
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
rgw_service = RGWService()
# create pool
pool_name = '.rgw.buckets.special'
pg_num = '8'
pgp_num = '8'
pool_create = 'sudo ceph osd pool create "%s" %s %s replicated' % (
pool_name, pg_num, pgp_num)
pool_create_exec = utils.exec_shell_cmd(pool_create)
if pool_create_exec is False:
raise TestExecError("Pool creation failed")
# create realm
realm_name = 'buz-tickets'
log.info('creating realm name')
realm_create = 'sudo radosgw-admin realm create --rgw-realm=%s' % realm_name
realm_create_exec = utils.exec_shell_cmd(realm_create)
if realm_create_exec is False:
raise TestExecError("cmd execution failed")
# sample output of create realm
"""
{
"id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62",
"name": "buz-tickets",
"current_period": "1950b710-3e63-4c41-a19e-46a715000980",
"epoch": 1
}
"""
log.info('modify zonegroup ')
modify = 'sudo radosgw-admin zonegroup modify --rgw-zonegroup=default --rgw-realm=%s --master' % realm_name
modify_exec = utils.exec_shell_cmd(modify)
if modify_exec is False:
raise TestExecError("cmd execution failed")
# get the zonegroup
zonegroup_file = 'zonegroup.json'
get_zonegroup = 'sudo radosgw-admin zonegroup --rgw-zonegroup=default get > %s' % zonegroup_file
get_zonegroup_exec = utils.exec_shell_cmd(get_zonegroup)
if get_zonegroup_exec is False:
raise TestExecError("cmd execution failed")
add_to_placement_targets = {"name": "special-placement", "tags": []}
fp = open(zonegroup_file, 'r')
zonegroup_txt = fp.read()
fp.close()
log.info('got zonegroup info: \n%s' % zonegroup_txt)
zonegroup = json.loads(zonegroup_txt)
log.info('adding placement targets')
zonegroup['placement_targets'].append(add_to_placement_targets)
with open(zonegroup_file, 'w') as fp:
json.dump(zonegroup, fp)
zonegroup_set = 'sudo radosgw-admin zonegroup set < %s' % zonegroup_file
zonegroup_set_exec = utils.exec_shell_cmd(zonegroup_set)
if zonegroup_set_exec is False:
raise TestExecError("cmd execution failed")
log.info('zone group update completed')
log.info('getting zone file')
# get zone
log.info('getting zone info')
zone_file = 'zone.json'
get_zone = 'sudo radosgw-admin zone --rgw-zone=default get > zone.json'
get_zone_exec = utils.exec_shell_cmd(get_zone)
if get_zone_exec is False:
raise TestExecError("cmd execution failed")
fp = open(zone_file, 'r')
zone_info = fp.read()
fp.close()
log.info('zone_info :\n%s' % zone_info)
zone_info_cleaned = json.loads(zone_info)
special_placement_info = {
"key": "special-placement",
"val": {
"index_pool": ".rgw.buckets.index",
"data_pool": ".rgw.buckets.special",
"data_extra_pool": ".rgw.buckets.extra"
}
}
log.info('adding special placement info')
zone_info_cleaned['placement_pools'].append(special_placement_info)
with open(zone_file, 'w+') as fp:
json.dump(zone_info_cleaned, fp)
zone_file_set = 'sudo radosgw-admin zone set < %s' % zone_file
zone_file_set_exec = utils.exec_shell_cmd(zone_file_set)
if zone_file_set_exec is False:
raise TestExecError("cmd execution failed")
log.info('zone info updated ')
zone_group_update_set = 'radosgw-admin period update --commit'
zone_group_update_set_exec = utils.exec_shell_cmd(zone_group_update_set)
log.info(zone_group_update_set_exec)
restarted = rgw_service.restart()
if restarted is False:
raise TestExecError("service restart failed")
if config.rgw_client == 'rgw':
log.info('client type is rgw')
rgw_user_info = s3_swift_lib.create_users(1)
auth = Auth(rgw_user_info)
rgw_conn = auth.do_auth()
# create bucket
bucket_name = utils.gen_bucket_name_from_userid(
rgw_user_info['user_id'], 0)
bucket = resuables.create_bucket(bucket_name, rgw_conn, rgw_user_info)
# create object
s3_object_name = utils.gen_s3_object_name(bucket_name, 0)
resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config,
rgw_user_info)
if config.rgw_client == 'swift':
log.info('client type is swift')
user_names = ['tuffy', 'scooby', 'max']
tenant = 'tenant'
umgmt = UserMgmt()
umgmt.create_tenant_user(tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0])
user_info = umgmt.create_subuser(tenant_name=tenant,
user_id=user_names[0])
auth = Auth(user_info)
rgw = auth.do_auth()
container_name = utils.gen_bucket_name_from_userid(
user_info['user_id'], rand_no=0)
container = s3_swift_lib.resource_op({
'obj': rgw,
'resource': 'put_container',
'args': [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
swift_object_name = utils.gen_s3_object_name(
'%s.container.%s' % (user_names[0], 0), 0)
log.info('object name: %s' % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info('object path: %s' % object_path)
object_size = utils.get_file_size(config.objects_size_range['min'],
config.objects_size_range['max'])
data_info = manage_data.io_generator(object_path, object_size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info('uploading object: %s' % object_path)
with open(object_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
def upload_mutipart_object(s3_object_name,
bucket,
TEST_DATA_PATH,
config,
user_info,
append_data=False,
append_msg=None):
log.info('s3 object name: %s' % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
log.info('s3 object path: %s' % s3_object_path)
s3_object_size = config.obj_size
split_size = config.split_size if hasattr(config, 'split_size') else 5
log.info('split size: %s' % split_size)
if append_data is True:
data_info = manage_data.io_generator(
s3_object_path,
s3_object_size,
op='append',
**{'message': '\n%s' % append_msg})
else:
data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if data_info is False:
TestExecError("data creation failed")
mp_dir = os.path.join(TEST_DATA_PATH, s3_object_name + '.mp.parts')
log.info('mp part dir: %s' % mp_dir)
log.info('making multipart object part dir')
mkdir = utils.exec_shell_cmd('sudo mkdir %s' % mp_dir)
if mkdir is False:
raise TestExecError('mkdir failed creating mp_dir_name')
utils.split_file(s3_object_path, split_size, mp_dir + "/")
parts_list = sorted(glob.glob(mp_dir + '/' + '*'))
log.info('parts_list: %s' % parts_list)
log.info('uploading s3 object: %s' % s3_object_path)
upload_info = dict(
{
'access_key': user_info['access_key'],
'upload_type': 'multipart'
}, **data_info)
s3_obj = s3lib.resource_op({
'obj': bucket,
'resource': 'Object',
'args': [s3_object_name],
})
log.info('initiating multipart upload')
mpu = s3lib.resource_op({
'obj': s3_obj,
'resource': 'initiate_multipart_upload',
'args': None,
'extra_info': upload_info
})
part_number = 1
parts_info = {'Parts': []}
log.info('no of parts: %s' % len(parts_list))
for each_part in parts_list:
log.info('trying to upload part: %s' % each_part)
part = mpu.Part(part_number)
# part_upload_response = part.upload(Body=open(each_part))
part_upload_response = s3lib.resource_op({
'obj':
part,
'resource':
'upload',
'kwargs':
dict(Body=open(each_part, mode="rb"))
})
if part_upload_response is not False:
response = HttpResponseParser(part_upload_response)
if response.status_code == 200:
log.info('part uploaded')
if config.local_file_delete is True:
log.info('deleting local file part')
utils.exec_shell_cmd('sudo rm -rf %s' % each_part)
else:
raise TestExecError("part uploading failed")
part_info = {
'PartNumber': part_number,
'ETag': part_upload_response['ETag']
}
parts_info['Parts'].append(part_info)
if each_part != parts_list[-1]:
# increase the part number only if the current part is not the last part
part_number += 1
log.info('curr part_number: %s' % part_number)
# log.info('parts_info so far: %s'% parts_info)
if len(parts_list) == part_number:
log.info('all parts upload completed')
mpu.complete(MultipartUpload=parts_info)
log.info('multipart upload complete for key: %s' % s3_object_name)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
# create user
config.user_count = 1
tenant1 = "MountEverest"
tenant2 = "Himalayas"
tenant1_user_info = s3lib.create_tenant_users(
tenant_name=tenant1, no_of_users_to_create=config.user_count)
tenant1_user1_info = tenant1_user_info[0]
tenant2_user_info = s3lib.create_tenant_users(
tenant_name=tenant2, no_of_users_to_create=config.user_count)
tenant2_user1_info = tenant2_user_info[0]
tenant1_user1_auth = Auth(tenant1_user1_info, ssl=config.ssl)
tenant2_user1_auth = Auth(tenant2_user1_info, ssl=config.ssl)
rgw_tenant1_user1 = tenant1_user1_auth.do_auth()
rgw_tenant1_user1_c = tenant1_user1_auth.do_auth_using_client()
rgw_tenant2_user1 = tenant2_user1_auth.do_auth()
rgw_tenant2_user1_c = tenant2_user1_auth.do_auth_using_client()
bucket_name1 = utils.gen_bucket_name_from_userid(
tenant1_user1_info["user_id"], rand_no=1)
t1_u1_bucket1 = reusable.create_bucket(
bucket_name1,
rgw_tenant1_user1,
tenant1_user1_info,
)
bucket_name2 = utils.gen_bucket_name_from_userid(
tenant1_user1_info["user_id"], rand_no=2)
t1_u1_bucket2 = reusable.create_bucket(
bucket_name2,
rgw_tenant1_user1,
tenant1_user1_info,
)
bucket_policy_generated = s3_bucket_policy.gen_bucket_policy(
tenants_list=[tenant1],
userids_list=[tenant2_user1_info["user_id"]],
actions_list=["CreateBucket"],
resources=[t1_u1_bucket1.name],
)
bucket_policy = json.dumps(bucket_policy_generated)
log.info("jsoned policy:%s\n" % bucket_policy)
log.info("bucket_policy_generated:%s\n" % bucket_policy_generated)
bucket_policy_obj = s3lib.resource_op({
"obj": rgw_tenant1_user1,
"resource": "BucketPolicy",
"args": [t1_u1_bucket1.name],
})
put_policy = s3lib.resource_op({
"obj":
bucket_policy_obj,
"resource":
"put",
"kwargs":
dict(ConfirmRemoveSelfBucketAccess=True, Policy=bucket_policy),
})
log.info("put policy response:%s\n" % put_policy)
if put_policy is False:
raise TestExecError(
"Resource execution failed: bucket creation faield")
if put_policy is not None:
response = HttpResponseParser(put_policy)
if response.status_code == 200 or response.status_code == 204:
log.info("bucket policy created")
else:
raise TestExecError("bucket policy creation failed")
else:
raise TestExecError("bucket policy creation failed")
# get policy
get_policy = rgw_tenant1_user1_c.get_bucket_policy(
Bucket=t1_u1_bucket1.name)
log.info("got bucket policy:%s\n" % get_policy["Policy"])
# modifying bucket policy to take new policy
if config.bucket_policy_op == "modify":
# adding new action list: ListBucket to existing action: CreateBucket
log.info("modifying buckey policy")
actions_list = ["ListBucket", "CreateBucket"]
actions = list(map(s3_bucket_policy.gen_action, actions_list))
bucket_policy2_generated = s3_bucket_policy.gen_bucket_policy(
tenants_list=[tenant1],
userids_list=[tenant2_user1_info["user_id"]],
actions_list=actions_list,
resources=[t1_u1_bucket1.name],
)
bucket_policy2 = json.dumps(bucket_policy2_generated)
put_policy = s3lib.resource_op({
"obj":
bucket_policy_obj,
"resource":
"put",
"kwargs":
dict(ConfirmRemoveSelfBucketAccess=True, Policy=bucket_policy2),
})
log.info("put policy response:%s\n" % put_policy)
if put_policy is False:
raise TestExecError(
"Resource execution failed: bucket creation faield")
if put_policy is not None:
response = HttpResponseParser(put_policy)
if response.status_code == 200 or response.status_code == 204:
log.info("bucket policy created")
else:
raise TestExecError("bucket policy creation failed")
else:
raise TestExecError("bucket policy creation failed")
get_modified_policy = rgw_tenant1_user1_c.get_bucket_policy(
Bucket=t1_u1_bucket1.name)
modified_policy = json.loads(get_modified_policy["Policy"])
log.info("got bucket policy:%s\n" % modified_policy)
actions_list_from_modified_policy = modified_policy["Statement"][0][
"Action"]
cleaned_actions_list_from_modified_policy = list(
map(str, actions_list_from_modified_policy))
log.info("cleaned_actions_list_from_modified_policy: %s" %
cleaned_actions_list_from_modified_policy)
log.info("actions list to be modified: %s" % actions)
cmp_val = utils.cmp(actions, cleaned_actions_list_from_modified_policy)
log.info("cmp_val: %s" % cmp_val)
if cmp_val != 0:
raise TestExecError("modification of bucket policy failed ")
if config.bucket_policy_op == "replace":
log.info("replacing new bucket policy")
new_policy_generated = s3_bucket_policy.gen_bucket_policy(
tenants_list=[tenant1],
userids_list=[tenant2_user1_info["user_id"]],
actions_list=["ListBucket"],
resources=[t1_u1_bucket2.name],
)
new_policy = json.dumps(new_policy_generated)
put_policy = s3lib.resource_op({
"obj":
bucket_policy_obj,
"resource":
"put",
"kwargs":
dict(ConfirmRemoveSelfBucketAccess=True, Policy=new_policy),
})
log.info("put policy response:%s\n" % put_policy)
if put_policy is False:
raise TestExecError(
"Resource execution failed: bucket creation faield")
if put_policy is not None:
response = HttpResponseParser(put_policy)
if response.status_code == 200 or response.status_code == 204:
log.info("new bucket policy created")
else:
raise TestExecError("bucket policy creation failed")
else:
raise TestExecError("bucket policy creation failed")
if config.bucket_policy_op == "delete":
log.info("in delete bucket policy")
delete_policy = s3lib.resource_op({
"obj": bucket_policy_obj,
"resource": "delete",
"args": None
})
if delete_policy is False:
raise TestExecError(
"Resource execution failed: bucket creation faield")
if delete_policy is not None:
response = HttpResponseParser(delete_policy)
if response.status_code == 200 or response.status_code == 204:
log.info("bucket policy deleted")
else:
raise TestExecError("bucket policy deletion failed")
else:
raise TestExecError("bucket policy deletion failed")
# confirming once again by calling get_bucket_policy
try:
rgw_tenant1_user1_c.get_bucket_policy(Bucket=t1_u1_bucket1.name)
raise TestExecError("bucket policy did not get deleted")
except boto3exception.ClientError as e:
log.info(e.response)
response = HttpResponseParser(e.response)
if response.error["Code"] == "NoSuchBucketPolicy":
log.info("bucket policy deleted")
else:
raise TestExecError("bucket policy did not get deleted")
# log.info('get_policy after deletion: %s' % get_policy)
# check sync status if a multisite cluster
reusable.check_sync_status()
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
test_info = AddTestInfo("create m buckets with n objects with bucket life cycle")
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
# create user
all_users_info = s3lib.create_users(config.user_count, config.cluster_name)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
# create buckets
if config.test_ops["create_bucket"] is True:
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=1
)
bucket = resuables.create_bucket(bucket_name, rgw_conn, each_user)
if config.test_ops["create_object"] is True:
# uploading data
log.info("s3 objects to create: %s" % config.objects_count)
for oc in range(config.objects_count):
s3_object_name = utils.gen_s3_object_name(bucket.name, oc)
resuables.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
)
bucket_life_cycle = s3lib.resource_op(
{
"obj": rgw_conn,
"resource": "BucketLifecycleConfiguration",
"args": [bucket.name],
}
)
life_cycle = basic_lifecycle_config(
prefix="key", days=20, id="rul1"
)
put_bucket_life_cycle = s3lib.resource_op(
{
"obj": bucket_life_cycle,
"resource": "put",
"kwargs": dict(LifecycleConfiguration=life_cycle),
}
)
log.info("put bucket life cycle:\n%s" % put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError("bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op(
{
"obj": rgw_conn2,
"resource": "get_bucket_lifecycle_configuration",
"kwargs": dict(Bucket=bucket.name),
}
)
if get_bucket_life_cycle_config is False:
raise TestExecError("bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(get_bucket_life_cycle_config)
if response.status_code == 200:
log.info("bucket life cycle retrieved")
else:
raise TestExecError(
"bucket lifecycle config retrieval failed"
)
else:
raise TestExecError("bucket life cycle retrieved")
test_info.success_status("test passed")
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
# create user
all_users_info = s3lib.create_users(config.user_count)
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
rgw_conn2 = auth.do_auth_using_client()
# create buckets
if config.test_ops["create_bucket"] is True:
log.info("no of buckets to create: %s" % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name = utils.gen_bucket_name_from_userid(
each_user["user_id"], rand_no=1)
bucket = reusable.create_bucket(bucket_name, rgw_conn,
each_user)
if config.test_ops["enable_versioning"] is True:
log.info("bucket versionig test on bucket: %s" %
bucket.name)
# bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name)
bucket_versioning = s3lib.resource_op({
"obj":
rgw_conn,
"resource":
"BucketVersioning",
"args": [bucket.name],
})
version_status = s3lib.resource_op({
"obj": bucket_versioning,
"resource": "status",
"args": None
})
if version_status is None:
log.info("bucket versioning still not enabled")
# enabling bucket versioning
version_enable_status = s3lib.resource_op({
"obj": bucket_versioning,
"resource": "enable",
"args": None
})
response = HttpResponseParser(version_enable_status)
if response.status_code == 200:
log.info("version enabled")
else:
raise TestExecError("version enable failed")
if config.test_ops["create_object"] is True:
# upload data
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
s3_object_name = utils.gen_s3_object_name(
bucket.name, oc)
if config.test_ops["version_count"] > 0:
for vc in range(config.test_ops["version_count"]):
log.info("version count for %s is %s" %
(s3_object_name, str(vc)))
log.info("modifying data: %s" % s3_object_name)
reusable.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
append_data=True,
append_msg="hello object for version: %s\n"
% str(vc),
)
else:
log.info("s3 objects to create: %s" %
config.objects_count)
reusable.upload_object(
s3_object_name,
bucket,
TEST_DATA_PATH,
config,
each_user,
)
bucket_life_cycle = s3lib.resource_op({
"obj": rgw_conn,
"resource": "BucketLifecycleConfiguration",
"args": [bucket.name],
})
life_cycle = basic_lifecycle_config(prefix="key",
days=20,
id="rul1")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(LifecycleConfiguration=life_cycle),
})
log.info("put bucket life cycle:\n%s" % put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield")
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError("bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
"get_bucket_lifecycle_configuration",
"kwargs":
dict(Bucket=bucket.name),
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(get_bucket_life_cycle_config)
if response.status_code == 200:
log.info("bucket life cycle retrieved")
else:
raise TestExecError(
"bucket lifecycle config retrieval failed")
else:
raise TestExecError("bucket life cycle retrieved")
if config.test_ops["create_object"] is True:
for oc in range(config.objects_count):
s3_object_name = utils.gen_s3_object_name(
bucket.name, oc)
if config.test_ops["version_count"] > 0:
if (config.test_ops.get("delete_versioned_object",
None) is True):
log.info(
"list all the versions of the object and delete the "
"current version of the object")
log.info("all versions for the object: %s\n" %
s3_object_name)
versions = bucket.object_versions.filter(
Prefix=s3_object_name)
t1 = []
for version in versions:
log.info(
"key_name: %s --> version_id: %s" %
(version.object_key,
version.version_id))
t1.append(version.version_id)
s3_object = s3lib.resource_op({
"obj":
rgw_conn,
"resource":
"Object",
"args": [bucket.name, s3_object_name],
})
# log.info('object version to delete: %s -> %s' % (versions[0].object_key,
# versions[0].version_id))
delete_response = s3_object.delete()
log.info("delete response: %s" %
delete_response)
if delete_response["DeleteMarker"] is True:
log.info(
"object delete marker is set to true")
else:
raise TestExecError(
"'object delete marker is set to false"
)
log.info(
"available versions for the object after delete marker is set"
)
t2 = []
versions_after_delete_marker_is_set = (
bucket.object_versions.filter(
Prefix=s3_object_name))
for version in versions_after_delete_marker_is_set:
log.info(
"key_name: %s --> version_id: %s" %
(version.object_key,
version.version_id))
t2.append(version.version_id)
t2.pop()
if t1 == t2:
log.info("versions remained intact")
else:
raise TestExecError(
"versions are not intact after delete marker is set"
)
# modify bucket lifecycle configuration, modify expiration days here for the test case.
if config.test_ops.get("modify_lifecycle", False) is True:
log.info("modifying lifecycle configuration")
life_cycle_modifed = basic_lifecycle_config(
prefix="key", days=15, id="rul1", status="Disabled")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(LifecycleConfiguration=life_cycle_modifed),
})
log.info("put bucket life cycle:\n%s" %
put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError(
"bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
"get_bucket_lifecycle_configuration",
"kwargs":
dict(Bucket=bucket.name),
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(
get_bucket_life_cycle_config)
modified_expiration_days = get_bucket_life_cycle_config[
"Rules"][0]["Expiration"]["Days"]
log.info("modified expiration days: %s" %
modified_expiration_days)
if (response.status_code == 200
and modified_expiration_days == 15):
log.info(
"bucket life cycle retrieved after modifying")
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after modifying"
)
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after modifying"
)
# disable bucket lifecycle configuration
if config.test_ops.get("disable_lifecycle", False) is True:
log.info("disabling lifecycle configuration")
life_cycle_disabled_config = basic_lifecycle_config(
prefix="key", days=20, id="rul1", status="Disabled")
put_bucket_life_cycle = s3lib.resource_op({
"obj":
bucket_life_cycle,
"resource":
"put",
"kwargs":
dict(
LifecycleConfiguration=life_cycle_disabled_config),
})
log.info("put bucket life cycle:\n%s" %
put_bucket_life_cycle)
if put_bucket_life_cycle is False:
raise TestExecError(
"Resource execution failed: bucket creation faield"
)
if put_bucket_life_cycle is not None:
response = HttpResponseParser(put_bucket_life_cycle)
if response.status_code == 200:
log.info("bucket life cycle added")
else:
raise TestExecError(
"bucket lifecycle addition failed")
else:
raise TestExecError("bucket lifecycle addition failed")
log.info("trying to retrieve bucket lifecycle config")
get_bucket_life_cycle_config = s3lib.resource_op({
"obj":
rgw_conn2,
"resource":
"get_bucket_lifecycle_configuration",
"kwargs":
dict(Bucket=bucket.name),
})
if get_bucket_life_cycle_config is False:
raise TestExecError(
"bucket lifecycle config retrieval failed")
if get_bucket_life_cycle_config is not None:
response = HttpResponseParser(
get_bucket_life_cycle_config)
if (response.status_code == 200
and get_bucket_life_cycle_config["Rules"][0]
["Status"] == "Disabled"):
log.info("disabled_status: %s" %
get_bucket_life_cycle_config["Rules"][0]
["Status"])
log.info(
"bucket life cycle retrieved after disabled")
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after disabled"
)
else:
raise TestExecError(
"bucket lifecycle config retrieval failed after disabled"
)
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
write_bucket_io_info = BucketIoInfo()
write_key_io_info = KeyIoInfo()
io_info_initialize.initialize(basic_io_structure.initial())
# create user
all_users_info = s3lib.create_users(config.user_count)
extra_user = s3lib.create_users(1)[0]
extra_user_auth = Auth(extra_user, ssl=config.ssl)
extra_user_conn = extra_user_auth.do_auth()
for each_user in all_users_info:
# authenticate
auth = Auth(each_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
s3_object_names = []
# create buckets
log.info('no of buckets to create: %s' % config.bucket_count)
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc)
log.info('creating bucket with name: %s' % bucket_name_to_create)
# bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create)
bucket = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Bucket',
'args': [bucket_name_to_create]})
# created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']})
created = s3lib.resource_op({'obj': bucket,
'resource': 'create',
'args': None,
'extra_info': {'access_key': each_user['access_key']}})
if created is False:
raise TestExecError("Resource execution failed: bucket creation faield")
if created is not None:
response = HttpResponseParser(created)
if response.status_code == 200:
log.info('bucket created')
else:
raise TestExecError("bucket creation failed")
else:
raise TestExecError("bucket creation failed")
# getting bucket version object
if config.test_ops['enable_version'] is True:
log.info('bucket versionig test on bucket: %s' % bucket.name)
# bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name)
bucket_versioning = s3lib.resource_op({'obj': rgw_conn,
'resource': 'BucketVersioning',
'args': [bucket.name]})
# checking the versioning status
# version_status = s3_ops.resource_op(bucket_versioning, 'status')
version_status = s3lib.resource_op({'obj': bucket_versioning,
'resource': 'status',
'args': None
})
if version_status is None:
log.info('bucket versioning still not enabled')
# enabling bucket versioning
# version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable')
version_enable_status = s3lib.resource_op({'obj': bucket_versioning,
'resource': 'enable',
'args': None,
})
response = HttpResponseParser(version_enable_status)
if response.status_code == 200:
log.info('version enabled')
write_bucket_io_info.add_versioning_status(each_user['access_key'],bucket.name,
VERSIONING_STATUS['ENABLED'])
else:
raise TestExecError("version enable failed")
if config.objects_count > 0:
log.info('s3 objects to create: %s' % config.objects_count)
for oc, s3_object_size in list(config.mapped_sizes.items()):
# versioning upload
s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc))
s3_object_names.append(s3_object_name)
log.info('s3 object name: %s' % s3_object_name)
log.info('versioning count: %s' % config.version_count)
s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc))
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
original_data_info = manage_data.io_generator(s3_object_path, s3_object_size)
if original_data_info is False:
TestExecError("data creation failed")
created_versions_count = 0
for vc in range(config.version_count):
log.info('version count for %s is %s' % (s3_object_name, str(vc)))
log.info('modifying data: %s' % s3_object_name)
modified_data_info = manage_data.io_generator(s3_object_path, s3_object_size,
op='append',
**{'message': '\nhello for version: %s\n'
% str(vc)})
if modified_data_info is False:
TestExecError("data modification failed")
log.info('uploading s3 object: %s' % s3_object_path)
upload_info = dict({'access_key': each_user['access_key'],
'versioning_status': VERSIONING_STATUS['ENABLED'],
'version_count_no': vc}, **modified_data_info)
s3_obj = s3lib.resource_op({'obj': bucket,
'resource': 'Object',
'args': [s3_object_name],
'extra_info': upload_info, })
object_uploaded_status = s3lib.resource_op({'obj': s3_obj,
'resource': 'upload_file',
'args': [modified_data_info['name']],
'extra_info': upload_info})
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info('object uploaded')
s3_obj = rgw_conn.Object(bucket.name, s3_object_name)
log.info('current_version_id: %s' % s3_obj.version_id)
key_version_info = basic_io_structure.version_info(
**{'version_id': s3_obj.version_id,
'md5_local': upload_info['md5'],
'count_no': vc,
'size': upload_info['size']})
log.info('key_version_info: %s' % key_version_info)
write_key_io_info.add_versioning_info(each_user['access_key'], bucket.name,
s3_object_path, key_version_info)
created_versions_count += 1
log.info('created_versions_count: %s' % created_versions_count)
log.info('adding metadata')
metadata1 = {"m_data1": "this is the meta1 for this obj"}
s3_obj.metadata.update(metadata1)
metadata2 = {"m_data2": "this is the meta2 for this obj"}
s3_obj.metadata.update(metadata2)
log.info('metadata for this object: %s' % s3_obj.metadata)
log.info('metadata count for object: %s' % (len(s3_obj.metadata)))
if not s3_obj.metadata:
raise TestExecError('metadata not created even adding metadata')
versions = bucket.object_versions.filter(Prefix=s3_object_name)
created_versions_count_from_s3 = len([v.version_id for v in versions])
log.info('created versions count on s3: %s' % created_versions_count_from_s3)
if created_versions_count is created_versions_count_from_s3:
log.info('no new versions are created when added metdata')
else:
raise TestExecError("version count missmatch, "
"possible creation of version on adding metadata")
s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download")
object_downloaded_status = s3lib.resource_op({'obj': bucket,
'resource': 'download_file',
'args': [s3_object_name,
s3_object_download_path],
})
if object_downloaded_status is False:
raise TestExecError("Resource execution failed: object download failed")
if object_downloaded_status is None:
log.info('object downloaded')
# checking md5 of the downloaded file
s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path)
log.info('downloaded_md5: %s' % s3_object_downloaded_md5)
log.info('uploaded_md5: %s' % modified_data_info['md5'])
# tail_op = utils.exec_shell_cmd('tail -l %s' % s3_object_download_path)
log.info('all versions for the object: %s\n' % s3_object_name)
versions = bucket.object_versions.filter(Prefix=s3_object_name)
for version in versions:
log.info('key_name: %s --> version_id: %s' % (version.object_key, version.version_id))
if config.test_ops.get('set_acl', None) is True:
s3_obj_acl = s3lib.resource_op({'obj': rgw_conn,
'resource': 'ObjectAcl',
'args': [bucket.name, s3_object_name]})
# setting acl to private, just need to set to any acl and
# check if its set - check by response code
acls_set_status = s3_obj_acl.put(ACL='private')
response = HttpResponseParser(acls_set_status)
if response.status_code == 200:
log.info('ACLs set')
else:
raise TestExecError("Acls not Set")
# get obj details based on version id
for version in versions:
log.info('getting info for version id: %s' % version.version_id)
obj = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('obj get detils :%s\n' % (obj.get(VersionId=version.version_id)))
if config.test_ops['copy_to_version'] is True:
# reverting object to one of the versions ( randomly chosen )
version_id_to_copy = random.choice([v.version_id for v in versions])
log.info('version_id_to_copy: %s' % version_id_to_copy)
s3_obj = rgw_conn.Object(bucket.name, s3_object_name)
log.info('current version_id: %s' % s3_obj.version_id)
copy_response = s3_obj.copy_from(CopySource={'Bucket': bucket.name,
'Key': s3_object_name,
'VersionId': version_id_to_copy})
log.info('copy_response: %s' % copy_response)
if copy_response is None:
raise TestExecError("copy object from version id failed")
# current_version_id = copy_response['VersionID']
log.info('current_version_id: %s' % s3_obj.version_id)
# delete the version_id_to_copy object
s3_obj.delete(VersionId=version_id_to_copy)
log.info('all versions for the object after the copy operation: %s\n' % s3_object_name)
for version in versions:
log.info(
'key_name: %s --> version_id: %s' % (version.object_key, version.version_id))
# log.info('downloading current s3object: %s' % s3_object_name)
# s3_obj.download_file(s3_object_name + ".download")
if config.test_ops['delete_object_versions'] is True:
log.info('deleting s3_obj keys and its versions')
s3_obj = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('deleting versions for s3 obj: %s' % s3_object_name)
for version in versions:
log.info('trying to delete obj version: %s' % version.version_id)
del_obj_version = s3lib.resource_op({'obj': s3_obj,
'resource': 'delete',
'kwargs': dict(VersionId=version.version_id)})
log.info('response:\n%s' % del_obj_version)
if del_obj_version is not None:
response = HttpResponseParser(del_obj_version)
if response.status_code == 204:
log.info('version deleted ')
write_key_io_info.delete_version_info(each_user['access_key'], bucket.name,
s3_object_path, version.version_id)
else:
raise TestExecError("version deletion failed")
else:
raise TestExecError("version deletion failed")
log.info('available versions for the object')
versions = bucket.object_versions.filter(Prefix=s3_object_name)
for version in versions:
log.info('key_name: %s --> version_id: %s' % (
version.object_key, version.version_id))
if config.test_ops.get('delete_from_extra_user') is True:
log.info('trying to delete objects from extra user')
s3_obj = s3lib.resource_op({'obj': extra_user_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('deleting versions for s3 obj: %s' % s3_object_name)
for version in versions:
log.info('trying to delete obj version: %s' % version.version_id)
del_obj_version = s3lib.resource_op({'obj': s3_obj,
'resource': 'delete',
'kwargs': dict(
VersionId=version.version_id)})
log.info('response:\n%s' % del_obj_version)
if del_obj_version is not False:
response = HttpResponseParser(del_obj_version)
if response.status_code == 204:
log.info('version deleted ')
write_key_io_info.delete_version_info(each_user['access_key'],
bucket.name,
s3_object_path,
version.version_id)
raise TestExecError("version and deleted, this should not happen")
else:
log.info('version did not delete, expected behaviour')
else:
log.info('version did not delete, expected behaviour')
if config.local_file_delete is True:
log.info('deleting local file')
utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_path)
if config.test_ops['suspend_version'] is True:
log.info('suspending versioning')
# suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend')
suspend_version_status = s3lib.resource_op({'obj': bucket_versioning,
'resource': 'suspend',
'args': None})
response = HttpResponseParser(suspend_version_status)
if response.status_code == 200:
log.info('versioning suspended')
write_bucket_io_info.add_versioning_status(each_user['access_key'], bucket.name,
VERSIONING_STATUS['SUSPENDED'])
else:
raise TestExecError("version suspend failed")
# getting all objects in the bucket
log.info('getting all objects in the bucket')
objects = s3lib.resource_op({'obj': bucket,
'resource': 'objects',
'args': None})
log.info('objects :%s' % objects)
all_objects = s3lib.resource_op({'obj': objects,
'resource': 'all',
'args': None})
log.info('all objects: %s' % all_objects)
log.info('all objects2 :%s ' % bucket.objects.all())
for obj in all_objects:
log.info('object_name: %s' % obj.key)
versions = bucket.object_versions.filter(Prefix=obj.key)
log.info('displaying all versions of the object')
for version in versions:
log.info(
'key_name: %s --> version_id: %s' % (version.object_key, version.version_id))
if config.test_ops.get('suspend_from_extra_user') is True:
log.info('suspending versioning from extra user')
# suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend')
bucket_versioning = s3lib.resource_op({'obj': extra_user_conn,
'resource': 'BucketVersioning',
'args': [bucket.name]})
suspend_version_status = s3lib.resource_op({'obj': bucket_versioning,
'resource': 'suspend',
'args': None})
if suspend_version_status is not False:
response = HttpResponseParser(suspend_version_status)
if response.status_code == 200:
log.info('versioning suspended')
write_bucket_io_info.add_versioning_status(each_user['access_key'], bucket.name,
VERSIONING_STATUS['SUSPENDED'])
raise TestExecError('version suspended, this should not happen')
else:
log.info('versioning not suspended, expected behaviour')
if config.test_ops.get('upload_after_suspend') is True:
log.info('trying to upload after suspending versioning on bucket')
for oc, s3_object_size in list(config.mapped_sizes.items()):
# non versioning upload
s3_object_name = s3_object_names[oc] + ".after_version_suspending"
log.info('s3 object name: %s' % s3_object_name)
s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name)
non_version_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op="append",
**{
'message': '\nhello for non version\n'})
if non_version_data_info is False:
TestExecError("data creation failed")
log.info('uploading s3 object: %s' % s3_object_path)
upload_info = dict({'access_key': each_user['access_key'],
'versioning_status': 'suspended'},**non_version_data_info)
s3_obj = s3lib.resource_op({'obj': bucket,
'resource': 'Object',
'args': [s3_object_name],
'extra_info': upload_info})
object_uploaded_status = s3lib.resource_op({'obj': s3_obj,
'resource': 'upload_file',
'args': [non_version_data_info['name']],
'extra_info': upload_info})
if object_uploaded_status is False:
raise TestExecError("Resource execution failed: object upload failed")
if object_uploaded_status is None:
log.info('object uploaded')
s3_obj = s3lib.resource_op({'obj': rgw_conn,
'resource': 'Object',
'args': [bucket.name, s3_object_name]})
log.info('version_id: %s' % s3_obj.version_id)
if s3_obj.version_id is None:
log.info('Versions are not created after suspending')
else:
raise TestExecError('Versions are created even after suspending')
s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download")
object_downloaded_status = s3lib.resource_op({'obj': bucket,
'resource': 'download_file',
'args': [s3_object_name,
s3_object_download_path],
})
if object_downloaded_status is False:
raise TestExecError("Resource execution failed: object download failed")
if object_downloaded_status is None:
log.info('object downloaded')
# checking md5 of the downloaded file
s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path)
log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5)
log.info('s3_object_uploaded_md5: %s' % non_version_data_info['md5'])
if config.local_file_delete is True:
utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_path)
def test_exec(config):
test_info = AddTestInfo('test swift user key gen')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
try:
test_info.started_info()
# preparing data
user_names = ['tuffy', 'scooby', 'max']
tenant = 'tenant'
tenant_user_info = umgmt.create_tenant_user(
tenant_name=tenant,
user_id=user_names[0],
displayname=user_names[0],
cluster_name=config.cluster_name)
user_info = umgmt.create_subuser(tenant_name=tenant,
user_id=user_names[0],
cluster_name=config.cluster_name)
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
container_name = utils.gen_bucket_name_from_userid(
user_info['user_id'], rand_no=cc)
container = swiftlib.resource_op({
'obj': rgw,
'resource': 'put_container',
'args': [container_name]
})
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield")
for oc in range(config.objects_count):
swift_object_name = utils.gen_s3_object_name(
'%s.container.%s' % (user_names[0], cc), oc)
log.info('object name: %s' % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info('object path: %s' % object_path)
object_size = utils.get_file_size(
config.objects_size_range['min'],
config.objects_size_range['max'])
data_info = manage_data.io_generator(object_path, object_size)
# upload object
if data_info is False:
TestExecError("data creation failed")
log.info('uploading object: %s' % object_path)
with open(object_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
# download object
swift_object_download_fname = swift_object_name + ".download"
log.info('download object name: %s' %
swift_object_download_fname)
swift_object_download_path = os.path.join(
TEST_DATA_PATH, swift_object_download_fname)
log.info('download object path: %s' %
swift_object_download_path)
swift_object_downloaded = rgw.get_object(
container_name, swift_object_name)
with open(swift_object_download_path, 'w') as fp:
fp.write(swift_object_downloaded[1])
# modify and re-upload
log.info('appending new message to test_data')
message_to_append = 'adding new msg after download'
fp = open(swift_object_download_path, 'a+')
fp.write(message_to_append)
fp.close()
with open(swift_object_download_path, 'r') as fp:
rgw.put_object(container_name,
swift_object_name,
contents=fp.read(),
content_type='text/plain')
# delete object
log.info('deleting swift object')
rgw.delete_object(container_name, swift_object_name)
# delete container
log.info('deleting swift container')
rgw.delete_container(container_name)
test_info.success_status('test passed')
sys.exit(0)
except Exception, e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
test_info = AddTestInfo("test swift user key gen")
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
# preparing data
user_names = ["tuffy", "scooby", "max"]
tenant1 = "tenant"
cmd = (
'radosgw-admin user create --uid=%s --display-name="%s" --tenant=%s --cluster %s'
% (user_names[0], user_names[0], tenant1, config.cluster_name)
)
out = utils.exec_shell_cmd(cmd)
if out is False:
raise TestExecError("RGW User creation error")
log.info("output :%s" % out)
v1_as_json = json.loads(out)
log.info("creted user_id: %s" % v1_as_json["user_id"])
cmd2 = (
"radosgw-admin subuser create --uid=%s$%s --subuser=%s:swift --tenant=%s --access=full --cluster %s"
% (tenant1, user_names[0], user_names[0], tenant1, config.cluster_name)
)
out2 = utils.exec_shell_cmd(cmd2)
if out2 is False:
raise TestExecError("sub-user creation error")
v2_as_json = json.loads(out2)
log.info("created subuser: %s" % v2_as_json["subusers"][0]["id"])
cmd3 = (
"radosgw-admin key create --subuser=%s:swift --uid=%s$%s --tenant=%s --key-type=swift --gen-secret "
"--cluster %s"
% (user_names[0], user_names[0], tenant1, tenant1, config.cluster_name)
)
out3 = utils.exec_shell_cmd(cmd3)
if out3 is False:
raise TestExecError("secret_key gen error")
v3_as_json = json.loads(out3)
log.info(
"created subuser: %s\nsecret_key generated: %s"
% (
v3_as_json["swift_keys"][0]["user"],
v3_as_json["swift_keys"][0]["secret_key"],
)
)
user_info = {
"user_id": v3_as_json["swift_keys"][0]["user"],
"key": v3_as_json["swift_keys"][0]["secret_key"],
}
auth = Auth(user_info)
rgw = auth.do_auth()
for cc in range(config.container_count):
container_name = utils.gen_bucket_name_from_userid(
user_info["user_id"], rand_no=cc
)
container = swiftlib.resource_op(
{"obj": rgw, "resource": "put_container", "args": [container_name]}
)
if container is False:
raise TestExecError(
"Resource execution failed: container creation faield"
)
for oc in range(config.objects_count):
swift_object_name = utils.gen_s3_object_name(
"%s.container.%s" % (user_names[0], cc), oc
)
log.info("object name: %s" % swift_object_name)
object_path = os.path.join(TEST_DATA_PATH, swift_object_name)
log.info("object path: %s" % object_path)
object_size = utils.get_file_size(
config.objects_size_range["min"], config.objects_size_range["max"]
)
data_info = manage_data.io_generator(object_path, object_size)
if data_info is False:
TestExecError("data creation failed")
log.info("uploading object: %s" % object_path)
with open(object_path, "r") as fp:
rgw.put_object(
container_name,
swift_object_name,
contents=fp.read(),
content_type="text/plain",
)
test_info.success_status("test passed")
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status("test failed")
sys.exit(1)
def test_exec(config):
test_info = AddTestInfo('test versioning with objects')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
write_bucket_io_info = BucketIoInfo()
write_key_io_info = KeyIoInfo()
try:
test_info.started_info()
version_count = 3
# create user
s3_user = s3lib.create_users(1)[0]
# authenticate
auth = Auth(s3_user, ssl=config.ssl)
rgw_conn = auth.do_auth()
b1_name = 'bucky.1e' # bucket 1
b1_k1_name = b1_name + ".key.1" # key1
b1_k2_name = b1_name + ".key.2" # key2
b2_name = 'bucky.2e' # bucket 2
b2_k1_name = b2_name + ".key.1" # key1
b2_k2_name = b2_name + ".key.2" # key2
b1 = resuables.create_bucket(b1_name, rgw_conn, s3_user)
b2 = resuables.create_bucket(b2_name, rgw_conn, s3_user)
# enable versioning on b1
resuables.enable_versioning(b1, rgw_conn, s3_user,
write_bucket_io_info)
# upload object to version enabled bucket b1
obj_sizes = list(config.mapped_sizes.values())
config.obj_size = obj_sizes[0]
for vc in range(version_count):
resuables.upload_object(b1_k1_name,
b1,
TEST_DATA_PATH,
config,
s3_user,
append_data=True,
append_msg='hello vc count: %s' % str(vc))
# upload object to non version bucket b2
config.obj_size = obj_sizes[1]
resuables.upload_object(b2_k1_name, b2, TEST_DATA_PATH, config,
s3_user)
# copy b2_k1 to b1 and check if version id is created, expectation: version id should be created
# copy b1_k1 to b2 and check if version id is created, expectation: version id should not be present
b1_k2 = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'Object',
'args': [b1.name, b1_k2_name]
})
b2_k2 = s3lib.resource_op({
'obj': rgw_conn,
'resource': 'Object',
'args': [b2.name, b2_k2_name]
})
log.info(
'copy from b2_k1 key to b1_k2 key to bucket 1 -> version enabled bucket'
)
copy_response = b1_k2.copy_from(CopySource={
'Bucket': b2.name,
'Key': b2_k1_name,
})
log.info('copy_response: %s' % copy_response)
if copy_response is None:
raise TestExecError("copy object failed")
log.info('checking if copies object has version id created')
b1_k2_version_id = b1_k2.version_id
log.info('version id: %s' % b1_k2_version_id)
if b1_k2_version_id is None:
raise TestExecError(
'Version ID not created for the copied object on to the versioned enabled bucket'
)
else:
log.info(
'Version ID created for the copied object on to the versioned bucket'
)
all_objects_in_b1 = b1.objects.all()
log.info('all objects in bucket 1')
for obj in all_objects_in_b1:
log.info('object_name: %s' % obj.key)
versions = b1.object_versions.filter(Prefix=obj.key)
log.info('displaying all versions of the object')
for version in versions:
log.info('key_name: %s --> version_id: %s' %
(version.object_key, version.version_id))
log.info('-------------------------------------------')
log.info(
'copy from b1_k1 key to b2_k2 to bucket 2 -> non version bucket')
copy_response = b2_k2.copy_from(CopySource={
'Bucket': b1.name,
'Key': b1_k1_name,
})
log.info('copy_response: %s' % copy_response)
if copy_response is None:
raise TestExecError("copy object failed")
log.info('checking if copies object has version id created')
b2_k2_version_id = b2_k2.version_id
log.info('version id: %s' % b2_k2_version_id)
if b2_k2_version_id is None:
log.info(
'Version ID not created for the copied object on to the non versioned bucket'
)
else:
raise TestExecError(
'Version ID created for the copied object on to the non versioned bucket'
)
all_objects_in_b2 = b2.objects.all()
log.info('all objects in bucket 2')
for obj in all_objects_in_b2:
log.info('object_name: %s' % obj.key)
versions = b2.object_versions.filter(Prefix=obj.key)
log.info('displaying all versions of the object')
for version in versions:
log.info('key_name: %s --> version_id: %s' %
(version.object_key, version.version_id))
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(rgw_user_info_file, config):
test_info = AddTestInfo('Test Basic IO on S3')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
write_user_info = AddUserInfo()
try:
test_info.started_info()
with open(rgw_user_info_yaml, 'r') as f:
rgw_user_info = yaml.load(f)
mount_point = rgw_user_info['nfs_mnt_point']
nfs_ganesha = PrepNFSGanesha(rgw_user_info_file=rgw_user_info_file)
mounted = nfs_ganesha.initialize(write_io_info=False)
if mounted is False:
raise TestExecError("mount failed")
if nfs_ganesha.rgw_user_info[
'nfs_version'] == 4 and nfs_ganesha.rgw_user_info[
'Pseudo'] is not None:
log.info('nfs version: 4')
log.info('adding Pseudo path to writable mount point')
mount_point = os.path.join(mount_point,
nfs_ganesha.rgw_user_info['Pseudo'])
log.info('writable mount point with Pseudo: %s' % mount_point)
log.info('authenticating rgw user')
# authenticate
auth = Auth(rgw_user_info)
rgw_conn = auth.do_auth()
# add user_info io_info yaml file
user_info_add = basic_io_structure.user(**rgw_user_info)
write_user_info.add_user_info(user_info_add)
if config.io_op_config.get('create', None) is True:
# create buckets
for bc in range(config.bucket_count):
bucket_name_to_create = utils.gen_bucket_name_from_userid(
rgw_user_info['user_id'], rand_no=bc)
bucket = s3_reusables.create_bucket(bucket_name_to_create,
rgw_conn, rgw_user_info)
# uploading data
log.info('s3 objects to create: %s' % config.objects_count)
for oc in range(config.objects_count):
s3_object_name = utils.gen_s3_object_name(
bucket_name_to_create, oc)
s3_reusables.upload_object(s3_object_name, bucket,
TEST_DATA_PATH, config,
rgw_user_info)
log.info('verification Starts on NFS mount after %s seconds' %
SLEEP_TIME)
time.sleep(SLEEP_TIME)
read_io_info_on_nfs = ReadIOInfoOnNFS(mount_point)
read_io_info_on_nfs.yaml_fname = 'io_info.yaml'
read_io_info_on_nfs.initialize_verify_io()
read_io_info_on_nfs.verify_if_basedir_created()
read_io_info_on_nfs.verify_if_files_created()
log.info('verification complete, data intact')
created_buckets = read_io_info_on_nfs.base_dirs
created_objects = read_io_info_on_nfs.files
if config.io_op_config.get('delete', None) is True:
log.info('delete operation starts')
for bucket_name in created_buckets:
bucket = s3lib.resource_op({
'obj':
rgw_conn,
'resource':
'Bucket',
'args': [os.path.basename(bucket_name)]
}) # buckets are base dirs in NFS
objects = s3lib.resource_op({
'obj': bucket,
'resource': 'objects',
'args': None
})
log.info('deleting all objects in bucket')
objects_deleted = s3lib.resource_op({
'obj': objects,
'resource': 'delete',
'args': None
})
log.info('objects_deleted: %s' % objects_deleted)
if objects_deleted is False:
raise TestExecError(
'Resource execution failed: Object deletion failed'
)
if objects_deleted is not None:
response = HttpResponseParser(objects_deleted[0])
if response.status_code == 200:
log.info('objects deleted ')
else:
raise TestExecError("objects deletion failed")
else:
raise TestExecError("objects deletion failed")
log.info('deleting bucket: %s' % bucket.name)
bucket_deleted_status = s3lib.resource_op({
'obj': bucket,
'resource': 'delete',
'args': None
})
log.info('bucket_deleted_status: %s' %
bucket_deleted_status)
if bucket_deleted_status is not None:
response = HttpResponseParser(bucket_deleted_status)
if response.status_code == 204:
log.info('bucket deleted ')
else:
raise TestExecError("bucket deletion failed")
else:
raise TestExecError("bucket deletion failed")
log.info(
'verification on NFS will start after %s seconds for delete operation'
% SLEEP_TIME)
time.sleep(200)
for basedir in created_buckets:
exists = os.path.exists(basedir)
log.info('exists status: %s' % exists)
if exists is True:
raise TestExecError(
"Basedir or Basedir: %s not deleted on NFS" %
basedir)
log.info('basedirs deleted')
for each_file in created_objects:
log.info('verifying existence for: %s' % each_file['file'])
exists = os.path.exists(each_file['file'])
if exists:
raise TestExecError("files not created")
log.info('file deleted')
log.info(
'verification of files complete, files exists and data intact'
)
if config.io_op_config.get('move', None) is True:
log.info('move operation starts')
for each_file in created_objects:
# in s3 move operation is achieved by copying the same object with the new name and
# deleting the old object
log.info('move operation for :%s' % each_file['file'])
new_obj_name = os.path.basename(
each_file['file']) + ".moved"
log.info('new file name: %s' % new_obj_name)
new_object = s3lib.resource_op({
'obj':
rgw_conn,
'resource':
'Object',
'args': [each_file['bucket'], new_obj_name],
})
new_object.copy_from(
CopySource='%s/%s' %
(each_file['bucket'],
os.path.basename(
each_file['file']))) # old object name
old_object = s3lib.resource_op({
'obj':
rgw_conn,
'resource':
'Object',
'args': [
each_file['bucket'],
os.path.basename(each_file['file'])
],
})
old_object.delete()
each_file['file'] = os.path.abspath(
os.path.join(mount_point, each_file['bucket'],
new_obj_name))
log.info(
'verification on NFS for move operation will start after %s seconds'
% SLEEP_TIME)
time.sleep(SLEEP_TIME)
read_io_info_on_nfs.verify_if_files_created()
log.info('move completed, data intact')
test_info.success_status('test passed')
sys.exit(0)
except Exception, e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
def test_exec(config):
test_info = AddTestInfo('test bucket policy')
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
try:
test_info.started_info()
# preparing data
user_names = ['user1', 'user2', 'user3']
Bucket_names = ['bucket1', 'bucket2', 'bucket3']
object_names = ['o1', 'o2']
tenant1 = 'tenant1'
tenant2 = 'tenant2'
t1_u1_info = create_tenant_user(tenant_name=tenant1,
user_id=user_names[0],
cluster_name=config.cluster_name)
t1_u1_auth = Auth(t1_u1_info)
t1_u1 = t1_u1_auth.do_auth()
t2_u1_info = create_tenant_user(tenant_name=tenant2,
user_id=user_names[0],
cluster_name=config.cluster_name)
t2_u1_auth = Auth(t2_u1_info)
t2_u1 = t2_u1_auth.do_auth()
t1_u1_b1 = resuables.create_bucket(bucket_name=Bucket_names[0],
rgw=t1_u1,
user_info=t1_u1_info)
t2_u1_b1 = resuables.create_bucket(bucket_name=Bucket_names[0],
rgw=t2_u1,
user_info=t2_u1_info)
resuables.upload_object(s3_object_name=object_names[0],
bucket=t1_u1_b1,
TEST_DATA_PATH=TEST_DATA_PATH,
config=config,
user_info=t1_u1_info)
resuables.upload_object(s3_object_name=object_names[0],
bucket=t2_u1_b1,
TEST_DATA_PATH=TEST_DATA_PATH,
config=config,
user_info=t1_u1_info)
t2_u2_info = create_tenant_user(tenant_name=tenant2,
user_id=user_names[1],
cluster_name=config.cluster_name)
t2_u2_auth = Auth(t2_u2_info)
t2_u2 = t2_u2_auth.do_auth()
# will try to access the bucket and objects in both tenants
# access t1_u1_b1
log.info('trying to access tenant1->user1->bucket1')
t1_u1_b1_from_t2_u2 = s3lib.resource_op({
'obj': t2_u2,
'resource': 'Bucket',
'args': [Bucket_names[0]]
})
log.info(
'trying to download tenant1->user1->bucket1->object1 from tenant2->user2'
)
download_path1 = TEST_DATA_PATH + "/t1_u1_b1_%s.download" % object_names[
0]
t1_u1_b1_o1_download = s3lib.resource_op({
'obj':
t1_u1_b1_from_t2_u2,
'resource':
'download_file',
'args': [object_names[0], download_path1]
})
if t1_u1_b1_o1_download is False:
log.info('object not downloaded\n')
if t1_u1_b1_o1_download is None:
raise TestExecError(
"object downloaded for tenant1->user1->bucket1->object1, this should not happen"
)
log.info(
'trying to access tenant2->user1->bucket1 from user2 in tenant 2')
t2_u1_b1_from_t2_u2 = s3lib.resource_op({
'obj': t2_u2,
'resource': 'Bucket',
'args': [Bucket_names[0]]
})
log.info(
'trying to download tenant2->user1->bucket1->object1 from tenant2->user2'
)
download_path2 = TEST_DATA_PATH + "/t2_u1_b1_%s.download" % object_names[
0]
t2_u1_b1_o1_download = s3lib.resource_op({
'obj':
t2_u1_b1_from_t2_u2,
'resource':
'download_file',
'args': [object_names[0], download_path2]
})
if t2_u1_b1_o1_download is False:
log.info('object did not download, worked as expected')
if t1_u1_b1_o1_download is None:
raise TestExecError(
'object downloaded\n'
'downloaded tenant2->user1->bucket1->object1, this should not happen'
)
test_info.success_status('test passed')
sys.exit(0)
except Exception as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
except TestExecError as e:
log.info(e)
log.info(traceback.format_exc())
test_info.failed_status('test failed')
sys.exit(1)
