def upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, user_info, append_data=False, append_msg=None): log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) s3_object_size = config.obj_size if append_data is True: data_info = manage_data.io_generator(s3_object_path, s3_object_size, op='append', **{'message': '\n%s' % append_msg}) else: data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': user_info['access_key']}, **data_info) s3_obj = s3lib.resource_op({'obj': bucket, 'resource': 'Object', 'args': [s3_object_name], }) object_uploaded_status = s3lib.resource_op({'obj': s3_obj, 'resource': 'upload_file', 'args': [s3_object_path], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded')
def upload_object_with_tagging( s3_object_name, bucket, TEST_DATA_PATH, config, user_info, obj_tag, append_data=False, append_msg=None, ): log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) s3_object_size = config.obj_size if append_data is True: data_info = manage_data.io_generator( s3_object_path, s3_object_size, op="append", **{"message": "\n%s" % append_msg}, ) else: data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") log.info("uploading s3 object with object tagging enabled: %s" % s3_object_path) bucket.put_object(Key=s3_object_name, Body=s3_object_path, Tagging=obj_tag)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) # preparing data user_names = ['tuffy', 'scooby', 'max'] tenant1 = 'tenant' cmd = 'radosgw-admin user create --uid=%s --display-name="%s" --tenant=%s' % ( user_names[0], user_names[0], tenant1) out = utils.exec_shell_cmd(cmd) if out is False: raise TestExecError("RGW User creation error") log.info('output :%s' % out) v1_as_json = json.loads(out) log.info('creted user_id: %s' % v1_as_json['user_id']) cmd2 = 'radosgw-admin subuser create --uid=%s$%s --subuser=%s:swift --tenant=%s --access=full' % ( tenant1, user_names[0], user_names[0], tenant1) out2 = utils.exec_shell_cmd(cmd2) if out2 is False: raise TestExecError("sub-user creation error") v2_as_json = json.loads(out2) log.info('created subuser: %s' % v2_as_json['subusers'][0]['id']) cmd3 = 'radosgw-admin key create --subuser=%s:swift --uid=%s$%s --tenant=%s --key-type=swift --gen-secret' % ( user_names[0], user_names[0], tenant1, tenant1) out3 = utils.exec_shell_cmd(cmd3) if out3 is False: raise TestExecError("secret_key gen error") v3_as_json = json.loads(out3) log.info('created subuser: %s\nsecret_key generated: %s' % ( v3_as_json['swift_keys'][0]['user'], v3_as_json['swift_keys'][0]['secret_key'])) user_info = {'user_id': v3_as_json['swift_keys'][0]['user'], 'key': v3_as_json['swift_keys'][0]['secret_key']} auth = Auth(user_info) rgw = auth.do_auth() for cc in range(config.container_count): container_name = utils.gen_bucket_name_from_userid(user_info['user_id'], rand_no=cc) container = swiftlib.resource_op({'obj': rgw, 'resource': 'put_container', 'args': [container_name]}) if container is False: raise TestExecError("Resource execution failed: container creation faield") for oc,size in list(config.mapped_sizes.items()): swift_object_name = utils.gen_s3_object_name('%s.container.%s' % (user_names[0], cc), oc) log.info('object name: %s' % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info('object path: %s' % object_path) data_info = manage_data.io_generator(object_path, size) if data_info is False: TestExecError("data creation failed") log.info('uploading object: %s' % object_path) with open(object_path, 'r') as fp: rgw.put_object(container_name, swift_object_name, contents=fp.read(), content_type='text/plain') # check for any crashes during the execution crash_info=reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def upload_file(bucket_name, file_name=None, file_size=1024, test_data_path=None): """ Uploads file to the bucket Args: bucket_name(str): Name of the bucket file_name(str): Name of the file to be uploaded file_size(int): Size of the file to be uploaded, defaults to 1024 test_data_path(str): Local test data path Returns: File information """ # If no file_name passed, it generates file_name and returns in file information # It is to have support for file which is already created if file_name is None: file_name = utils.gen_s3_object_name(bucket_name, 1) local_file_path = test_data_path + "/" + file_name file_info = io_generator(local_file_path, file_size) file_info["name"] = file_name upload_file_method = S3CMD(operation="put") remote_s3_path = f"s3://{bucket_name}/{file_name}" command = upload_file_method.command( params=[local_file_path, remote_s3_path]) try: upload_file_response = exec_shell_cmd(command) log.debug(f"Response for upload file command: {upload_file_response}") except Exception as e: raise S3CommandExecError(message=str(e)) assert "100%" in str( upload_file_response), "upload file operation not succeeded" return file_info
def fill_container(rgw, container_name, user_id, oc, cc, size): """ Uploads objects to the container Args: rgw(object): RGW object container_name(str): Container name user_id(str): User ID oc(int): object count cc(int): container count size(int): Object size """ swift_object_name = utils.gen_s3_object_name("%s.container.%s" % (user_id, cc), oc) log.info("object name: %s" % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info("object path: %s" % object_path) data_info = manage_data.io_generator(object_path, size) # upload object if data_info is False: raise TestExecError("data creation failed") log.info("uploading object: %s" % object_path) with open(object_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", ) return swift_object_name
def upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, user_info): log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': user_info['access_key']}, **data_info) object_uploaded_status = s3lib.resource_op({ 'obj': bucket, 'resource': 'upload_file', 'args': [s3_object_path, s3_object_name], 'extra_info': upload_info }) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded')
def upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, user_info, append_data=False, append_msg=None, ): log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) s3_object_size = config.obj_size if append_data is True: data_info = manage_data.io_generator( s3_object_path, s3_object_size, op="append", **{"message": "\n%s" % append_msg}, ) else: data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict({"access_key": user_info["access_key"]}, **data_info) s3_obj = s3lib.resource_op( { "obj": bucket, "resource": "Object", "args": [s3_object_name], } ) object_uploaded_status = s3lib.resource_op( { "obj": s3_obj, "resource": "upload_file", "args": [s3_object_path], "extra_info": upload_info, } ) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info("object uploaded")
def fill_container( rgw, container_name, user_id, oc, cc, size, multipart=False, split_size=0, header=None, ): swift_object_name = utils.gen_s3_object_name( "%s.container.%s" % (user_id, cc), oc) log.info("object name: %s" % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info("object path: %s" % object_path) data_info = manage_data.io_generator(object_path, size) # upload object if multipart == True: mp_dir = os.path.join(TEST_DATA_PATH, swift_object_name + ".mp.parts") log.info(f"mp part dir: {mp_dir}") log.info("making multipart object part dir") mkdir = utils.exec_shell_cmd("sudo mkdir %s" % mp_dir) if mkdir is False: raise TestExecError("mkdir failed creating mp_dir_name") utils.split_file(object_path, split_size, mp_dir + "/") parts_list = sorted(glob.glob(mp_dir + "/" + "*")) log.info("parts_list: %s" % parts_list) log.info("no of parts: %s" % len(parts_list)) for each_part in parts_list: log.info("trying to upload part: %s" % each_part) with open(each_part, "r") as fp: etag = rgw.put_object( container_name, swift_object_name + "/" + each_part, contents=fp.read(), content_type="text/plain", headers=header, ) return swift_object_name else: if data_info is False: raise TestExecError("data creation failed") log.info("uploading object: %s" % object_path) with open(object_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", headers=header, ) return swift_object_name
def test_acls_public_read(u1_rgw_conn, u1, u2_rgw_conn, u1_bucket, u2_bucket): # test for public_read s3_ops = ResourceOps() u1_bucket_acl = s3_ops.resource_op(u1_rgw_conn, 'BucketAcl', u1_bucket.name) log.info('setting bucket acl: %s' % ACLS[1]) u1_bucket_acl.put(ACL=ACLS[1]) # access bucket_info of u1_bucket from u2 log.info('u1 bucket info') u1_bucket_info = s3_ops.resource_op(u1_rgw_conn, 'Bucket', u1_bucket.name) log.info(u1_bucket_info.name) log.info(u1_bucket_info.creation_date) log.info(u1_bucket_info.load()) s3_object_name = utils.gen_s3_object_name(u1_bucket.name, rand_no=0) log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': u1['access_key']}, **data_info) object_uploaded_status = s3_ops.resource_op(u1_bucket, 'upload_file', s3_object_path, s3_object_name, **upload_info) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') log.info( 'trying to access u1 bucket and its objects info from u2 after setting u1 bucket acls to public read' ) access_u1_bucket_from_u2 = s3_ops.resource_op(u2_rgw_conn, 'Bucket', u1_bucket.name) try: all_objects = access_u1_bucket_from_u2.objects.all() for obj in all_objects: log.info('obj name: %s' % obj.key) except Exception as e: msg = 'access given to read, but still failing to read' raise TestExecError(msg) log.info('tryring to delete u1_bucket from u2') try: u1_bucket_deleted_response = access_u1_bucket_from_u2.delete() response = HttpResponseParser(u1_bucket_deleted_response) log.info(response) except Exception as e: msg = 'access not given to write, hence fialing' log.info(msg) else: raise TestExecError("acces not given, but still bucket got deleted")
def upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, user_info): log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) s3_object_size = utils.get_file_size( config.objects_size_range["min"], config.objects_size_range["max"] ) data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict({"access_key": user_info["access_key"]}, **data_info) object_uploaded_status = s3lib.resource_op( { "obj": bucket, "resource": "upload_file", "args": [s3_object_path, s3_object_name], "extra_info": upload_info, } ) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info("object uploaded")
def write(self, io_type, fname, size=0): """ This function is to write IO on the mount point Parameters: io_type: basedir | subdir | file fname(char): file name size(int): size Returns: """ # io_type should be: basedir | subdir | file log.info("io_type: %s" % io_type) log.info("fname: %s" % fname) log.info("size: %s" % size) s3_conv = NFS_CONVENTIONS.get(io_type) ioinfo = IOInfo() path = os.path.abspath(self.mnt_point) full_path = os.path.join(path, fname) log.info("abs_path: %s" % full_path) try: if io_type == "basedir" or io_type == "subdir": log.info("creating dir, type: %s" % io_type) os.makedirs(full_path) io_info = { "name": os.path.basename(fname), "type": "dir", "s3_convention": s3_conv, "bucket": "self" if s3_conv == "bucket" else fname.split("/")[0], "md5": None, } log.info("io_info: %s" % io_info) ioinfo.add_io_info(self.rgw_user_info["access_key"], io_info) if io_type == "file": log.info("io_type is file: %s" % io_type) log.info("creating file with size: %s" % size) finfo = manage_date.io_generator(full_path, size) io_info = { "name": os.path.basename(fname), "type": "file", "s3_convention": s3_conv, "bucket": fname.split("/")[0], "md5": finfo["md5"], } log.info("io_info: %s" % io_info) ioinfo.add_io_info(self.rgw_user_info["access_key"], io_info) except (Exception) as e: log.error("Write IO Execution failed") log.error(e) return False
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() write_key_io_info = KeyIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) # create user all_users_info = s3lib.create_users(config.user_count) extra_user = s3lib.create_users(1)[0] extra_user_auth = Auth(extra_user, ssl=config.ssl) extra_user_conn = extra_user_auth.do_auth() for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Bucket', 'args': [bucket_name_to_create]}) # created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']}) created = s3lib.resource_op({'obj': bucket, 'resource': 'create', 'args': None, 'extra_info': {'access_key': each_user['access_key']}}) if created is False: raise TestExecError("Resource execution failed: bucket creation faield") if created is not None: response = HttpResponseParser(created) if response.status_code == 200: log.info('bucket created') else: raise TestExecError("bucket creation failed") else: raise TestExecError("bucket creation failed") # getting bucket version object if config.test_ops['enable_version'] is True: log.info('bucket versionig test on bucket: %s' % bucket.name) # bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name) bucket_versioning = s3lib.resource_op({'obj': rgw_conn, 'resource': 'BucketVersioning', 'args': [bucket.name]}) # checking the versioning status # version_status = s3_ops.resource_op(bucket_versioning, 'status') version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'status', 'args': None }) if version_status is None: log.info('bucket versioning still not enabled') # enabling bucket versioning # version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable') version_enable_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'enable', 'args': None, }) response = HttpResponseParser(version_enable_status) if response.status_code == 200: log.info('version enabled') write_bucket_io_info.add_versioning_status(each_user['access_key'],bucket.name, VERSIONING_STATUS['ENABLED']) else: raise TestExecError("version enable failed") if config.objects_count > 0: log.info('s3 objects to create: %s' % config.objects_count) for oc, s3_object_size in list(config.mapped_sizes.items()): # versioning upload s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc)) s3_object_names.append(s3_object_name) log.info('s3 object name: %s' % s3_object_name) log.info('versioning count: %s' % config.version_count) s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc)) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, s3_object_size) if original_data_info is False: TestExecError("data creation failed") created_versions_count = 0 for vc in range(config.version_count): log.info('version count for %s is %s' % (s3_object_name, str(vc))) log.info('modifying data: %s' % s3_object_name) modified_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op='append', **{'message': '\nhello for version: %s\n' % str(vc)}) if modified_data_info is False: TestExecError("data modification failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key'], 'versioning_status': VERSIONING_STATUS['ENABLED'], 'version_count_no': vc}, **modified_data_info) s3_obj = s3lib.resource_op({'obj': bucket, 'resource': 'Object', 'args': [s3_object_name], 'extra_info': upload_info, }) object_uploaded_status = s3lib.resource_op({'obj': s3_obj, 'resource': 'upload_file', 'args': [modified_data_info['name']], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current_version_id: %s' % s3_obj.version_id) key_version_info = basic_io_structure.version_info( **{'version_id': s3_obj.version_id, 'md5_local': upload_info['md5'], 'count_no': vc, 'size': upload_info['size']}) log.info('key_version_info: %s' % key_version_info) write_key_io_info.add_versioning_info(each_user['access_key'], bucket.name, s3_object_path, key_version_info) created_versions_count += 1 log.info('created_versions_count: %s' % created_versions_count) log.info('adding metadata') metadata1 = {"m_data1": "this is the meta1 for this obj"} s3_obj.metadata.update(metadata1) metadata2 = {"m_data2": "this is the meta2 for this obj"} s3_obj.metadata.update(metadata2) log.info('metadata for this object: %s' % s3_obj.metadata) log.info('metadata count for object: %s' % (len(s3_obj.metadata))) if not s3_obj.metadata: raise TestExecError('metadata not created even adding metadata') versions = bucket.object_versions.filter(Prefix=s3_object_name) created_versions_count_from_s3 = len([v.version_id for v in versions]) log.info('created versions count on s3: %s' % created_versions_count_from_s3) if created_versions_count is created_versions_count_from_s3: log.info('no new versions are created when added metdata') else: raise TestExecError("version count missmatch, " "possible creation of version on adding metadata") s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError("Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('downloaded_md5: %s' % s3_object_downloaded_md5) log.info('uploaded_md5: %s' % modified_data_info['md5']) # tail_op = utils.exec_shell_cmd('tail -l %s' % s3_object_download_path) log.info('all versions for the object: %s\n' % s3_object_name) versions = bucket.object_versions.filter(Prefix=s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) if config.test_ops.get('set_acl', None) is True: s3_obj_acl = s3lib.resource_op({'obj': rgw_conn, 'resource': 'ObjectAcl', 'args': [bucket.name, s3_object_name]}) # setting acl to private, just need to set to any acl and # check if its set - check by response code acls_set_status = s3_obj_acl.put(ACL='private') response = HttpResponseParser(acls_set_status) if response.status_code == 200: log.info('ACLs set') else: raise TestExecError("Acls not Set") # get obj details based on version id for version in versions: log.info('getting info for version id: %s' % version.version_id) obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('obj get detils :%s\n' % (obj.get(VersionId=version.version_id))) if config.test_ops['copy_to_version'] is True: # reverting object to one of the versions ( randomly chosen ) version_id_to_copy = random.choice([v.version_id for v in versions]) log.info('version_id_to_copy: %s' % version_id_to_copy) s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current version_id: %s' % s3_obj.version_id) copy_response = s3_obj.copy_from(CopySource={'Bucket': bucket.name, 'Key': s3_object_name, 'VersionId': version_id_to_copy}) log.info('copy_response: %s' % copy_response) if copy_response is None: raise TestExecError("copy object from version id failed") # current_version_id = copy_response['VersionID'] log.info('current_version_id: %s' % s3_obj.version_id) # delete the version_id_to_copy object s3_obj.delete(VersionId=version_id_to_copy) log.info('all versions for the object after the copy operation: %s\n' % s3_object_name) for version in versions: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) # log.info('downloading current s3object: %s' % s3_object_name) # s3_obj.download_file(s3_object_name + ".download") if config.test_ops['delete_object_versions'] is True: log.info('deleting s3_obj keys and its versions') s3_obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('deleting versions for s3 obj: %s' % s3_object_name) for version in versions: log.info('trying to delete obj version: %s' % version.version_id) del_obj_version = s3lib.resource_op({'obj': s3_obj, 'resource': 'delete', 'kwargs': dict(VersionId=version.version_id)}) log.info('response:\n%s' % del_obj_version) if del_obj_version is not None: response = HttpResponseParser(del_obj_version) if response.status_code == 204: log.info('version deleted ') write_key_io_info.delete_version_info(each_user['access_key'], bucket.name, s3_object_path, version.version_id) else: raise TestExecError("version deletion failed") else: raise TestExecError("version deletion failed") log.info('available versions for the object') versions = bucket.object_versions.filter(Prefix=s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' % ( version.object_key, version.version_id)) if config.test_ops.get('delete_from_extra_user') is True: log.info('trying to delete objects from extra user') s3_obj = s3lib.resource_op({'obj': extra_user_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('deleting versions for s3 obj: %s' % s3_object_name) for version in versions: log.info('trying to delete obj version: %s' % version.version_id) del_obj_version = s3lib.resource_op({'obj': s3_obj, 'resource': 'delete', 'kwargs': dict( VersionId=version.version_id)}) log.info('response:\n%s' % del_obj_version) if del_obj_version is not False: response = HttpResponseParser(del_obj_version) if response.status_code == 204: log.info('version deleted ') write_key_io_info.delete_version_info(each_user['access_key'], bucket.name, s3_object_path, version.version_id) raise TestExecError("version and deleted, this should not happen") else: log.info('version did not delete, expected behaviour') else: log.info('version did not delete, expected behaviour') if config.local_file_delete is True: log.info('deleting local file') utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_path) if config.test_ops['suspend_version'] is True: log.info('suspending versioning') # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') suspend_version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'suspend', 'args': None}) response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info('versioning suspended') write_bucket_io_info.add_versioning_status(each_user['access_key'], bucket.name, VERSIONING_STATUS['SUSPENDED']) else: raise TestExecError("version suspend failed") # getting all objects in the bucket log.info('getting all objects in the bucket') objects = s3lib.resource_op({'obj': bucket, 'resource': 'objects', 'args': None}) log.info('objects :%s' % objects) all_objects = s3lib.resource_op({'obj': objects, 'resource': 'all', 'args': None}) log.info('all objects: %s' % all_objects) log.info('all objects2 :%s ' % bucket.objects.all()) for obj in all_objects: log.info('object_name: %s' % obj.key) versions = bucket.object_versions.filter(Prefix=obj.key) log.info('displaying all versions of the object') for version in versions: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) if config.test_ops.get('suspend_from_extra_user') is True: log.info('suspending versioning from extra user') # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') bucket_versioning = s3lib.resource_op({'obj': extra_user_conn, 'resource': 'BucketVersioning', 'args': [bucket.name]}) suspend_version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'suspend', 'args': None}) if suspend_version_status is not False: response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info('versioning suspended') write_bucket_io_info.add_versioning_status(each_user['access_key'], bucket.name, VERSIONING_STATUS['SUSPENDED']) raise TestExecError('version suspended, this should not happen') else: log.info('versioning not suspended, expected behaviour') if config.test_ops.get('upload_after_suspend') is True: log.info('trying to upload after suspending versioning on bucket') for oc, s3_object_size in list(config.mapped_sizes.items()): # non versioning upload s3_object_name = s3_object_names[oc] + ".after_version_suspending" log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) non_version_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op="append", **{ 'message': '\nhello for non version\n'}) if non_version_data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key'], 'versioning_status': 'suspended'},**non_version_data_info) s3_obj = s3lib.resource_op({'obj': bucket, 'resource': 'Object', 'args': [s3_object_name], 'extra_info': upload_info}) object_uploaded_status = s3lib.resource_op({'obj': s3_obj, 'resource': 'upload_file', 'args': [non_version_data_info['name']], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('version_id: %s' % s3_obj.version_id) if s3_obj.version_id is None: log.info('Versions are not created after suspending') else: raise TestExecError('Versions are created even after suspending') s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError("Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5) log.info('s3_object_uploaded_md5: %s' % non_version_data_info['md5']) if config.local_file_delete is True: utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_path)
def test_exec(config): test_info = AddTestInfo('create m buckets with n objects') try: test_info.started_info() # get user with open('user_details') as fout: all_users_info = simplejson.load(fout) for each_user in all_users_info: # authenticate auth = Auth(each_user) rgw_conn = auth.do_auth_using_client() rgw = auth.do_auth() bucket_list = [] buckets = rgw_conn.list_buckets() log.info('buckets are %s' % buckets) for each_bucket in buckets['Buckets']: bucket_list.append(each_bucket['Name']) for bucket_name in bucket_list: # create 'bucket' resource object bucket = rgw.Bucket(bucket_name) log.info('In bucket: %s' % bucket_name) if config.test_ops['create_object'] is True: # uploading data log.info('s3 objects to create: %s' % config.objects_count) for oc in range(config.objects_count): s3_object_name = utils.gen_s3_object_name(bucket_name, oc) log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key']}, **data_info) # object_uploaded_status = bucket.upload_file(s3_object_path, s3_object_name) object_uploaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'upload_file', 'args': [s3_object_path, s3_object_name], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') if config.test_ops['download_object'] is True: log.info('trying to download object: %s' % s3_object_name) s3_object_download_name = s3_object_name + "." + "download" s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_download_name) log.info('s3_object_download_path: %s' % s3_object_download_path) log.info('downloading to filename: %s' % s3_object_download_name) # object_downloaded_status = bucket.download_file(s3_object_path, s3_object_name) object_downloaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError("Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') if config.test_ops['delete_bucket_object'] is True: log.info('listing all objects in bucket: %s' % bucket.name) # objects = s3_ops.resource_op(bucket, 'objects', None) objects = s3lib.resource_op({'obj': bucket, 'resource': 'objects', 'args': None}) log.info('objects :%s' % objects) # all_objects = s3_ops.resource_op(objects, 'all') all_objects = s3lib.resource_op({'obj': objects, 'resource': 'all', 'args': None}) log.info('all objects: %s' % all_objects) for obj in all_objects: log.info('object_name: %s' % obj.key) log.info('deleting all objects in bucket') # objects_deleted = s3_ops.resource_op(objects, 'delete') objects_deleted = s3lib.resource_op({'obj': objects, 'resource': 'delete', 'args': None}) log.info('objects_deleted: %s' % objects_deleted) if objects_deleted is False: raise TestExecError('Resource execution failed: Object deletion failed') if objects_deleted is not None: response = HttpResponseParser(objects_deleted[0]) if response.status_code == 200: log.info('objects deleted ') else: raise TestExecError("objects deletion failed") else: raise TestExecError("objects deletion failed") # wait for object delete info to sync time.sleep(60) log.info('deleting bucket: %s' % bucket.name) # bucket_deleted_status = s3_ops.resource_op(bucket, 'delete') bucket_deleted_status = s3lib.resource_op({'obj': bucket, 'resource': 'delete', 'args': None}) log.info('bucket_deleted_status: %s' % bucket_deleted_status) if bucket_deleted_status is not None: response = HttpResponseParser(bucket_deleted_status) if response.status_code == 204: log.info('bucket deleted ') else: raise TestExecError("bucket deletion failed") else: raise TestExecError("bucket deletion failed") test_info.success_status('test passed') sys.exit(0) except Exception as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1) except TestExecError as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): test_info = AddTestInfo("test swift user key gen") io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # preparing data user_names = ["tuffy", "scooby", "max"] tenant1 = "tenant" cmd = ( 'radosgw-admin user create --uid=%s --display-name="%s" --tenant=%s --cluster %s' % (user_names[0], user_names[0], tenant1, config.cluster_name) ) out = utils.exec_shell_cmd(cmd) if out is False: raise TestExecError("RGW User creation error") log.info("output :%s" % out) v1_as_json = json.loads(out) log.info("creted user_id: %s" % v1_as_json["user_id"]) cmd2 = ( "radosgw-admin subuser create --uid=%s$%s --subuser=%s:swift --tenant=%s --access=full --cluster %s" % (tenant1, user_names[0], user_names[0], tenant1, config.cluster_name) ) out2 = utils.exec_shell_cmd(cmd2) if out2 is False: raise TestExecError("sub-user creation error") v2_as_json = json.loads(out2) log.info("created subuser: %s" % v2_as_json["subusers"][0]["id"]) cmd3 = ( "radosgw-admin key create --subuser=%s:swift --uid=%s$%s --tenant=%s --key-type=swift --gen-secret " "--cluster %s" % (user_names[0], user_names[0], tenant1, tenant1, config.cluster_name) ) out3 = utils.exec_shell_cmd(cmd3) if out3 is False: raise TestExecError("secret_key gen error") v3_as_json = json.loads(out3) log.info( "created subuser: %s\nsecret_key generated: %s" % ( v3_as_json["swift_keys"][0]["user"], v3_as_json["swift_keys"][0]["secret_key"], ) ) user_info = { "user_id": v3_as_json["swift_keys"][0]["user"], "key": v3_as_json["swift_keys"][0]["secret_key"], } auth = Auth(user_info) rgw = auth.do_auth() for cc in range(config.container_count): container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc ) container = swiftlib.resource_op( {"obj": rgw, "resource": "put_container", "args": [container_name]} ) if container is False: raise TestExecError( "Resource execution failed: container creation faield" ) for oc in range(config.objects_count): swift_object_name = utils.gen_s3_object_name( "%s.container.%s" % (user_names[0], cc), oc ) log.info("object name: %s" % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info("object path: %s" % object_path) object_size = utils.get_file_size( config.objects_size_range["min"], config.objects_size_range["max"] ) data_info = manage_data.io_generator(object_path, object_size) if data_info is False: TestExecError("data creation failed") log.info("uploading object: %s" % object_path) with open(object_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", ) test_info.success_status("test passed") sys.exit(0) except Exception as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status("test failed") sys.exit(1) except TestExecError as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status("test failed") sys.exit(1)
def test_exec(config): test_info = AddTestInfo('test versioning with objects') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # create user all_users_info = s3lib.create_users(config.user_count, config.cluster_name) for each_user in all_users_info: # authenticate auth = Auth(each_user) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Bucket', 'args': [bucket_name_to_create]}) # created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']}) created = s3lib.resource_op({'obj': bucket, 'resource': 'create', 'args': None, 'extra_info': {'access_key': each_user['access_key']}}) if created is False: raise TestExecError("Resource execution failed: bucket creation faield") if created is not None: response = HttpResponseParser(created) if response.status_code == 200: log.info('bucket created') else: raise TestExecError("bucket creation failed") else: raise TestExecError("bucket creation failed") # getting bucket version object if config.test_ops['enable_version'] is True: log.info('bucket versionig test on bucket: %s' % bucket.name) # bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name) bucket_versioning = s3lib.resource_op({'obj': rgw_conn, 'resource': 'BucketVersioning', 'args': [bucket.name]}) # checking the versioning status # version_status = s3_ops.resource_op(bucket_versioning, 'status') version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'status', 'args': None }) if version_status is None: log.info('bucket versioning still not enabled') # enabling bucket versioning # version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable') version_enable_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'enable', 'args': None}) response = HttpResponseParser(version_enable_status) if response.status_code == 200: log.info('version enabled') else: raise TestExecError("version enable failed") if config.objects_count > 0: log.info('s3 objects to create: %s' % config.objects_count) for oc in range(config.objects_count): # versioning upload s3_object_name = utils.gen_s3_object_name(bucket_name_to_create,str(oc)) s3_object_names.append(s3_object_name) log.info('s3 object name: %s' % s3_object_name) log.info('versioning count: %s' % config.version_count) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc)) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, s3_object_size) if original_data_info is False: TestExecError("data creation failed") for vc in range(config.version_count): log.info('version count for %s is %s' % (s3_object_name, str(vc))) log.info('modifying data: %s' % s3_object_name) modified_data_info = manage_data.io_generator(s3_object_path, s3_object_size, data='append', **{'message': '\nhello object for version: %s\n' % str(vc)}) if modified_data_info is False: TestExecError("data modification failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key']}, **modified_data_info) object_uploaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'upload_file', 'args': [modified_data_info['name'], s3_object_name], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') log.info('all versions for the object: %s\n' % s3_object_name) versions = bucket.object_versions.filter(Prefix=s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' %(version.object_key, version.version_id)) if config.test_ops['copy_to_version'] is True: # reverting object to one of the versions ( randomly chosen ) version_id_to_copy = random.choice([v.version_id for v in versions]) log.info('version_id_to_copy: %s' % version_id_to_copy) s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current version_id: %s' % s3_obj.version_id) copy_response = s3_obj.copy_from(CopySource={'Bucket': bucket.name, 'Key': s3_object_name, 'VersionId': version_id_to_copy}) log.info('copy_response: %s' % copy_response) if copy_response is None: raise TestExecError("copy object from version id failed") # current_version_id = copy_response['VersionID'] log.info('current_version_id: %s' % s3_obj.version_id ) # delete the version_id_to_copy object s3_obj.delete(VersionId=version_id_to_copy) log.info('all versions for the object after the copy operation: %s\n' % s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) # log.info('downloading current s3object: %s' % s3_object_name) # s3_obj.download_file(s3_object_name + ".download") if config.test_ops['delete_object_versions'] is True: log.info('deleting s3_obj keys and its versions') s3_obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('deleting versions for s3 obj: %s' % s3_object_name) for version in versions: log.info('trying to delete obj version: %s' % version.version_id) del_obj_version = s3lib.resource_op({'obj': s3_obj, 'resource': 'delete', 'kwargs': dict(VersionId=version.version_id)}) log.info('response:\n%s' % del_obj_version) if del_obj_version is not None: response = HttpResponseParser(del_obj_version) if response.status_code == 204: log.info('version deleted ') else: raise TestExecError("version deletion failed") else: raise TestExecError("version deletion failed") if config.test_ops['suspend_version'] is True: # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') suspend_version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'suspend', 'args': None}) response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info('versioning suspended') else: raise TestExecError("version suspend failed") if config.test_ops['upload_after_suspend'] is True: log.info('trying to upload after suspending versioning on bucket') for s3_object_name in s3_object_names: # non versioning upload log.info('s3 object name: %s' % s3_object_name) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) non_version_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op="append", **{'message': '\nhello object for non version\n'}) if non_version_data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key']}, **non_version_data_info) object_uploaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'upload_file', 'args': [non_version_data_info['name'], s3_object_name], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name+".download") object_downloaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError("Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5) log.info('s3_object_uploaded_md5: %s' % non_version_data_info['md5']) test_info.success_status('test passed') sys.exit(0) except Exception,e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) rgw_service = RGWService() # create pool pool_name = '.rgw.buckets.special' pg_num = '8' pgp_num = '8' pool_create = 'sudo ceph osd pool create "%s" %s %s replicated' % ( pool_name, pg_num, pgp_num) pool_create_exec = utils.exec_shell_cmd(pool_create) if pool_create_exec is False: raise TestExecError("Pool creation failed") # create realm realm_name = 'buz-tickets' log.info('creating realm name') realm_create = 'sudo radosgw-admin realm create --rgw-realm=%s' % realm_name realm_create_exec = utils.exec_shell_cmd(realm_create) if realm_create_exec is False: raise TestExecError("cmd execution failed") # sample output of create realm """ { "id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62", "name": "buz-tickets", "current_period": "1950b710-3e63-4c41-a19e-46a715000980", "epoch": 1 } """ log.info('modify zonegroup ') modify = 'sudo radosgw-admin zonegroup modify --rgw-zonegroup=default --rgw-realm=%s --master' % realm_name modify_exec = utils.exec_shell_cmd(modify) if modify_exec is False: raise TestExecError("cmd execution failed") # get the zonegroup zonegroup_file = 'zonegroup.json' get_zonegroup = 'sudo radosgw-admin zonegroup --rgw-zonegroup=default get > %s' % zonegroup_file get_zonegroup_exec = utils.exec_shell_cmd(get_zonegroup) if get_zonegroup_exec is False: raise TestExecError("cmd execution failed") add_to_placement_targets = {"name": "special-placement", "tags": []} fp = open(zonegroup_file, 'r') zonegroup_txt = fp.read() fp.close() log.info('got zonegroup info: \n%s' % zonegroup_txt) zonegroup = json.loads(zonegroup_txt) log.info('adding placement targets') zonegroup['placement_targets'].append(add_to_placement_targets) with open(zonegroup_file, 'w') as fp: json.dump(zonegroup, fp) zonegroup_set = 'sudo radosgw-admin zonegroup set < %s' % zonegroup_file zonegroup_set_exec = utils.exec_shell_cmd(zonegroup_set) if zonegroup_set_exec is False: raise TestExecError("cmd execution failed") log.info('zone group update completed') log.info('getting zone file') # get zone log.info('getting zone info') zone_file = 'zone.json' get_zone = 'sudo radosgw-admin zone --rgw-zone=default get > zone.json' get_zone_exec = utils.exec_shell_cmd(get_zone) if get_zone_exec is False: raise TestExecError("cmd execution failed") fp = open(zone_file, 'r') zone_info = fp.read() fp.close() log.info('zone_info :\n%s' % zone_info) zone_info_cleaned = json.loads(zone_info) special_placement_info = { "key": "special-placement", "val": { "index_pool": ".rgw.buckets.index", "data_pool": ".rgw.buckets.special", "data_extra_pool": ".rgw.buckets.extra" } } log.info('adding special placement info') zone_info_cleaned['placement_pools'].append(special_placement_info) with open(zone_file, 'w+') as fp: json.dump(zone_info_cleaned, fp) zone_file_set = 'sudo radosgw-admin zone set < %s' % zone_file zone_file_set_exec = utils.exec_shell_cmd(zone_file_set) if zone_file_set_exec is False: raise TestExecError("cmd execution failed") log.info('zone info updated ') zone_group_update_set = 'radosgw-admin period update --commit' zone_group_update_set_exec = utils.exec_shell_cmd(zone_group_update_set) log.info(zone_group_update_set_exec) restarted = rgw_service.restart() if restarted is False: raise TestExecError("service restart failed") if config.rgw_client == 'rgw': log.info('client type is rgw') rgw_user_info = s3_swift_lib.create_users(1) auth = Auth(rgw_user_info) rgw_conn = auth.do_auth() # create bucket bucket_name = utils.gen_bucket_name_from_userid( rgw_user_info['user_id'], 0) bucket = resuables.create_bucket(bucket_name, rgw_conn, rgw_user_info) # create object s3_object_name = utils.gen_s3_object_name(bucket_name, 0) resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, rgw_user_info) if config.rgw_client == 'swift': log.info('client type is swift') user_names = ['tuffy', 'scooby', 'max'] tenant = 'tenant' umgmt = UserMgmt() umgmt.create_tenant_user(tenant_name=tenant, user_id=user_names[0], displayname=user_names[0]) user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0]) auth = Auth(user_info) rgw = auth.do_auth() container_name = utils.gen_bucket_name_from_userid( user_info['user_id'], rand_no=0) container = s3_swift_lib.resource_op({ 'obj': rgw, 'resource': 'put_container', 'args': [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation faield") swift_object_name = utils.gen_s3_object_name( '%s.container.%s' % (user_names[0], 0), 0) log.info('object name: %s' % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info('object path: %s' % object_path) object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) data_info = manage_data.io_generator(object_path, object_size) # upload object if data_info is False: TestExecError("data creation failed") log.info('uploading object: %s' % object_path) with open(object_path, 'r') as fp: rgw.put_object(container_name, swift_object_name, contents=fp.read(), content_type='text/plain')
def upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, user_info, append_data=False, append_msg=None, ): log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) s3_object_size = config.obj_size split_size = config.split_size if hasattr(config, "split_size") else 5 log.info("split size: %s" % split_size) if append_data is True: data_info = manage_data.io_generator( s3_object_path, s3_object_size, op="append", **{"message": "\n%s" % append_msg}, ) else: data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") mp_dir = os.path.join(TEST_DATA_PATH, s3_object_name + ".mp.parts") log.info("mp part dir: %s" % mp_dir) log.info("making multipart object part dir") mkdir = utils.exec_shell_cmd("sudo mkdir %s" % mp_dir) if mkdir is False: raise TestExecError("mkdir failed creating mp_dir_name") utils.split_file(s3_object_path, split_size, mp_dir + "/") parts_list = sorted(glob.glob(mp_dir + "/" + "*")) log.info("parts_list: %s" % parts_list) log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict( { "access_key": user_info["access_key"], "upload_type": "multipart" }, **data_info) s3_obj = s3lib.resource_op({ "obj": bucket, "resource": "Object", "args": [s3_object_name], }) log.info("initiating multipart upload") mpu = s3lib.resource_op({ "obj": s3_obj, "resource": "initiate_multipart_upload", "args": None, "extra_info": upload_info, }) part_number = 1 parts_info = {"Parts": []} log.info("no of parts: %s" % len(parts_list)) for each_part in parts_list: log.info("trying to upload part: %s" % each_part) part = mpu.Part(part_number) # part_upload_response = part.upload(Body=open(each_part)) part_upload_response = s3lib.resource_op({ "obj": part, "resource": "upload", "kwargs": dict(Body=open(each_part, mode="rb")), }) if part_upload_response is not False: response = HttpResponseParser(part_upload_response) if response.status_code == 200: log.info("part uploaded") if config.local_file_delete is True: log.info("deleting local file part") utils.exec_shell_cmd("sudo rm -rf %s" % each_part) else: raise TestExecError("part uploading failed") part_info = { "PartNumber": part_number, "ETag": part_upload_response["ETag"] } parts_info["Parts"].append(part_info) if each_part != parts_list[-1]: # increase the part number only if the current part is not the last part part_number += 1 log.info("curr part_number: %s" % part_number) # log.info('parts_info so far: %s'% parts_info) if len(parts_list) == part_number: log.info("all parts upload completed") mpu.complete(MultipartUpload=parts_info) log.info("multipart upload complete for key: %s" % s3_object_name)
def upload_version_object(config, user_info, rgw_conn, s3_object_name, object_size, bucket, TEST_DATA_PATH): # versioning upload log.info("versioning count: %s" % config.version_count) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, object_size) if original_data_info is False: TestExecError("data creation failed") created_versions_count = 0 for vc in range(config.version_count): log.info("version count for %s is %s" % (s3_object_name, str(vc))) log.info("modifying data: %s" % s3_object_name) modified_data_info = manage_data.io_generator( s3_object_path, object_size, op="append", **{"message": "\nhello for version: %s\n" % str(vc)}, ) if modified_data_info is False: TestExecError("data modification failed") log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict( { "access_key": user_info["access_key"], "versioning_status": "enabled", "version_count_no": vc, }, **modified_data_info, ) s3_obj = s3lib.resource_op({ "obj": bucket, "resource": "Object", "args": [s3_object_name], "extra_info": upload_info, }) object_uploaded_status = s3lib.resource_op({ "obj": s3_obj, "resource": "upload_file", "args": [modified_data_info["name"]], "extra_info": upload_info, }) if object_uploaded_status is False: raise TestExecError( "Resource execution failed: object upload failed") if object_uploaded_status is None: log.info("object uploaded") s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info("current_version_id: %s" % s3_obj.version_id) key_version_info = basic_io_structure.version_info( **{ "version_id": s3_obj.version_id, "md5_local": upload_info["md5"], "count_no": vc, "size": upload_info["size"], }) log.info("key_version_info: %s" % key_version_info) write_key_io_info.add_versioning_info(user_info["access_key"], bucket.name, s3_object_path, key_version_info) created_versions_count += 1 log.info("created_versions_count: %s" % created_versions_count) log.info("adding metadata") metadata1 = {"m_data1": "this is the meta1 for this obj"} s3_obj.metadata.update(metadata1) metadata2 = {"m_data2": "this is the meta2 for this obj"} s3_obj.metadata.update(metadata2) log.info("metadata for this object: %s" % s3_obj.metadata) log.info("metadata count for object: %s" % (len(s3_obj.metadata))) if not s3_obj.metadata: raise TestExecError( "metadata not created even adding metadata") versions = bucket.object_versions.filter(Prefix=s3_object_name) created_versions_count_from_s3 = len( [v.version_id for v in versions]) log.info("created versions count on s3: %s" % created_versions_count_from_s3) if created_versions_count is created_versions_count_from_s3: log.info("no new versions are created when added metadata") else: raise TestExecError( "version count mismatch, " "possible creation of version on adding metadata") s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed") if object_downloaded_status is None: log.info("object downloaded") # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info("downloaded_md5: %s" % s3_object_downloaded_md5) log.info("uploaded_md5: %s" % modified_data_info["md5"]) log.info("deleting downloaded version file") utils.exec_shell_cmd("sudo rm -rf %s" % s3_object_download_path) log.info("all versions for the object: %s\n" % s3_object_name)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() write_key_io_info = KeyIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) # create user all_users_info = s3lib.create_users(config.user_count) extra_user = s3lib.create_users(1)[0] extra_user_auth = Auth(extra_user, ssl=config.ssl) extra_user_conn = extra_user_auth.do_auth() for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc) log.info("creating bucket with name: %s" % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = s3lib.resource_op({ "obj": rgw_conn, "resource": "Bucket", "args": [bucket_name_to_create] }) # created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']}) created = s3lib.resource_op({ "obj": bucket, "resource": "create", "args": None, "extra_info": { "access_key": each_user["access_key"] }, }) if created is False: raise TestExecError( "Resource execution failed: bucket creation faield") if created is not None: response = HttpResponseParser(created) if response.status_code == 200: log.info("bucket created") else: raise TestExecError("bucket creation failed") else: raise TestExecError("bucket creation failed") # getting bucket version object if config.test_ops["enable_version"] is True: log.info("bucket versionig test on bucket: %s" % bucket.name) # bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name) bucket_versioning = s3lib.resource_op({ "obj": rgw_conn, "resource": "BucketVersioning", "args": [bucket.name], }) # checking the versioning status # version_status = s3_ops.resource_op(bucket_versioning, 'status') version_status = s3lib.resource_op({ "obj": bucket_versioning, "resource": "status", "args": None }) if version_status is None: log.info("bucket versioning still not enabled") # enabling bucket versioning # version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable') version_enable_status = s3lib.resource_op({ "obj": bucket_versioning, "resource": "enable", "args": None, }) response = HttpResponseParser(version_enable_status) if response.status_code == 200: log.info("version enabled") write_bucket_io_info.add_versioning_status( each_user["access_key"], bucket.name, VERSIONING_STATUS["ENABLED"], ) else: raise TestExecError("version enable failed") if config.objects_count > 0: log.info("s3 objects to create: %s" % config.objects_count) for oc, s3_object_size in list( config.mapped_sizes.items()): # versioning upload s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, str(oc)) s3_object_names.append(s3_object_name) log.info("s3 object name: %s" % s3_object_name) log.info("versioning count: %s" % config.version_count) s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, str(oc)) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator( s3_object_path, s3_object_size) if original_data_info is False: TestExecError("data creation failed") created_versions_count = 0 for vc in range(config.version_count): log.info("version count for %s is %s" % (s3_object_name, str(vc))) log.info("modifying data: %s" % s3_object_name) modified_data_info = manage_data.io_generator( s3_object_path, s3_object_size, op="append", **{ "message": "\nhello for version: %s\n" % str(vc) }) if modified_data_info is False: TestExecError("data modification failed") log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict( { "access_key": each_user["access_key"], "versioning_status": VERSIONING_STATUS["ENABLED"], "version_count_no": vc, }, **modified_data_info) s3_obj = s3lib.resource_op({ "obj": bucket, "resource": "Object", "args": [s3_object_name], "extra_info": upload_info, }) object_uploaded_status = s3lib.resource_op({ "obj": s3_obj, "resource": "upload_file", "args": [modified_data_info["name"]], "extra_info": upload_info, }) if object_uploaded_status is False: raise TestExecError( "Resource execution failed: object upload failed" ) if object_uploaded_status is None: log.info("object uploaded") s3_obj = rgw_conn.Object( bucket.name, s3_object_name) log.info("current_version_id: %s" % s3_obj.version_id) key_version_info = basic_io_structure.version_info( **{ "version_id": s3_obj.version_id, "md5_local": upload_info["md5"], "count_no": vc, "size": upload_info["size"], }) log.info("key_version_info: %s" % key_version_info) write_key_io_info.add_versioning_info( each_user["access_key"], bucket.name, s3_object_path, key_version_info, ) created_versions_count += 1 log.info("created_versions_count: %s" % created_versions_count) log.info("adding metadata") metadata1 = { "m_data1": "this is the meta1 for this obj" } s3_obj.metadata.update(metadata1) metadata2 = { "m_data2": "this is the meta2 for this obj" } s3_obj.metadata.update(metadata2) log.info("metadata for this object: %s" % s3_obj.metadata) log.info("metadata count for object: %s" % (len(s3_obj.metadata))) if not s3_obj.metadata: raise TestExecError( "metadata not created even adding metadata" ) versions = bucket.object_versions.filter( Prefix=s3_object_name) created_versions_count_from_s3 = len( [v.version_id for v in versions]) log.info("created versions count on s3: %s" % created_versions_count_from_s3) if (created_versions_count is created_versions_count_from_s3): log.info( "no new versions are created when added metdata" ) else: raise TestExecError( "version count missmatch, " "possible creation of version on adding metadata" ) s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info("object downloaded") # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path) log.info("downloaded_md5: %s" % s3_object_downloaded_md5) log.info("uploaded_md5: %s" % modified_data_info["md5"]) # tail_op = utils.exec_shell_cmd('tail -l %s' % s3_object_download_path) log.info("all versions for the object: %s\n" % s3_object_name) versions = bucket.object_versions.filter( Prefix=s3_object_name) for version in versions: log.info("key_name: %s --> version_id: %s" % (version.object_key, version.version_id)) if config.test_ops.get("set_acl", None) is True: s3_obj_acl = s3lib.resource_op({ "obj": rgw_conn, "resource": "ObjectAcl", "args": [bucket.name, s3_object_name], }) # setting acl to private, just need to set to any acl and # check if its set - check by response code acls_set_status = s3_obj_acl.put(ACL="private") response = HttpResponseParser(acls_set_status) if response.status_code == 200: log.info("ACLs set") else: raise TestExecError("Acls not Set") # get obj details based on version id for version in versions: log.info("getting info for version id: %s" % version.version_id) obj = s3lib.resource_op({ "obj": rgw_conn, "resource": "Object", "args": [bucket.name, s3_object_name], }) log.info( "obj get detils :%s\n" % (obj.get(VersionId=version.version_id))) if config.test_ops["copy_to_version"] is True: # reverting object to one of the versions ( randomly chosen ) version_id_to_copy = random.choice( [v.version_id for v in versions]) log.info("version_id_to_copy: %s" % version_id_to_copy) s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info("current version_id: %s" % s3_obj.version_id) copy_response = s3_obj.copy_from( CopySource={ "Bucket": bucket.name, "Key": s3_object_name, "VersionId": version_id_to_copy, }) log.info("copy_response: %s" % copy_response) if copy_response is None: raise TestExecError( "copy object from version id failed") # current_version_id = copy_response['VersionID'] log.info("current_version_id: %s" % s3_obj.version_id) # delete the version_id_to_copy object s3_obj.delete(VersionId=version_id_to_copy) log.info( "all versions for the object after the copy operation: %s\n" % s3_object_name) for version in versions: log.info( "key_name: %s --> version_id: %s" % (version.object_key, version.version_id)) # log.info('downloading current s3object: %s' % s3_object_name) # s3_obj.download_file(s3_object_name + ".download") if config.test_ops["delete_object_versions"] is True: log.info("deleting s3_obj keys and its versions") s3_obj = s3lib.resource_op({ "obj": rgw_conn, "resource": "Object", "args": [bucket.name, s3_object_name], }) log.info("deleting versions for s3 obj: %s" % s3_object_name) for version in versions: log.info("trying to delete obj version: %s" % version.version_id) del_obj_version = s3lib.resource_op({ "obj": s3_obj, "resource": "delete", "kwargs": dict(VersionId=version.version_id), }) log.info("response:\n%s" % del_obj_version) if del_obj_version is not None: response = HttpResponseParser( del_obj_version) if response.status_code == 204: log.info("version deleted ") reusable.delete_version_object( bucket, version.version_id, s3_object_path, rgw_conn, each_user, ) else: raise TestExecError( "version deletion failed") else: raise TestExecError( "version deletion failed") log.info("available versions for the object") versions = bucket.object_versions.filter( Prefix=s3_object_name) for version in versions: log.info( "key_name: %s --> version_id: %s" % (version.object_key, version.version_id)) if config.test_ops.get( "delete_from_extra_user") is True: log.info( "trying to delete objects from extra user") s3_obj = s3lib.resource_op({ "obj": extra_user_conn, "resource": "Object", "args": [bucket.name, s3_object_name], }) log.info("deleting versions for s3 obj: %s" % s3_object_name) for version in versions: log.info("trying to delete obj version: %s" % version.version_id) del_obj_version = s3lib.resource_op({ "obj": s3_obj, "resource": "delete", "kwargs": dict(VersionId=version.version_id), }) log.info("response:\n%s" % del_obj_version) if del_obj_version is not False: response = HttpResponseParser( del_obj_version) if response.status_code == 204: log.info("version deleted ") write_key_io_info.delete_version_info( each_user["access_key"], bucket.name, s3_object_path, version.version_id, ) raise TestExecError( "version and deleted, this should not happen" ) else: log.info( "version did not delete, expected behaviour" ) else: log.info( "version did not delete, expected behaviour" ) if config.local_file_delete is True: log.info("deleting local file") utils.exec_shell_cmd("sudo rm -rf %s" % s3_object_path) if config.test_ops["suspend_version"] is True: log.info("suspending versioning") # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') suspend_version_status = s3lib.resource_op({ "obj": bucket_versioning, "resource": "suspend", "args": None }) response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info("versioning suspended") write_bucket_io_info.add_versioning_status( each_user["access_key"], bucket.name, VERSIONING_STATUS["SUSPENDED"], ) else: raise TestExecError("version suspend failed") # getting all objects in the bucket log.info("getting all objects in the bucket") objects = s3lib.resource_op({ "obj": bucket, "resource": "objects", "args": None }) log.info("objects :%s" % objects) all_objects = s3lib.resource_op({ "obj": objects, "resource": "all", "args": None }) log.info("all objects: %s" % all_objects) log.info("all objects2 :%s " % bucket.objects.all()) for obj in all_objects: log.info("object_name: %s" % obj.key) versions = bucket.object_versions.filter( Prefix=obj.key) log.info("displaying all versions of the object") for version in versions: log.info("key_name: %s --> version_id: %s" % (version.object_key, version.version_id)) if config.test_ops.get("suspend_from_extra_user") is True: log.info("suspending versioning from extra user") # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') bucket_versioning = s3lib.resource_op({ "obj": extra_user_conn, "resource": "BucketVersioning", "args": [bucket.name], }) suspend_version_status = s3lib.resource_op({ "obj": bucket_versioning, "resource": "suspend", "args": None }) if suspend_version_status is not False: response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info("versioning suspended") write_bucket_io_info.add_versioning_status( each_user["access_key"], bucket.name, VERSIONING_STATUS["SUSPENDED"], ) raise TestExecError( "version suspended, this should not happen") else: log.info( "versioning not suspended, expected behaviour") if config.test_ops.get("upload_after_suspend") is True: log.info( "trying to upload after suspending versioning on bucket") for oc, s3_object_size in list(config.mapped_sizes.items()): # non versioning upload s3_object_name = s3_object_names[ oc] + ".after_version_suspending" log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) non_version_data_info = manage_data.io_generator( s3_object_path, s3_object_size, op="append", **{"message": "\nhello for non version\n"}) if non_version_data_info is False: TestExecError("data creation failed") log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict( { "access_key": each_user["access_key"], "versioning_status": "suspended", }, **non_version_data_info) s3_obj = s3lib.resource_op({ "obj": bucket, "resource": "Object", "args": [s3_object_name], "extra_info": upload_info, }) object_uploaded_status = s3lib.resource_op({ "obj": s3_obj, "resource": "upload_file", "args": [non_version_data_info["name"]], "extra_info": upload_info, }) if object_uploaded_status is False: raise TestExecError( "Resource execution failed: object upload failed") if object_uploaded_status is None: log.info("object uploaded") s3_obj = s3lib.resource_op({ "obj": rgw_conn, "resource": "Object", "args": [bucket.name, s3_object_name], }) log.info("version_id: %s" % s3_obj.version_id) if s3_obj.version_id is None: log.info("Versions are not created after suspending") else: raise TestExecError( "Versions are created even after suspending") s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info("object downloaded") # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path) log.info("s3_object_downloaded_md5: %s" % s3_object_downloaded_md5) log.info("s3_object_uploaded_md5: %s" % non_version_data_info["md5"]) if config.local_file_delete is True: utils.exec_shell_cmd("sudo rm -rf %s" % s3_object_path) if config.test_ops.get("delete_bucket") is True: reusable.delete_bucket(bucket) # check sync status if a multisite cluster reusable.check_sync_status() # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user all_users_info = s3lib.create_users(config.user_count) for each_user in all_users_info: # authentication auth = Auth(each_user, ssl=config.ssl) s3_conn_client = auth.do_auth_using_client() # create buckets with object lock configuration if config.test_ops["create_bucket"] is True: log.info(f"no of buckets to create: {config.bucket_count}") for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc ) log.info(f"creating bucket with name: {bucket_name_to_create}") rgw_ip_and_port = get_rgw_ip_and_port() s3_conn_client.create_bucket( Bucket=bucket_name_to_create, ObjectLockEnabledForBucket=True ) # put object lock configuration for bucket s3_conn_client.put_object_lock_configuration( Bucket=bucket_name_to_create, ObjectLockConfiguration={ "ObjectLockEnabled": "Enabled", "Rule": {"DefaultRetention": {"Mode": "COMPLIANCE", "Days": 1}}, }, ) if config.test_ops["create_object"] is True: # uploading data log.info(f"s3 objects to create: {config.objects_count}") for oc, size in list(config.mapped_sizes.items()): s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, 0 ) log.info(f"s3 object name: {s3_object_name}") s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info(f"s3 object path: {s3_object_path}") log.info("upload type: normal") io_generator(TEST_DATA_PATH + "/" + s3_object_name, size) s3_conn_client.put_object( Body=TEST_DATA_PATH + "/" + s3_object_name, Bucket=bucket_name_to_create, Key=s3_object_name, ) log.info("Verify version count") # Verify version count versions = s3_conn_client.list_object_versions( Bucket=bucket_name_to_create ) versions_count = len(versions["Versions"]) error_message = ( f"Expected: {config.objects_count}, Actual: {versions_count}" ) if versions_count == config.objects_count: log.info("Expected and actual version count is same") else: raise ObjectVersionCountMismatch(error_message) # Verify delete disabled for object log.info("Verify delete disabled for object") for version_dict in versions["Versions"]: try: s3_conn_client.delete_object( Bucket=bucket_name_to_create, Key=s3_object_name, VersionId=version_dict["VersionId"], ) raise AccessDeniedObjectDeleted( "Access denied object deleted" ) except boto3exception.ClientError as e: expected_code = "AccessDenied" actual_code = e.response["Error"]["Code"] assert ( actual_code == expected_code ), "Expected: {expected_code}, Actual: {actual_code}" # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): test_info = AddTestInfo('test swift user key gen') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # preparing data user_names = ['tuffy', 'scooby', 'max'] tenant1 = 'tenant' cmd = 'radosgw-admin user create --uid=%s --display-name="%s" --tenant=%s --cluster %s' \ % (user_names[0], user_names[0], tenant1, config.cluster_name) out = utils.exec_shell_cmd(cmd) if out is False: raise TestExecError("RGW User creation error") log.info('output :%s' % out) v1_as_json = json.loads(out) log.info('creted user_id: %s' % v1_as_json['user_id']) cmd2 = 'radosgw-admin subuser create --uid=%s$%s --subuser=%s:swift --tenant=%s --access=full --cluster %s' \ % (tenant1, user_names[0], user_names[0], tenant1, config.cluster_name) out2 = utils.exec_shell_cmd(cmd2) if out2 is False: raise TestExecError("sub-user creation error") v2_as_json = json.loads(out2) log.info('created subuser: %s' % v2_as_json['subusers'][0]['id']) cmd3 = 'radosgw-admin key create --subuser=%s:swift --uid=%s$%s --tenant=%s --key-type=swift --gen-secret ' \ '--cluster %s' % (user_names[0], user_names[0], tenant1, tenant1, config.cluster_name) out3 = utils.exec_shell_cmd(cmd3) if out3 is False: raise TestExecError("secret_key gen error") v3_as_json = json.loads(out3) log.info('created subuser: %s\nsecret_key generated: %s' % (v3_as_json['swift_keys'][0]['user'], v3_as_json['swift_keys'][0]['secret_key'])) user_info = { 'user_id': v3_as_json['swift_keys'][0]['user'], 'key': v3_as_json['swift_keys'][0]['secret_key'] } auth = Auth(user_info) rgw = auth.do_auth() for cc in range(config.container_count): container_name = utils.gen_bucket_name_from_userid( user_info['user_id'], rand_no=cc) container = swiftlib.resource_op({ 'obj': rgw, 'resource': 'put_container', 'args': [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation faield") for oc in range(config.objects_count): swift_object_name = utils.gen_s3_object_name( '%s.container.%s' % (user_names[0], cc), oc) log.info('object name: %s' % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info('object path: %s' % object_path) object_size = utils.get_file_size( config.objects_size_range['min'], config.objects_size_range['max']) data_info = manage_data.io_generator(object_path, object_size) if data_info is False: TestExecError("data creation failed") log.info('uploading object: %s' % object_path) with open(object_path, 'r') as fp: rgw.put_object(container_name, swift_object_name, contents=fp.read(), content_type='text/plain') test_info.success_status('test passed') sys.exit(0) except Exception as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1) except TestExecError as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): test_info = AddTestInfo('test swift user key gen') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) umgmt = UserMgmt() try: test_info.started_info() # preparing data user_names = ['tuffy', 'scooby', 'max'] tenant = 'tenant' tenant_user_info = umgmt.create_tenant_user( tenant_name=tenant, user_id=user_names[0], displayname=user_names[0], cluster_name=config.cluster_name) user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0], cluster_name=config.cluster_name) auth = Auth(user_info) rgw = auth.do_auth() for cc in range(config.container_count): container_name = utils.gen_bucket_name_from_userid( user_info['user_id'], rand_no=cc) container = swiftlib.resource_op({ 'obj': rgw, 'resource': 'put_container', 'args': [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation faield") for oc in range(config.objects_count): swift_object_name = utils.gen_s3_object_name( '%s.container.%s' % (user_names[0], cc), oc) log.info('object name: %s' % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info('object path: %s' % object_path) object_size = utils.get_file_size( config.objects_size_range['min'], config.objects_size_range['max']) data_info = manage_data.io_generator(object_path, object_size) # upload object if data_info is False: TestExecError("data creation failed") log.info('uploading object: %s' % object_path) with open(object_path, 'r') as fp: rgw.put_object(container_name, swift_object_name, contents=fp.read(), content_type='text/plain') # download object swift_object_download_fname = swift_object_name + ".download" log.info('download object name: %s' % swift_object_download_fname) swift_object_download_path = os.path.join( TEST_DATA_PATH, swift_object_download_fname) log.info('download object path: %s' % swift_object_download_path) swift_object_downloaded = rgw.get_object( container_name, swift_object_name) with open(swift_object_download_path, 'w') as fp: fp.write(swift_object_downloaded[1]) # modify and re-upload log.info('appending new message to test_data') message_to_append = 'adding new msg after download' fp = open(swift_object_download_path, 'a+') fp.write(message_to_append) fp.close() with open(swift_object_download_path, 'r') as fp: rgw.put_object(container_name, swift_object_name, contents=fp.read(), content_type='text/plain') # delete object log.info('deleting swift object') rgw.delete_object(container_name, swift_object_name) # delete container log.info('deleting swift container') rgw.delete_container(container_name) test_info.success_status('test passed') sys.exit(0) except Exception, e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): test_info = AddTestInfo("test swift user key gen") io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) umgmt = UserMgmt() try: test_info.started_info() # preparing data user_names = ["tuffy", "scooby", "max"] tenant = "tenant" tenant_user_info = umgmt.create_tenant_user( tenant_name=tenant, user_id=user_names[0], displayname=user_names[0], cluster_name=config.cluster_name, ) user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0], cluster_name=config.cluster_name) auth = Auth(user_info) rgw = auth.do_auth() for cc in range(config.container_count): container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc) container = swiftlib.resource_op({ "obj": rgw, "resource": "put_container", "args": [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation faield") for oc in range(config.objects_count): swift_object_name = utils.gen_s3_object_name( "%s.container.%s" % (user_names[0], cc), oc) log.info("object name: %s" % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info("object path: %s" % object_path) object_size = utils.get_file_size( config.objects_size_range["min"], config.objects_size_range["max"]) data_info = manage_data.io_generator(object_path, object_size) # upload object if data_info is False: TestExecError("data creation failed") log.info("uploading object: %s" % object_path) with open(object_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", ) # download object swift_object_download_fname = swift_object_name + ".download" log.info("download object name: %s" % swift_object_download_fname) swift_object_download_path = os.path.join( TEST_DATA_PATH, swift_object_download_fname) log.info("download object path: %s" % swift_object_download_path) swift_object_downloaded = rgw.get_object( container_name, swift_object_name) with open(swift_object_download_path, "w") as fp: fp.write(swift_object_downloaded[1]) # modify and re-upload log.info("appending new message to test_data") message_to_append = "adding new msg after download" fp = open(swift_object_download_path, "a+") fp.write(message_to_append) fp.close() with open(swift_object_download_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", ) # delete object log.info("deleting swift object") rgw.delete_object(container_name, swift_object_name) # delete container log.info("deleting swift container") rgw.delete_container(container_name) test_info.success_status("test passed") sys.exit(0) except Exception as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status("test failed") sys.exit(1) except TestExecError as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status("test failed") sys.exit(1)
def write(self, io_type, fname, size=0): """ This function is to write IO on the mount point Parameters: io_type: basedir | subdir | file fname(char): file name size(int): size Returns: """ # io_type should be: basedir | subdir | file log.info('io_type: %s' % io_type) log.info('fname: %s' % fname) log.info('size: %s' % size) s3_conv = NFS_CONVENTIONS.get(io_type) ioinfo = IOInfo() path = os.path.abspath(self.mnt_point) full_path = os.path.join(path, fname) log.info('abs_path: %s' % full_path) try: if io_type == 'basedir' or io_type == 'subdir': log.info('creating dir, type: %s' % io_type) os.makedirs(full_path) io_info = { 'name': os.path.basename(fname), 'type': 'dir', 's3_convention': s3_conv, 'bucket': 'self' if s3_conv == 'bucket' else fname.split('/')[0], 'md5': None } log.info('io_info: %s' % io_info) ioinfo.add_io_info(self.rgw_user_info['access_key'], io_info) if io_type == 'file': log.info('io_type is file: %s' % io_type) log.info('creating file with size: %s' % size) finfo = manage_date.io_generator(full_path, size) io_info = { 'name': os.path.basename(fname), 'type': 'file', 's3_convention': s3_conv, 'bucket': fname.split('/')[0], 'md5': finfo['md5'] } log.info('io_info: %s' % io_info) ioinfo.add_io_info(self.rgw_user_info['access_key'], io_info) except (Exception) as e: log.error('Write IO Execution failed') log.error(e) return False
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) # preparing data user_names = ["tuffy", "scooby", "max"] tenant1 = "tenant" + "_" + str(random.randrange(1, 100)) cmd = 'radosgw-admin user create --uid=%s --display-name="%s" --tenant=%s' % ( user_names[0], user_names[0], tenant1, ) out = utils.exec_shell_cmd(cmd) if out is False: raise TestExecError("RGW User creation error") log.info("output :%s" % out) v1_as_json = json.loads(out) log.info("creted user_id: %s" % v1_as_json["user_id"]) cmd2 = ( "radosgw-admin subuser create --uid=%s$%s --subuser=%s:swift --tenant=%s --access=full" % (tenant1, user_names[0], user_names[0], tenant1)) out2 = utils.exec_shell_cmd(cmd2) if out2 is False: raise TestExecError("sub-user creation error") v2_as_json = json.loads(out2) log.info("created subuser: %s" % v2_as_json["subusers"][0]["id"]) cmd3 = ( "radosgw-admin key create --subuser=%s:swift --uid=%s$%s --tenant=%s --key-type=swift --gen-secret" % (user_names[0], user_names[0], tenant1, tenant1)) out3 = utils.exec_shell_cmd(cmd3) if out3 is False: raise TestExecError("secret_key gen error") v3_as_json = json.loads(out3) log.info("created subuser: %s\nsecret_key generated: %s" % ( v3_as_json["swift_keys"][0]["user"], v3_as_json["swift_keys"][0]["secret_key"], )) user_info = { "user_id": v3_as_json["swift_keys"][0]["user"], "key": v3_as_json["swift_keys"][0]["secret_key"], } auth = Auth(user_info, is_secure=config.ssl) rgw = auth.do_auth() for cc in range(config.container_count): container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc) container = swiftlib.resource_op({ "obj": rgw, "resource": "put_container", "args": [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation faield") for oc, size in list(config.mapped_sizes.items()): swift_object_name = utils.gen_s3_object_name( "%s.container.%s" % (user_names[0], cc), oc) log.info("object name: %s" % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info("object path: %s" % object_path) data_info = manage_data.io_generator(object_path, size) if data_info is False: TestExecError("data creation failed") log.info("uploading object: %s" % object_path) with open(object_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", ) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) umgmt = UserMgmt() # preparing data user_names = ['tuffy', 'scooby', 'max'] tenant = 'tenant' tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant, user_id=user_names[0], displayname=user_names[0]) user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0]) auth = Auth(user_info) rgw = auth.do_auth() for cc in range(config.container_count): container_name = utils.gen_bucket_name_from_userid( user_info['user_id'], rand_no=cc) container = swiftlib.resource_op({ 'obj': rgw, 'resource': 'put_container', 'args': [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation faield") for oc, size in list(config.mapped_sizes.items()): swift_object_name = utils.gen_s3_object_name( '%s.container.%s' % (user_names[0], cc), oc) log.info('object name: %s' % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info('object path: %s' % object_path) data_info = manage_data.io_generator(object_path, size) # upload object if data_info is False: TestExecError("data creation failed") log.info('uploading object: %s' % object_path) with open(object_path, 'r') as fp: rgw.put_object(container_name, swift_object_name, contents=fp.read(), content_type='text/plain') # download object swift_object_download_fname = swift_object_name + ".download" log.info('download object name: %s' % swift_object_download_fname) swift_object_download_path = os.path.join( TEST_DATA_PATH, swift_object_download_fname) log.info('download object path: %s' % swift_object_download_path) swift_object_downloaded = rgw.get_object(container_name, swift_object_name) with open(swift_object_download_path, 'w') as fp: fp.write(str(swift_object_downloaded[1])) # modify and re-upload log.info('appending new message to test_data') message_to_append = 'adding new msg after download' fp = open(swift_object_download_path, 'a+') fp.write(message_to_append) fp.close() with open(swift_object_download_path, 'r') as fp: rgw.put_object(container_name, swift_object_name, contents=fp.read(), content_type='text/plain') # delete object log.info('deleting swift object') rgw.delete_object(container_name, swift_object_name) # delete container log.info('deleting swift container') rgw.delete_container(container_name) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): test_info = AddTestInfo("storage_policy for %s" % config.rgw_client) io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) rgw_service = RGWService() try: # create pool pool_name = ".rgw.buckets.special" pg_num = "8" pgp_num = "8" pool_create = 'sudo ceph osd pool create "%s" %s %s replicated' % ( pool_name, pg_num, pgp_num, ) pool_create_exec = utils.exec_shell_cmd(pool_create) if pool_create_exec is False: raise TestExecError("Pool creation failed") # create realm realm_name = "buz-tickets" log.info("creating realm name") realm_create = ( "sudo radosgw-admin realm create --rgw-realm=%s --default" % realm_name ) realm_create_exec = utils.exec_shell_cmd(realm_create) if realm_create_exec is False: raise TestExecError("cmd execution failed") # sample output of create realm """ { "id": "0956b174-fe14-4f97-8b50-bb7ec5e1cf62", "name": "buz-tickets", "current_period": "1950b710-3e63-4c41-a19e-46a715000980", "epoch": 1 } """ log.info("modify zonegroup ") modify = ( "sudo radosgw-admin zonegroup modify --rgw-zonegroup=default --rgw-realm=%s --master --default" % realm_name ) modify_exec = utils.exec_shell_cmd(modify) if modify_exec is False: raise TestExecError("cmd execution failed") # get the zonegroup zonegroup_file = "zonegroup.json" get_zonegroup = ( "sudo radosgw-admin zonegroup --rgw-zonegroup=default get > %s" % zonegroup_file ) get_zonegroup_exec = utils.exec_shell_cmd(get_zonegroup) if get_zonegroup_exec is False: raise TestExecError("cmd execution failed") add_to_placement_targets = {"name": "special-placement", "tags": []} fp = open(zonegroup_file, "r") zonegroup_txt = fp.read() fp.close() log.info("got zonegroup info: \n%s" % zonegroup_txt) zonegroup = json.loads(zonegroup_txt) log.info("adding placement targets") zonegroup["placement_targets"].append(add_to_placement_targets) with open(zonegroup_file, "w") as fp: json.dump(zonegroup, fp) zonegroup_set = "sudo radosgw-admin zonegroup set < %s" % zonegroup_file zonegroup_set_exec = utils.exec_shell_cmd(zonegroup_set) if zonegroup_set_exec is False: raise TestExecError("cmd execution failed") log.info("zone group update completed") log.info("getting zone file") # get zone log.info("getting zone info") zone_file = "zone.json" get_zone = "sudo radosgw-admin zone --rgw-zone=default get > zone.json" get_zone_exec = utils.exec_shell_cmd(get_zone) if get_zone_exec is False: raise TestExecError("cmd execution failed") fp = open(zone_file, "r") zone_info = fp.read() fp.close() log.info("zone_info :\n%s" % zone_info) zone_info_cleaned = json.loads(zone_info) special_placement_info = { "key": "special-placement", "val": { "index_pool": ".rgw.buckets.index", "data_pool": ".rgw.buckets.special", "data_extra_pool": ".rgw.buckets.extra", }, } log.info("adding special placement info") zone_info_cleaned["placement_pools"].append(special_placement_info) print(zone_info_cleaned) with open(zone_file, "w+") as fp: json.dump(zone_info_cleaned, fp) zone_file_set = "sudo radosgw-admin zone set < %s" % zone_file zone_file_set_exec = utils.exec_shell_cmd(zone_file_set) if zone_file_set_exec is False: raise TestExecError("cmd execution failed") log.info("zone info updated ") restarted = rgw_service.restart() if restarted is False: raise TestExecError("service restart failed") if config.rgw_client == "rgw": log.info("client type is rgw") rgw_user_info = s3_swift_lib.create_users(1) auth = Auth(rgw_user_info) rgw_conn = auth.do_auth() # create bucket bucket_name = utils.gen_bucket_name_from_userid(rgw_user_info["user_id"], 0) bucket = resuables.create_bucket(bucket_name, rgw_conn, rgw_user_info) # create object s3_object_name = utils.gen_s3_object_name(bucket_name, 0) resuables.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, rgw_user_info ) if config.rgw_client == "swift": log.info("client type is swift") user_names = ["tuffy", "scooby", "max"] tenant = "tenant" umgmt = UserMgmt() umgmt.create_tenant_user( tenant_name=tenant, user_id=user_names[0], displayname=user_names[0] ) user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0]) auth = Auth(user_info) rgw = auth.do_auth() container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=0 ) container = s3_swift_lib.resource_op( {"obj": rgw, "resource": "put_container", "args": [container_name]} ) if container is False: raise TestExecError( "Resource execution failed: container creation faield" ) swift_object_name = utils.gen_s3_object_name( "%s.container.%s" % (user_names[0], 0), 0 ) log.info("object name: %s" % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info("object path: %s" % object_path) object_size = utils.get_file_size( config.objects_size_range["min"], config.objects_size_range["max"] ) data_info = manage_data.io_generator(object_path, object_size) # upload object if data_info is False: TestExecError("data creation failed") log.info("uploading object: %s" % object_path) with open(object_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", ) test_info.success_status("test passed") sys.exit(0) except Exception as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status("test failed") sys.exit(1) except TestExecError as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status("test failed") sys.exit(1)
def upload_mutipart_object(s3_object_name, bucket, TEST_DATA_PATH, config, user_info, append_data=False, append_msg=None): log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) s3_object_size = config.obj_size split_size = config.split_size if hasattr(config, 'split_size') else 5 log.info('split size: %s' % split_size) if append_data is True: data_info = manage_data.io_generator( s3_object_path, s3_object_size, op='append', **{'message': '\n%s' % append_msg}) else: data_info = manage_data.io_generator(s3_object_path, s3_object_size) if data_info is False: TestExecError("data creation failed") mp_dir = os.path.join(TEST_DATA_PATH, s3_object_name + '.mp.parts') log.info('mp part dir: %s' % mp_dir) log.info('making multipart object part dir') mkdir = utils.exec_shell_cmd('sudo mkdir %s' % mp_dir) if mkdir is False: raise TestExecError('mkdir failed creating mp_dir_name') utils.split_file(s3_object_path, split_size, mp_dir + "/") parts_list = sorted(glob.glob(mp_dir + '/' + '*')) log.info('parts_list: %s' % parts_list) log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict( { 'access_key': user_info['access_key'], 'upload_type': 'multipart' }, **data_info) s3_obj = s3lib.resource_op({ 'obj': bucket, 'resource': 'Object', 'args': [s3_object_name], }) log.info('initiating multipart upload') mpu = s3lib.resource_op({ 'obj': s3_obj, 'resource': 'initiate_multipart_upload', 'args': None, 'extra_info': upload_info }) part_number = 1 parts_info = {'Parts': []} log.info('no of parts: %s' % len(parts_list)) for each_part in parts_list: log.info('trying to upload part: %s' % each_part) part = mpu.Part(part_number) # part_upload_response = part.upload(Body=open(each_part)) part_upload_response = s3lib.resource_op({ 'obj': part, 'resource': 'upload', 'kwargs': dict(Body=open(each_part, mode="rb")) }) if part_upload_response is not False: response = HttpResponseParser(part_upload_response) if response.status_code == 200: log.info('part uploaded') if config.local_file_delete is True: log.info('deleting local file part') utils.exec_shell_cmd('sudo rm -rf %s' % each_part) else: raise TestExecError("part uploading failed") part_info = { 'PartNumber': part_number, 'ETag': part_upload_response['ETag'] } parts_info['Parts'].append(part_info) if each_part != parts_list[-1]: # increase the part number only if the current part is not the last part part_number += 1 log.info('curr part_number: %s' % part_number) # log.info('parts_info so far: %s'% parts_info) if len(parts_list) == part_number: log.info('all parts upload completed') mpu.complete(MultipartUpload=parts_info) log.info('multipart upload complete for key: %s' % s3_object_name)
def upload_version_object(config, user_info, rgw_conn, s3_object_name, object_size, bucket, TEST_DATA_PATH): # versioning upload log.info('versioning count: %s' % config.version_count) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, object_size) if original_data_info is False: TestExecError("data creation failed") created_versions_count = 0 for vc in range(config.version_count): log.info('version count for %s is %s' % (s3_object_name, str(vc))) log.info('modifying data: %s' % s3_object_name) modified_data_info = manage_data.io_generator( s3_object_path, object_size, op='append', **{'message': '\nhello for version: %s\n' % str(vc)}) if modified_data_info is False: TestExecError("data modification failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict( { 'access_key': user_info['access_key'], 'versioning_status': 'enabled', 'version_count_no': vc }, **modified_data_info) s3_obj = s3lib.resource_op({ 'obj': bucket, 'resource': 'Object', 'args': [s3_object_name], 'extra_info': upload_info, }) object_uploaded_status = s3lib.resource_op({ 'obj': s3_obj, 'resource': 'upload_file', 'args': [modified_data_info['name']], 'extra_info': upload_info }) if object_uploaded_status is False: raise TestExecError( "Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current_version_id: %s' % s3_obj.version_id) key_version_info = basic_io_structure.version_info( **{ 'version_id': s3_obj.version_id, 'md5_local': upload_info['md5'], 'count_no': vc, 'size': upload_info['size'] }) log.info('key_version_info: %s' % key_version_info) write_key_io_info.add_versioning_info(user_info['access_key'], bucket.name, s3_object_path, key_version_info) created_versions_count += 1 log.info('created_versions_count: %s' % created_versions_count) log.info('adding metadata') metadata1 = {"m_data1": "this is the meta1 for this obj"} s3_obj.metadata.update(metadata1) metadata2 = {"m_data2": "this is the meta2 for this obj"} s3_obj.metadata.update(metadata2) log.info('metadata for this object: %s' % s3_obj.metadata) log.info('metadata count for object: %s' % (len(s3_obj.metadata))) if not s3_obj.metadata: raise TestExecError( 'metadata not created even adding metadata') versions = bucket.object_versions.filter(Prefix=s3_object_name) created_versions_count_from_s3 = len( [v.version_id for v in versions]) log.info('created versions count on s3: %s' % created_versions_count_from_s3) if created_versions_count is created_versions_count_from_s3: log.info('no new versions are created when added metdata') else: raise TestExecError( "version count missmatch, " "possible creation of version on adding metadata") s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({ 'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('downloaded_md5: %s' % s3_object_downloaded_md5) log.info('uploaded_md5: %s' % modified_data_info['md5']) log.info('deleting downloaded version file') utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_download_path) log.info('all versions for the object: %s\n' % s3_object_name)