def io_generator(fname, size, type='txt', op='create', **kwargs): finfo = {'name': fname, 'size': None, 'md5': None} # fname should include path, ex: /some/path/filename try: if op == 'create': log.info('in create') if type == 'txt': fcreate = 'base64 /dev/urandom | head -c %sM > %s' % (size, fname) created = utils.exec_shell_cmd(fcreate) finfo['md5'] = utils.get_md5(fname) finfo['size'] = os.stat(fname).st_size if created is False: raise RGWIOGenException("file %s creation error" % fname) return finfo if op == 'append': log.info('in modify or append') message = kwargs['message'] log.info('message to append: %s' % message) fappend = open(fname, 'a+') fappend.write(message) fappend.close() finfo['md5'] = utils.get_md5(fname) finfo['size'] = os.stat(fname).st_size return finfo except RGWIOGenException as e: log.error(e) return False
def download_object(s3_object_name, bucket, TEST_DATA_PATH, s3_object_path, config): log.info("s3 object name to download: %s" % s3_object_name) s3_object_download_name = s3_object_name + "." + "download" s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_download_name) object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed") if object_downloaded_status is None: log.info("object downloaded") s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) s3_object_uploaded_md5 = utils.get_md5(s3_object_path) log.info("s3_object_downloaded_md5: %s" % s3_object_downloaded_md5) log.info("s3_object_uploaded_md5: %s" % s3_object_uploaded_md5) if str(s3_object_uploaded_md5) == str(s3_object_downloaded_md5): log.info("md5 match") utils.exec_shell_cmd("rm -rf %s" % s3_object_download_path) else: raise TestExecError("md5 mismatch") if config.local_file_delete is True: log.info("deleting local file created after the upload") utils.exec_shell_cmd("rm -rf %s" % s3_object_path)
def verify_if_files_created(self): if not self.files: log.info('no files are created') else: log.info('verifying files') for each_file in self.files: log.info('verifying existence for: %s' % each_file['file']) created = os.path.exists(each_file['file']) if not created: raise TestExecError("files not created") log.info('file created') md5 = utils.get_md5(each_file['file']) log.info('md5 on nfs mount point: %s' % md5) log.info('md5 on rgw_client: %s' % each_file['md5']) if md5 != each_file['md5']: raise TestExecError("md5 not matched") log.info('md5 matched') log.info( 'verification of files complete, files exists and data intact')
def verify_if_objects_created(self): log.info('verification of s3 objects') for each_key in self.objects: log.info('verifying data for key: %s' % os.path.basename(each_key['name'])) log.info('bucket: %s' % each_key['bucket']) key_from_s3 = self.rgw_conn.Object( each_key['bucket'], os.path.basename(each_key['name'])) log.info('got key name from s3: %s' % key_from_s3.key) if each_key['type'] == 'file': log.info('verifying md5') log.info('md5_local: %s' % each_key['md5']) key_from_s3.download_file('download.temp') downloaded_md5 = utils.get_md5('download.temp') log.info('md5_from_s3: %s' % downloaded_md5) if each_key['md5'] != downloaded_md5: raise TestExecError("md5 not matched") utils.exec_shell_cmd('sudo rm -rf download.temp')
def verify_key(each_key, bucket): """ This function verifies data of each key in the bucket Parameters: key(char): key to be verified bucket(char): bucket name """ log.info("verifying data for key: %s" % os.path.basename(each_key["name"])) check_object_exists(os.path.basename(each_key["name"]), bucket) key_from_s3 = bucket.Object(os.path.basename(each_key["name"])) log.info("verifying size") log.info("size from yaml: %s" % each_key["size"]) log.info("size from s3: %s" % key_from_s3.content_length) if int(each_key["size"]) != int(key_from_s3.content_length): raise TestExecError("Size not matched") log.info("verifying md5") log.info("md5_local: %s" % each_key["md5_local"]) key_from_s3.download_file("download.temp") downloaded_md5 = utils.get_md5("download.temp") log.info("md5_from_s3: %s" % downloaded_md5) if each_key["md5_local"] != downloaded_md5: raise TestExecError("Md5 not matched") utils.exec_shell_cmd("sudo rm -rf download.temp") log.info("verification complete for the key: %s" % key_from_s3.key)
def verify_key_with_version(each_key, bucket): log.info('verifying data for key: %s' % os.path.basename(each_key['name'])) key_from_s3 = bucket.Object(os.path.basename(each_key['name'])) no_of_versions = len(each_key['versioning_info']) log.info('no of versions: %s' % no_of_versions) for each_version in each_key['versioning_info']: log.info('version_id: %s' % each_version['version_id']) key_from_s3_with_version = key_from_s3.get( VersionId=each_version['version_id']) log.info('verifying size') log.info('size from yaml: %s' % each_version['size']) log.info('size from s3 %s' % key_from_s3_with_version['ContentLength']) if int(each_version['size'] != int( key_from_s3_with_version['ContentLength'])): raise TestExecError('Size not matched') log.info('verifying md5') log.info('md5_local: %s' % each_version['md5_local']) key_from_s3.download_file( 'download.temp', ExtraArgs={'VersionId': each_version['version_id']}) downloaded_md5 = utils.get_md5('download.temp') log.info('md5_from_s3: %s' % downloaded_md5) if each_version['md5_local'] != downloaded_md5: raise TestExecError("Md5 not matched") utils.exec_shell_cmd('sudo rm -rf download.temp') log.info('verification complete for the key: %s ---> version_id: %s' % (key_from_s3.key, each_version['version_id']))
def verify_key(each_key, bucket): """ This function verifies data of each key in the bucket Parameters: key(char): key to be verified bucket(char): bucket name Retuns: """ log.info('verifying data for key: %s' % os.path.basename(each_key['name'])) key_from_s3 = bucket.Object(os.path.basename(each_key['name'])) log.info('verifying size') log.info('size from yaml: %s' % each_key['size']) log.info('size from s3: %s' % key_from_s3.content_length) if int(each_key['size']) != int(key_from_s3.content_length): raise TestExecError("Size not matched") log.info('verifying md5') log.info('md5_local: %s' % each_key['md5_local']) key_from_s3.download_file('download.temp') downloaded_md5 = utils.get_md5('download.temp') log.info('md5_from_s3: %s' % downloaded_md5) if each_key['md5_local'] != downloaded_md5: raise TestExecError("Md5 not matched") utils.exec_shell_cmd('sudo rm -rf download.temp') log.info('verification complete for the key: %s' % key_from_s3.key)
def io_generator(fname, size, type="txt", op="create", **kwargs): """ Function to generate IOs This function helps in creating file with random data Parameters: op(char): create or append. Returns: finfo : file information is returned. """ finfo = {"name": fname, "size": None, "md5": None} # fname should include path, ex: /some/path/filename try: if op == "create": log.info("in create") if type == "txt": fcreate = "base64 /dev/urandom | head -c %s > %s" % (size, fname) created = utils.exec_shell_cmd(fcreate) finfo["md5"] = utils.get_md5(fname) finfo["size"] = os.stat(fname).st_size if created is False: raise RGWIOGenException("file %s creation error" % fname) return finfo if op == "append": log.info("in modify or append") message = kwargs["message"] log.info("message to append: %s" % message) fappend = open(fname, "a+") fappend.write(message) fappend.close() finfo["md5"] = utils.get_md5(fname) finfo["size"] = os.stat(fname).st_size return finfo except RGWIOGenException as e: log.error(e) return False
def verify_if_objects_created(self): log.info("verification of s3 objects") for each_key in self.objects: log.info("verifying data for key: %s" % os.path.basename(each_key["name"])) log.info("bucket: %s" % each_key["bucket"]) key_from_s3 = self.rgw_conn.Object( each_key["bucket"], os.path.basename(each_key["name"]) ) log.info("got key name from s3: %s" % key_from_s3.key) if each_key["type"] == "file": log.info("verifying md5") log.info("md5_local: %s" % each_key["md5"]) key_from_s3.download_file("download.temp") downloaded_md5 = utils.get_md5("download.temp") log.info("md5_from_s3: %s" % downloaded_md5) if each_key["md5"] != downloaded_md5: raise TestExecError("md5 not matched") utils.exec_shell_cmd("sudo rm -rf download.temp")
def verify_key_with_version(each_key, bucket): """ This function verifies data of each key in a versioned bucket Parameters: key(char): key to be verified bucket(char): name of the versoiined bucket Returns: """ log.info("verifying data for key: %s" % os.path.basename(each_key["name"])) check_object_exists(os.path.basename(each_key["name"]), bucket) key_from_s3 = bucket.Object(os.path.basename(each_key["name"])) no_of_versions = len(each_key["versioning_info"]) log.info("no of versions: %s" % no_of_versions) for each_version in each_key["versioning_info"]: log.info("version_id: %s" % each_version["version_id"]) key_from_s3_with_version = key_from_s3.get(VersionId=each_version["version_id"]) log.info("verifying size") log.info("size from yaml: %s" % each_version["size"]) log.info("size from s3 %s" % key_from_s3_with_version["ContentLength"]) if int(each_version["size"] != int(key_from_s3_with_version["ContentLength"])): raise TestExecError("Size not matched") log.info("verifying md5") log.info("md5_local: %s" % each_version["md5_local"]) key_from_s3.download_file( "download.temp", ExtraArgs={"VersionId": each_version["version_id"]} ) downloaded_md5 = utils.get_md5("download.temp") log.info("md5_from_s3: %s" % downloaded_md5) if each_version["md5_local"] != downloaded_md5: raise TestExecError("Md5 not matched") utils.exec_shell_cmd("sudo rm -rf download.temp") log.info( "verification complete for the key: %s ---> version_id: %s" % (key_from_s3.key, each_version["version_id"]) )
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() write_key_io_info = KeyIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) # create user all_users_info = s3lib.create_users(config.user_count) extra_user = s3lib.create_users(1)[0] extra_user_auth = Auth(extra_user, ssl=config.ssl) extra_user_conn = extra_user_auth.do_auth() for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Bucket', 'args': [bucket_name_to_create]}) # created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']}) created = s3lib.resource_op({'obj': bucket, 'resource': 'create', 'args': None, 'extra_info': {'access_key': each_user['access_key']}}) if created is False: raise TestExecError("Resource execution failed: bucket creation faield") if created is not None: response = HttpResponseParser(created) if response.status_code == 200: log.info('bucket created') else: raise TestExecError("bucket creation failed") else: raise TestExecError("bucket creation failed") # getting bucket version object if config.test_ops['enable_version'] is True: log.info('bucket versionig test on bucket: %s' % bucket.name) # bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name) bucket_versioning = s3lib.resource_op({'obj': rgw_conn, 'resource': 'BucketVersioning', 'args': [bucket.name]}) # checking the versioning status # version_status = s3_ops.resource_op(bucket_versioning, 'status') version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'status', 'args': None }) if version_status is None: log.info('bucket versioning still not enabled') # enabling bucket versioning # version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable') version_enable_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'enable', 'args': None, }) response = HttpResponseParser(version_enable_status) if response.status_code == 200: log.info('version enabled') write_bucket_io_info.add_versioning_status(each_user['access_key'],bucket.name, VERSIONING_STATUS['ENABLED']) else: raise TestExecError("version enable failed") if config.objects_count > 0: log.info('s3 objects to create: %s' % config.objects_count) for oc, s3_object_size in list(config.mapped_sizes.items()): # versioning upload s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc)) s3_object_names.append(s3_object_name) log.info('s3 object name: %s' % s3_object_name) log.info('versioning count: %s' % config.version_count) s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc)) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, s3_object_size) if original_data_info is False: TestExecError("data creation failed") created_versions_count = 0 for vc in range(config.version_count): log.info('version count for %s is %s' % (s3_object_name, str(vc))) log.info('modifying data: %s' % s3_object_name) modified_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op='append', **{'message': '\nhello for version: %s\n' % str(vc)}) if modified_data_info is False: TestExecError("data modification failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key'], 'versioning_status': VERSIONING_STATUS['ENABLED'], 'version_count_no': vc}, **modified_data_info) s3_obj = s3lib.resource_op({'obj': bucket, 'resource': 'Object', 'args': [s3_object_name], 'extra_info': upload_info, }) object_uploaded_status = s3lib.resource_op({'obj': s3_obj, 'resource': 'upload_file', 'args': [modified_data_info['name']], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current_version_id: %s' % s3_obj.version_id) key_version_info = basic_io_structure.version_info( **{'version_id': s3_obj.version_id, 'md5_local': upload_info['md5'], 'count_no': vc, 'size': upload_info['size']}) log.info('key_version_info: %s' % key_version_info) write_key_io_info.add_versioning_info(each_user['access_key'], bucket.name, s3_object_path, key_version_info) created_versions_count += 1 log.info('created_versions_count: %s' % created_versions_count) log.info('adding metadata') metadata1 = {"m_data1": "this is the meta1 for this obj"} s3_obj.metadata.update(metadata1) metadata2 = {"m_data2": "this is the meta2 for this obj"} s3_obj.metadata.update(metadata2) log.info('metadata for this object: %s' % s3_obj.metadata) log.info('metadata count for object: %s' % (len(s3_obj.metadata))) if not s3_obj.metadata: raise TestExecError('metadata not created even adding metadata') versions = bucket.object_versions.filter(Prefix=s3_object_name) created_versions_count_from_s3 = len([v.version_id for v in versions]) log.info('created versions count on s3: %s' % created_versions_count_from_s3) if created_versions_count is created_versions_count_from_s3: log.info('no new versions are created when added metdata') else: raise TestExecError("version count missmatch, " "possible creation of version on adding metadata") s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError("Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('downloaded_md5: %s' % s3_object_downloaded_md5) log.info('uploaded_md5: %s' % modified_data_info['md5']) # tail_op = utils.exec_shell_cmd('tail -l %s' % s3_object_download_path) log.info('all versions for the object: %s\n' % s3_object_name) versions = bucket.object_versions.filter(Prefix=s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) if config.test_ops.get('set_acl', None) is True: s3_obj_acl = s3lib.resource_op({'obj': rgw_conn, 'resource': 'ObjectAcl', 'args': [bucket.name, s3_object_name]}) # setting acl to private, just need to set to any acl and # check if its set - check by response code acls_set_status = s3_obj_acl.put(ACL='private') response = HttpResponseParser(acls_set_status) if response.status_code == 200: log.info('ACLs set') else: raise TestExecError("Acls not Set") # get obj details based on version id for version in versions: log.info('getting info for version id: %s' % version.version_id) obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('obj get detils :%s\n' % (obj.get(VersionId=version.version_id))) if config.test_ops['copy_to_version'] is True: # reverting object to one of the versions ( randomly chosen ) version_id_to_copy = random.choice([v.version_id for v in versions]) log.info('version_id_to_copy: %s' % version_id_to_copy) s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current version_id: %s' % s3_obj.version_id) copy_response = s3_obj.copy_from(CopySource={'Bucket': bucket.name, 'Key': s3_object_name, 'VersionId': version_id_to_copy}) log.info('copy_response: %s' % copy_response) if copy_response is None: raise TestExecError("copy object from version id failed") # current_version_id = copy_response['VersionID'] log.info('current_version_id: %s' % s3_obj.version_id) # delete the version_id_to_copy object s3_obj.delete(VersionId=version_id_to_copy) log.info('all versions for the object after the copy operation: %s\n' % s3_object_name) for version in versions: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) # log.info('downloading current s3object: %s' % s3_object_name) # s3_obj.download_file(s3_object_name + ".download") if config.test_ops['delete_object_versions'] is True: log.info('deleting s3_obj keys and its versions') s3_obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('deleting versions for s3 obj: %s' % s3_object_name) for version in versions: log.info('trying to delete obj version: %s' % version.version_id) del_obj_version = s3lib.resource_op({'obj': s3_obj, 'resource': 'delete', 'kwargs': dict(VersionId=version.version_id)}) log.info('response:\n%s' % del_obj_version) if del_obj_version is not None: response = HttpResponseParser(del_obj_version) if response.status_code == 204: log.info('version deleted ') write_key_io_info.delete_version_info(each_user['access_key'], bucket.name, s3_object_path, version.version_id) else: raise TestExecError("version deletion failed") else: raise TestExecError("version deletion failed") log.info('available versions for the object') versions = bucket.object_versions.filter(Prefix=s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' % ( version.object_key, version.version_id)) if config.test_ops.get('delete_from_extra_user') is True: log.info('trying to delete objects from extra user') s3_obj = s3lib.resource_op({'obj': extra_user_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('deleting versions for s3 obj: %s' % s3_object_name) for version in versions: log.info('trying to delete obj version: %s' % version.version_id) del_obj_version = s3lib.resource_op({'obj': s3_obj, 'resource': 'delete', 'kwargs': dict( VersionId=version.version_id)}) log.info('response:\n%s' % del_obj_version) if del_obj_version is not False: response = HttpResponseParser(del_obj_version) if response.status_code == 204: log.info('version deleted ') write_key_io_info.delete_version_info(each_user['access_key'], bucket.name, s3_object_path, version.version_id) raise TestExecError("version and deleted, this should not happen") else: log.info('version did not delete, expected behaviour') else: log.info('version did not delete, expected behaviour') if config.local_file_delete is True: log.info('deleting local file') utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_path) if config.test_ops['suspend_version'] is True: log.info('suspending versioning') # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') suspend_version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'suspend', 'args': None}) response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info('versioning suspended') write_bucket_io_info.add_versioning_status(each_user['access_key'], bucket.name, VERSIONING_STATUS['SUSPENDED']) else: raise TestExecError("version suspend failed") # getting all objects in the bucket log.info('getting all objects in the bucket') objects = s3lib.resource_op({'obj': bucket, 'resource': 'objects', 'args': None}) log.info('objects :%s' % objects) all_objects = s3lib.resource_op({'obj': objects, 'resource': 'all', 'args': None}) log.info('all objects: %s' % all_objects) log.info('all objects2 :%s ' % bucket.objects.all()) for obj in all_objects: log.info('object_name: %s' % obj.key) versions = bucket.object_versions.filter(Prefix=obj.key) log.info('displaying all versions of the object') for version in versions: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) if config.test_ops.get('suspend_from_extra_user') is True: log.info('suspending versioning from extra user') # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') bucket_versioning = s3lib.resource_op({'obj': extra_user_conn, 'resource': 'BucketVersioning', 'args': [bucket.name]}) suspend_version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'suspend', 'args': None}) if suspend_version_status is not False: response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info('versioning suspended') write_bucket_io_info.add_versioning_status(each_user['access_key'], bucket.name, VERSIONING_STATUS['SUSPENDED']) raise TestExecError('version suspended, this should not happen') else: log.info('versioning not suspended, expected behaviour') if config.test_ops.get('upload_after_suspend') is True: log.info('trying to upload after suspending versioning on bucket') for oc, s3_object_size in list(config.mapped_sizes.items()): # non versioning upload s3_object_name = s3_object_names[oc] + ".after_version_suspending" log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) non_version_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op="append", **{ 'message': '\nhello for non version\n'}) if non_version_data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key'], 'versioning_status': 'suspended'},**non_version_data_info) s3_obj = s3lib.resource_op({'obj': bucket, 'resource': 'Object', 'args': [s3_object_name], 'extra_info': upload_info}) object_uploaded_status = s3lib.resource_op({'obj': s3_obj, 'resource': 'upload_file', 'args': [non_version_data_info['name']], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('version_id: %s' % s3_obj.version_id) if s3_obj.version_id is None: log.info('Versions are not created after suspending') else: raise TestExecError('Versions are created even after suspending') s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError("Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5) log.info('s3_object_uploaded_md5: %s' % non_version_data_info['md5']) if config.local_file_delete is True: utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_path)
def test_exec(config): """ Executes test based on configuration passed Args: config(object): Test configuration """ io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) umgmt = UserMgmt() ceph_conf = CephConfOp() rgw_service = RGWService() # preparing data user_name = resource_op.create_users(no_of_users_to_create=1)[0]["user_id"] tenant = "tenant" tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant, user_id=user_name, displayname=user_name) umgmt.create_subuser(tenant_name=tenant, user_id=user_name) ip_and_port = s3cmd_reusable.get_rgw_ip_and_port() s3_auth.do_auth(tenant_user_info, ip_and_port) bucket_name = utils.gen_bucket_name_from_userid(user_name, rand_no=0) # Create a bucket s3cmd_reusable.create_bucket(bucket_name) log.info(f"Bucket {bucket_name} created") # Upload a 2GB file to bucket uploaded_file_info = s3cmd_reusable.upload_file( bucket_name, file_size=2147483648, test_data_path=TEST_DATA_PATH) uploaded_file = uploaded_file_info["name"] uploaded_file_md5 = uploaded_file_info["md5"] log.info(f"Uploaded file {uploaded_file} to bucket {bucket_name}") if config.gc_verification is True: log.info("making changes to ceph.conf") config.rgw_gc_obj_min_wait = 5 ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_gc_obj_min_wait, str(config.rgw_gc_obj_min_wait), ) log.info("trying to restart services") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") log.info( "download large object again to make gc list with shadow entries") downloaded_file1 = s3cmd_reusable.download_file( bucket_name, uploaded_file, local_file_name="download1.img", test_data_path=TEST_DATA_PATH, ) time.sleep(5) downloaded_file1_md5 = utils.get_md5(downloaded_file1) assert uploaded_file_md5 == downloaded_file1_md5 gc_list_output = json.loads( utils.exec_shell_cmd("radosgw-admin gc list --include-all")) log.info(gc_list_output) if gc_list_output: log.info( "Shadow obj found after setting rgw_gc_obj_min_wait to 5 sec") utils.exec_shell_cmd("radosgw-admin gc process --include-all") log.info( "Object download should not error out in 404 NoSuchKey error") downloaded_file2 = s3cmd_reusable.download_file( bucket_name, uploaded_file, local_file_name="download2.img", test_data_path=TEST_DATA_PATH, ) downloaded_file2_md5 = utils.get_md5(downloaded_file2) assert uploaded_file_md5 == downloaded_file2_md5 # Delete file from bucket s3cmd_reusable.delete_file(bucket_name, uploaded_file) log.info(f"Deleted file {uploaded_file} from bucket {bucket_name}") # Delete bucket s3cmd_reusable.delete_bucket(bucket_name) log.info(f"Bucket {bucket_name} deleted") # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user if config.dbr_scenario == "brownfield": user_brownfiled = "brownfield_user" all_users_info = s3lib.create_users(config.user_count, user_brownfiled) else: all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get("encryption_algorithm", None) is not None: log.info("encryption enabled, making ceph config changes") ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{"signature_version": "s3v4"}) else: rgw_conn = auth.do_auth() # enabling sharding if config.test_ops["sharding"]["enable"] is True: log.info("enabling sharding on buckets") max_shards = config.test_ops["sharding"]["max_shards"] log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_override_bucket_index_max_shards, str(max_shards), ) log.info("trying to restart services ") srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.test_ops["compression"]["enable"] is True: compression_type = config.test_ops["compression"]["type"] log.info("enabling compression") cmd = "radosgw-admin zone get" out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = ( "radosgw-admin zone placement modify --rgw-zone=%s " "--placement-id=default-placement --compression=%s" % (zone, compression_type) ) out = utils.exec_shell_cmd(cmd) ceph_version = utils.exec_shell_cmd("ceph version").split()[4] try: data = json.loads(out) if ceph_version == "luminous": if ( data["placement_pools"][0]["val"]["compression"] == compression_type ): log.info("Compression enabled successfully") else: if ceph_version in ["nautilus", "octopus"]: if ( data["placement_pools"][0]["val"]["storage_classes"][ "STANDARD" ]["compression_type"] == compression_type ): log.info("Compression enabled successfully") except ValueError as e: exit(str(e)) log.info("trying to restart rgw services ") srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.gc_verification is True: conf = config.ceph_conf reusable.set_gc_conf(ceph_conf, conf) if config.dynamic_resharding is True: if utils.check_dbr_support(): log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_max_objs_per_shard, str(config.max_objects_per_shard), ) srv_restarted = rgw_service.restart() # create buckets if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc ) if config.bucket_sync_crash is True: is_primary = utils.is_cluster_primary() if is_primary: bucket_name_to_create = "bkt_crash_check" if config.dbr_scenario == "brownfield": bucket_name_to_create = "brownfield_bucket" log.info("creating bucket with name: %s" % bucket_name_to_create) bucket = reusable.create_bucket( bucket_name_to_create, rgw_conn, each_user ) if config.dynamic_resharding is True: reusable.check_sync_status() op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) old_num_shards = json_doc["num_shards"] log.info(f"no_of_shards_created: {old_num_shards}") if config.test_ops["create_object"] is True: # uploading data log.info("s3 objects to create: %s" % config.objects_count) if utils.check_dbr_support(): if bucket_name_to_create == "brownfield_bucket": op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) if bool(json_doc["usage"]): num_object = json_doc["usage"]["rgw.main"][ "num_objects" ] config.objects_count = ( num_object * 2 + config.objects_count ) config.mapped_sizes = utils.make_mapped_sizes(config) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc ) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) if config.test_ops["download_object"] is True: log.info("trying to download object: %s" % s3_object_name) s3_object_download_name = s3_object_name + "." + "download" s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_download_name ) log.info( "s3_object_download_path: %s" % s3_object_download_path ) log.info( "downloading to filename: %s" % s3_object_download_name ) if ( config.test_ops.get("encryption_algorithm", None) is not None ): log.info("encryption download") log.info( "encryption algorithm: %s" % config.test_ops["encryption_algorithm"] ) object_downloaded_status = bucket.download_file( s3_object_name, s3_object_download_path, ExtraArgs={ "SSECustomerKey": encryption_key, "SSECustomerAlgorithm": config.test_ops[ "encryption_algorithm" ], }, ) else: object_downloaded_status = s3lib.resource_op( { "obj": bucket, "resource": "download_file", "args": [ s3_object_name, s3_object_download_path, ], } ) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info("object downloaded") s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path ) s3_object_uploaded_md5 = utils.get_md5(s3_object_path) log.info( "s3_object_downloaded_md5: %s" % s3_object_downloaded_md5 ) log.info( "s3_object_uploaded_md5: %s" % s3_object_uploaded_md5 ) if str(s3_object_uploaded_md5) == str( s3_object_downloaded_md5 ): log.info("md5 match") utils.exec_shell_cmd( "rm -rf %s" % s3_object_download_path ) else: raise TestExecError("md5 mismatch") if config.local_file_delete is True: log.info("deleting local file created after the upload") utils.exec_shell_cmd("rm -rf %s" % s3_object_path) if config.bucket_sync_crash is True: is_primary = utils.is_cluster_primary() if is_primary is False: crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!") realm, source_zone = utils.get_realm_source_zone_info() log.info(f"Realm name: {realm}") log.info(f"Source zone name: {source_zone}") for i in range(600): # Running sync command for 600 times op = utils.exec_shell_cmd( f"radosgw-admin bucket sync run --bucket bkt_crash_check --rgw-curl-low-speed-time=0 --source-zone {source_zone} --rgw-realm {realm}" ) crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!") time.sleep(1) if config.dynamic_resharding is True: if utils.check_dbr_support(): reusable.check_sync_status() for i in range(10): time.sleep( 60 ) # Adding delay for processing reshard list op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) new_num_shards = json_doc["num_shards"] log.info(f"no_of_shards_created: {new_num_shards}") if new_num_shards > old_num_shards: break else: raise TestExecError( "num shards are same after processing resharding" ) if config.manual_resharding is True: if utils.check_dbr_support(): op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) old_num_shards = json_doc["num_shards"] log.info(f"no_of_shards_created: {old_num_shards}") op = utils.exec_shell_cmd( f"radosgw-admin reshard add --bucket {bucket.name} --num-shards {config.shards}" ) op = utils.exec_shell_cmd("radosgw-admin reshard process") time.sleep(60) op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) new_num_shards = json_doc["num_shards"] log.info(f"no_of_shards_created: {new_num_shards}") if new_num_shards <= old_num_shards: raise TestExecError( "num shards are same after processing resharding" ) # verification of shards after upload if config.test_datalog_trim_command is True: shard_id, end_marker = reusable.get_datalog_marker() cmd = f"sudo radosgw-admin datalog trim --shard-id {shard_id} --end-marker {end_marker} --debug_ms=1 --debug_rgw=20" out, err = utils.exec_shell_cmd(cmd, debug_info=True) if "Segmentation fault" in err: raise TestExecError("Segmentation fault occured") if config.test_ops["sharding"]["enable"] is True: cmd = ( "radosgw-admin metadata get bucket:%s | grep bucket_id" % bucket.name ) out = utils.exec_shell_cmd(cmd) b_id = ( out.replace('"', "") .strip() .split(":")[1] .strip() .replace(",", "") ) cmd2 = "rados -p default.rgw.buckets.index ls | grep %s" % b_id out = utils.exec_shell_cmd(cmd2) log.info("got output from sharing verification.--------") # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops["compression"]["enable"] is True: cmd = "radosgw-admin bucket stats --bucket=%s" % bucket.name out = utils.exec_shell_cmd(cmd) # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops["compression"]["enable"] is True: cmd = "radosgw-admin bucket stats --bucket=%s" % bucket.name out = utils.exec_shell_cmd(cmd) if config.test_ops["delete_bucket_object"] is True: reusable.delete_objects(bucket) time.sleep(10) reusable.check_sync_status() reusable.delete_bucket(bucket) # disable compression after test if config.test_ops["compression"]["enable"] is True: log.info("disable compression") cmd = "radosgw-admin zone get" out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = ( "radosgw-admin zone placement modify --rgw-zone=%s " "--placement-id=default-placement --compression=none" % zone ) out = utils.exec_shell_cmd(cmd) srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.gc_verification is True: final_op = reusable.verify_gc() if final_op != -1: test_info.failed_status("test failed") sys.exit(1) # check sync status if a multisite cluster reusable.check_sync_status() # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get("encryption_algorithm", None) is not None: log.info("encryption enabled, making ceph config changes") ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{"signature_version": "s3v4"}) else: rgw_conn = auth.do_auth() # enabling sharding if config.test_ops["sharding"]["enable"] is True: log.info("enabling sharding on buckets") max_shards = config.test_ops["sharding"]["max_shards"] log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_override_bucket_index_max_shards, str(max_shards), ) log.info("trying to restart services ") srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.test_ops["compression"]["enable"] is True: compression_type = config.test_ops["compression"]["type"] log.info("enabling compression") cmd = "radosgw-admin zone get" out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = ("radosgw-admin zone placement modify --rgw-zone=%s " "--placement-id=default-placement --compression=%s" % (zone, compression_type)) out = utils.exec_shell_cmd(cmd) ceph_version = utils.exec_shell_cmd("ceph version").split()[4] try: data = json.loads(out) if ceph_version == "luminous": if (data["placement_pools"][0]["val"]["compression"] == compression_type): log.info("Compression enabled successfully") else: if ceph_version in ["nautilus", "octopus"]: if (data["placement_pools"][0]["val"] ["storage_classes"]["STANDARD"]["compression_type"] == compression_type): log.info("Compression enabled successfully") except ValueError as e: exit(str(e)) log.info("trying to restart rgw services ") srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.gc_verification is True: conf = config.ceph_conf reusable.set_gc_conf(ceph_conf, conf) # create buckets if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc) log.info("creating bucket with name: %s" % bucket_name_to_create) bucket = reusable.create_bucket(bucket_name_to_create, rgw_conn, each_user) if config.test_ops["create_object"] is True: # uploading data log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) if config.test_ops["download_object"] is True: log.info("trying to download object: %s" % s3_object_name) s3_object_download_name = s3_object_name + "." + "download" s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_download_name) log.info("s3_object_download_path: %s" % s3_object_download_path) log.info("downloading to filename: %s" % s3_object_download_name) if (config.test_ops.get("encryption_algorithm", None) is not None): log.info("encryption download") log.info( "encryption algorithm: %s" % config.test_ops["encryption_algorithm"]) object_downloaded_status = bucket.download_file( s3_object_name, s3_object_download_path, ExtraArgs={ "SSECustomerKey": encryption_key, "SSECustomerAlgorithm": config. test_ops["encryption_algorithm"], }, ) else: object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [ s3_object_name, s3_object_download_path, ], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info("object downloaded") s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path) s3_object_uploaded_md5 = utils.get_md5( s3_object_path) log.info("s3_object_downloaded_md5: %s" % s3_object_downloaded_md5) log.info("s3_object_uploaded_md5: %s" % s3_object_uploaded_md5) if str(s3_object_uploaded_md5) == str( s3_object_downloaded_md5): log.info("md5 match") utils.exec_shell_cmd("rm -rf %s" % s3_object_download_path) else: raise TestExecError("md5 mismatch") if config.local_file_delete is True: log.info( "deleting local file created after the upload") utils.exec_shell_cmd("rm -rf %s" % s3_object_path) # verification of shards after upload if config.test_ops["sharding"]["enable"] is True: cmd = ( "radosgw-admin metadata get bucket:%s | grep bucket_id" % bucket.name) out = utils.exec_shell_cmd(cmd) b_id = (out.replace( '"', "").strip().split(":")[1].strip().replace(",", "")) cmd2 = "rados -p default.rgw.buckets.index ls | grep %s" % b_id out = utils.exec_shell_cmd(cmd2) log.info( "got output from sharing verification.--------") # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops["compression"]["enable"] is True: cmd = "radosgw-admin bucket stats --bucket=%s" % bucket.name out = utils.exec_shell_cmd(cmd) # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops["compression"]["enable"] is True: cmd = "radosgw-admin bucket stats --bucket=%s" % bucket.name out = utils.exec_shell_cmd(cmd) if config.test_ops["delete_bucket_object"] is True: reusable.delete_objects(bucket) log.info( "set debug_rgw to 20 before delete the bucket") config.debug_rgw = 20 ceph_conf.set_to_ceph_conf("global", ConfigOpts.debug_rgw, str(config.debug_rgw)) log.info("trying to restart services") srv_restarted = rgw_service.restart() time.sleep(20) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") reusable.delete_bucket(bucket) # disable compression after test if config.test_ops["compression"]["enable"] is True: log.info("disable compression") cmd = "radosgw-admin zone get" out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = ("radosgw-admin zone placement modify --rgw-zone=%s " "--placement-id=default-placement --compression=none" % zone) out = utils.exec_shell_cmd(cmd) srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.gc_verification is True: final_op = reusable.verify_gc() if final_op != -1: test_info.failed_status("test failed") sys.exit(1) # check sync status if a multisite cluster reusable.check_sync_status() # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() write_key_io_info = KeyIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) # create user all_users_info = s3lib.create_users(config.user_count) extra_user = s3lib.create_users(1)[0] extra_user_auth = Auth(extra_user, ssl=config.ssl) extra_user_conn = extra_user_auth.do_auth() for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc) log.info("creating bucket with name: %s" % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = s3lib.resource_op({ "obj": rgw_conn, "resource": "Bucket", "args": [bucket_name_to_create] }) # created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']}) created = s3lib.resource_op({ "obj": bucket, "resource": "create", "args": None, "extra_info": { "access_key": each_user["access_key"] }, }) if created is False: raise TestExecError( "Resource execution failed: bucket creation faield") if created is not None: response = HttpResponseParser(created) if response.status_code == 200: log.info("bucket created") else: raise TestExecError("bucket creation failed") else: raise TestExecError("bucket creation failed") # getting bucket version object if config.test_ops["enable_version"] is True: log.info("bucket versionig test on bucket: %s" % bucket.name) # bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name) bucket_versioning = s3lib.resource_op({ "obj": rgw_conn, "resource": "BucketVersioning", "args": [bucket.name], }) # checking the versioning status # version_status = s3_ops.resource_op(bucket_versioning, 'status') version_status = s3lib.resource_op({ "obj": bucket_versioning, "resource": "status", "args": None }) if version_status is None: log.info("bucket versioning still not enabled") # enabling bucket versioning # version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable') version_enable_status = s3lib.resource_op({ "obj": bucket_versioning, "resource": "enable", "args": None, }) response = HttpResponseParser(version_enable_status) if response.status_code == 200: log.info("version enabled") write_bucket_io_info.add_versioning_status( each_user["access_key"], bucket.name, VERSIONING_STATUS["ENABLED"], ) else: raise TestExecError("version enable failed") if config.objects_count > 0: log.info("s3 objects to create: %s" % config.objects_count) for oc, s3_object_size in list( config.mapped_sizes.items()): # versioning upload s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, str(oc)) s3_object_names.append(s3_object_name) log.info("s3 object name: %s" % s3_object_name) log.info("versioning count: %s" % config.version_count) s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, str(oc)) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator( s3_object_path, s3_object_size) if original_data_info is False: TestExecError("data creation failed") created_versions_count = 0 for vc in range(config.version_count): log.info("version count for %s is %s" % (s3_object_name, str(vc))) log.info("modifying data: %s" % s3_object_name) modified_data_info = manage_data.io_generator( s3_object_path, s3_object_size, op="append", **{ "message": "\nhello for version: %s\n" % str(vc) }) if modified_data_info is False: TestExecError("data modification failed") log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict( { "access_key": each_user["access_key"], "versioning_status": VERSIONING_STATUS["ENABLED"], "version_count_no": vc, }, **modified_data_info) s3_obj = s3lib.resource_op({ "obj": bucket, "resource": "Object", "args": [s3_object_name], "extra_info": upload_info, }) object_uploaded_status = s3lib.resource_op({ "obj": s3_obj, "resource": "upload_file", "args": [modified_data_info["name"]], "extra_info": upload_info, }) if object_uploaded_status is False: raise TestExecError( "Resource execution failed: object upload failed" ) if object_uploaded_status is None: log.info("object uploaded") s3_obj = rgw_conn.Object( bucket.name, s3_object_name) log.info("current_version_id: %s" % s3_obj.version_id) key_version_info = basic_io_structure.version_info( **{ "version_id": s3_obj.version_id, "md5_local": upload_info["md5"], "count_no": vc, "size": upload_info["size"], }) log.info("key_version_info: %s" % key_version_info) write_key_io_info.add_versioning_info( each_user["access_key"], bucket.name, s3_object_path, key_version_info, ) created_versions_count += 1 log.info("created_versions_count: %s" % created_versions_count) log.info("adding metadata") metadata1 = { "m_data1": "this is the meta1 for this obj" } s3_obj.metadata.update(metadata1) metadata2 = { "m_data2": "this is the meta2 for this obj" } s3_obj.metadata.update(metadata2) log.info("metadata for this object: %s" % s3_obj.metadata) log.info("metadata count for object: %s" % (len(s3_obj.metadata))) if not s3_obj.metadata: raise TestExecError( "metadata not created even adding metadata" ) versions = bucket.object_versions.filter( Prefix=s3_object_name) created_versions_count_from_s3 = len( [v.version_id for v in versions]) log.info("created versions count on s3: %s" % created_versions_count_from_s3) if (created_versions_count is created_versions_count_from_s3): log.info( "no new versions are created when added metdata" ) else: raise TestExecError( "version count missmatch, " "possible creation of version on adding metadata" ) s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info("object downloaded") # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path) log.info("downloaded_md5: %s" % s3_object_downloaded_md5) log.info("uploaded_md5: %s" % modified_data_info["md5"]) # tail_op = utils.exec_shell_cmd('tail -l %s' % s3_object_download_path) log.info("all versions for the object: %s\n" % s3_object_name) versions = bucket.object_versions.filter( Prefix=s3_object_name) for version in versions: log.info("key_name: %s --> version_id: %s" % (version.object_key, version.version_id)) if config.test_ops.get("set_acl", None) is True: s3_obj_acl = s3lib.resource_op({ "obj": rgw_conn, "resource": "ObjectAcl", "args": [bucket.name, s3_object_name], }) # setting acl to private, just need to set to any acl and # check if its set - check by response code acls_set_status = s3_obj_acl.put(ACL="private") response = HttpResponseParser(acls_set_status) if response.status_code == 200: log.info("ACLs set") else: raise TestExecError("Acls not Set") # get obj details based on version id for version in versions: log.info("getting info for version id: %s" % version.version_id) obj = s3lib.resource_op({ "obj": rgw_conn, "resource": "Object", "args": [bucket.name, s3_object_name], }) log.info( "obj get detils :%s\n" % (obj.get(VersionId=version.version_id))) if config.test_ops["copy_to_version"] is True: # reverting object to one of the versions ( randomly chosen ) version_id_to_copy = random.choice( [v.version_id for v in versions]) log.info("version_id_to_copy: %s" % version_id_to_copy) s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info("current version_id: %s" % s3_obj.version_id) copy_response = s3_obj.copy_from( CopySource={ "Bucket": bucket.name, "Key": s3_object_name, "VersionId": version_id_to_copy, }) log.info("copy_response: %s" % copy_response) if copy_response is None: raise TestExecError( "copy object from version id failed") # current_version_id = copy_response['VersionID'] log.info("current_version_id: %s" % s3_obj.version_id) # delete the version_id_to_copy object s3_obj.delete(VersionId=version_id_to_copy) log.info( "all versions for the object after the copy operation: %s\n" % s3_object_name) for version in versions: log.info( "key_name: %s --> version_id: %s" % (version.object_key, version.version_id)) # log.info('downloading current s3object: %s' % s3_object_name) # s3_obj.download_file(s3_object_name + ".download") if config.test_ops["delete_object_versions"] is True: log.info("deleting s3_obj keys and its versions") s3_obj = s3lib.resource_op({ "obj": rgw_conn, "resource": "Object", "args": [bucket.name, s3_object_name], }) log.info("deleting versions for s3 obj: %s" % s3_object_name) for version in versions: log.info("trying to delete obj version: %s" % version.version_id) del_obj_version = s3lib.resource_op({ "obj": s3_obj, "resource": "delete", "kwargs": dict(VersionId=version.version_id), }) log.info("response:\n%s" % del_obj_version) if del_obj_version is not None: response = HttpResponseParser( del_obj_version) if response.status_code == 204: log.info("version deleted ") reusable.delete_version_object( bucket, version.version_id, s3_object_path, rgw_conn, each_user, ) else: raise TestExecError( "version deletion failed") else: raise TestExecError( "version deletion failed") log.info("available versions for the object") versions = bucket.object_versions.filter( Prefix=s3_object_name) for version in versions: log.info( "key_name: %s --> version_id: %s" % (version.object_key, version.version_id)) if config.test_ops.get( "delete_from_extra_user") is True: log.info( "trying to delete objects from extra user") s3_obj = s3lib.resource_op({ "obj": extra_user_conn, "resource": "Object", "args": [bucket.name, s3_object_name], }) log.info("deleting versions for s3 obj: %s" % s3_object_name) for version in versions: log.info("trying to delete obj version: %s" % version.version_id) del_obj_version = s3lib.resource_op({ "obj": s3_obj, "resource": "delete", "kwargs": dict(VersionId=version.version_id), }) log.info("response:\n%s" % del_obj_version) if del_obj_version is not False: response = HttpResponseParser( del_obj_version) if response.status_code == 204: log.info("version deleted ") write_key_io_info.delete_version_info( each_user["access_key"], bucket.name, s3_object_path, version.version_id, ) raise TestExecError( "version and deleted, this should not happen" ) else: log.info( "version did not delete, expected behaviour" ) else: log.info( "version did not delete, expected behaviour" ) if config.local_file_delete is True: log.info("deleting local file") utils.exec_shell_cmd("sudo rm -rf %s" % s3_object_path) if config.test_ops["suspend_version"] is True: log.info("suspending versioning") # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') suspend_version_status = s3lib.resource_op({ "obj": bucket_versioning, "resource": "suspend", "args": None }) response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info("versioning suspended") write_bucket_io_info.add_versioning_status( each_user["access_key"], bucket.name, VERSIONING_STATUS["SUSPENDED"], ) else: raise TestExecError("version suspend failed") # getting all objects in the bucket log.info("getting all objects in the bucket") objects = s3lib.resource_op({ "obj": bucket, "resource": "objects", "args": None }) log.info("objects :%s" % objects) all_objects = s3lib.resource_op({ "obj": objects, "resource": "all", "args": None }) log.info("all objects: %s" % all_objects) log.info("all objects2 :%s " % bucket.objects.all()) for obj in all_objects: log.info("object_name: %s" % obj.key) versions = bucket.object_versions.filter( Prefix=obj.key) log.info("displaying all versions of the object") for version in versions: log.info("key_name: %s --> version_id: %s" % (version.object_key, version.version_id)) if config.test_ops.get("suspend_from_extra_user") is True: log.info("suspending versioning from extra user") # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') bucket_versioning = s3lib.resource_op({ "obj": extra_user_conn, "resource": "BucketVersioning", "args": [bucket.name], }) suspend_version_status = s3lib.resource_op({ "obj": bucket_versioning, "resource": "suspend", "args": None }) if suspend_version_status is not False: response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info("versioning suspended") write_bucket_io_info.add_versioning_status( each_user["access_key"], bucket.name, VERSIONING_STATUS["SUSPENDED"], ) raise TestExecError( "version suspended, this should not happen") else: log.info( "versioning not suspended, expected behaviour") if config.test_ops.get("upload_after_suspend") is True: log.info( "trying to upload after suspending versioning on bucket") for oc, s3_object_size in list(config.mapped_sizes.items()): # non versioning upload s3_object_name = s3_object_names[ oc] + ".after_version_suspending" log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) non_version_data_info = manage_data.io_generator( s3_object_path, s3_object_size, op="append", **{"message": "\nhello for non version\n"}) if non_version_data_info is False: TestExecError("data creation failed") log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict( { "access_key": each_user["access_key"], "versioning_status": "suspended", }, **non_version_data_info) s3_obj = s3lib.resource_op({ "obj": bucket, "resource": "Object", "args": [s3_object_name], "extra_info": upload_info, }) object_uploaded_status = s3lib.resource_op({ "obj": s3_obj, "resource": "upload_file", "args": [non_version_data_info["name"]], "extra_info": upload_info, }) if object_uploaded_status is False: raise TestExecError( "Resource execution failed: object upload failed") if object_uploaded_status is None: log.info("object uploaded") s3_obj = s3lib.resource_op({ "obj": rgw_conn, "resource": "Object", "args": [bucket.name, s3_object_name], }) log.info("version_id: %s" % s3_obj.version_id) if s3_obj.version_id is None: log.info("Versions are not created after suspending") else: raise TestExecError( "Versions are created even after suspending") s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info("object downloaded") # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path) log.info("s3_object_downloaded_md5: %s" % s3_object_downloaded_md5) log.info("s3_object_uploaded_md5: %s" % non_version_data_info["md5"]) if config.local_file_delete is True: utils.exec_shell_cmd("sudo rm -rf %s" % s3_object_path) if config.test_ops.get("delete_bucket") is True: reusable.delete_bucket(bucket) # check sync status if a multisite cluster reusable.check_sync_status() # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get('encryption_algorithm', None) is not None: log.info('encryption enabled, making ceph config changes') ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{'signature_version': 's3v4'}) else: rgw_conn = auth.do_auth() # enabling sharding if config.test_ops['sharding']['enable'] is True: log.info('enabling sharding on buckets') max_shards = config.test_ops['sharding']['max_shards'] log.info('making changes to ceph.conf') ceph_conf.set_to_ceph_conf( 'global', ConfigOpts.rgw_override_bucket_index_max_shards, str(max_shards)) log.info('trying to restart services ') srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') if config.test_ops['compression']['enable'] is True: compression_type = config.test_ops['compression']['type'] log.info('enabling compression') cmd = 'radosgw-admin zone get' out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = 'radosgw-admin zone placement modify --rgw-zone=%s ' \ '--placement-id=default-placement --compression=%s' % (zone,compression_type) out = utils.exec_shell_cmd(cmd) try: data = json.loads(out) if data['placement_pools'][0]['val']['storage_classes'][ 'STANDARD']['compression_type'] == compression_type: log.info('Compression enabled successfully') else: raise ValueError('failed to enable compression') except ValueError as e: exit(str(e)) log.info('trying to restart rgw services ') srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') # create buckets if config.test_ops['create_bucket'] is True: log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) bucket = resuables.create_bucket(bucket_name_to_create, rgw_conn, each_user) if config.test_ops['create_object'] is True: # uploading data log.info('s3 objects to create: %s' % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) if config.test_ops.get('upload_type') == 'multipart': log.info('upload type: multipart') resuables.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user) else: log.info('upload type: normal') resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user) if config.test_ops['download_object'] is True: log.info('trying to download object: %s' % s3_object_name) s3_object_download_name = s3_object_name + "." + "download" s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_download_name) log.info('s3_object_download_path: %s' % s3_object_download_path) log.info('downloading to filename: %s' % s3_object_download_name) if config.test_ops.get('encryption_algorithm', None) is not None: log.info('encryption download') log.info( 'encryption algorithm: %s' % config.test_ops['encryption_algorithm']) object_downloaded_status = bucket.download_file( s3_object_name, s3_object_download_path, ExtraArgs={ 'SSECustomerKey': encryption_key, 'SSECustomerAlgorithm': config.test_ops['encryption_algorithm'] }) else: object_downloaded_status = s3lib.resource_op({ 'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info('object downloaded') s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path) s3_object_uploaded_md5 = utils.get_md5( s3_object_path) log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5) log.info('s3_object_uploaded_md5: %s' % s3_object_uploaded_md5) if str(s3_object_uploaded_md5) == str( s3_object_downloaded_md5): log.info('md5 match') utils.exec_shell_cmd('rm -rf %s' % s3_object_download_path) else: raise TestExecError('md5 mismatch') if config.local_file_delete is True: log.info( 'deleting local file created after the upload') utils.exec_shell_cmd('rm -rf %s' % s3_object_path) # verification of shards after upload if config.test_ops['sharding']['enable'] is True: cmd = 'radosgw-admin metadata get bucket:%s | grep bucket_id' % bucket.name out = utils.exec_shell_cmd(cmd) b_id = out.replace( '"', '').strip().split(":")[1].strip().replace(',', '') cmd2 = 'rados -p default.rgw.buckets.index ls | grep %s' % b_id out = utils.exec_shell_cmd(cmd2) log.info( 'got output from sharing verification.--------') # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops['compression']['enable'] is True: cmd = 'radosgw-admin bucket stats --bucket=%s' % bucket.name out = utils.exec_shell_cmd(cmd) # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops['compression']['enable'] is True: cmd = 'radosgw-admin bucket stats --bucket=%s' % bucket.name out = utils.exec_shell_cmd(cmd) if config.test_ops['delete_bucket_object'] is True: log.info('listing all objects in bucket: %s' % bucket.name) objects = s3lib.resource_op({ 'obj': bucket, 'resource': 'objects', 'args': None }) log.info('objects :%s' % objects) all_objects = s3lib.resource_op({ 'obj': objects, 'resource': 'all', 'args': None }) log.info('all objects: %s' % all_objects) for obj in all_objects: log.info('object_name: %s' % obj.key) log.info('deleting all objects in bucket') objects_deleted = s3lib.resource_op({ 'obj': objects, 'resource': 'delete', 'args': None }) log.info('objects_deleted: %s' % objects_deleted) if objects_deleted is False: raise TestExecError( 'Resource execution failed: Object deletion failed' ) if objects_deleted is not None: response = HttpResponseParser(objects_deleted[0]) if response.status_code == 200: log.info('objects deleted ') else: raise TestExecError("objects deletion failed") else: raise TestExecError("objects deletion failed") log.info('deleting bucket: %s' % bucket.name) # bucket_deleted_status = s3_ops.resource_op(bucket, 'delete') bucket_deleted_status = s3lib.resource_op({ 'obj': bucket, 'resource': 'delete', 'args': None }) log.info('bucket_deleted_status: %s' % bucket_deleted_status) if bucket_deleted_status is not None: response = HttpResponseParser( bucket_deleted_status) if response.status_code == 204: log.info('bucket deleted ') else: raise TestExecError("bucket deletion failed") else: raise TestExecError("bucket deletion failed") # disable compression after test if config.test_ops['compression']['enable'] is True: log.info('disable compression') cmd = 'radosgw-admin zone get' out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = 'radosgw-admin zone placement modify --rgw-zone=%s ' \ '--placement-id=default-placement --compression=none' % zone out = utils.exec_shell_cmd(cmd) srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted')
def test_exec(config): test_info = AddTestInfo('test versioning with objects') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # create user all_users_info = s3lib.create_users(config.user_count, config.cluster_name) for each_user in all_users_info: # authenticate auth = Auth(each_user) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Bucket', 'args': [bucket_name_to_create]}) # created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']}) created = s3lib.resource_op({'obj': bucket, 'resource': 'create', 'args': None, 'extra_info': {'access_key': each_user['access_key']}}) if created is False: raise TestExecError("Resource execution failed: bucket creation faield") if created is not None: response = HttpResponseParser(created) if response.status_code == 200: log.info('bucket created') else: raise TestExecError("bucket creation failed") else: raise TestExecError("bucket creation failed") # getting bucket version object if config.test_ops['enable_version'] is True: log.info('bucket versionig test on bucket: %s' % bucket.name) # bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name) bucket_versioning = s3lib.resource_op({'obj': rgw_conn, 'resource': 'BucketVersioning', 'args': [bucket.name]}) # checking the versioning status # version_status = s3_ops.resource_op(bucket_versioning, 'status') version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'status', 'args': None }) if version_status is None: log.info('bucket versioning still not enabled') # enabling bucket versioning # version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable') version_enable_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'enable', 'args': None}) response = HttpResponseParser(version_enable_status) if response.status_code == 200: log.info('version enabled') else: raise TestExecError("version enable failed") if config.objects_count > 0: log.info('s3 objects to create: %s' % config.objects_count) for oc in range(config.objects_count): # versioning upload s3_object_name = utils.gen_s3_object_name(bucket_name_to_create,str(oc)) s3_object_names.append(s3_object_name) log.info('s3 object name: %s' % s3_object_name) log.info('versioning count: %s' % config.version_count) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc)) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, s3_object_size) if original_data_info is False: TestExecError("data creation failed") for vc in range(config.version_count): log.info('version count for %s is %s' % (s3_object_name, str(vc))) log.info('modifying data: %s' % s3_object_name) modified_data_info = manage_data.io_generator(s3_object_path, s3_object_size, data='append', **{'message': '\nhello object for version: %s\n' % str(vc)}) if modified_data_info is False: TestExecError("data modification failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key']}, **modified_data_info) object_uploaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'upload_file', 'args': [modified_data_info['name'], s3_object_name], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') log.info('all versions for the object: %s\n' % s3_object_name) versions = bucket.object_versions.filter(Prefix=s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' %(version.object_key, version.version_id)) if config.test_ops['copy_to_version'] is True: # reverting object to one of the versions ( randomly chosen ) version_id_to_copy = random.choice([v.version_id for v in versions]) log.info('version_id_to_copy: %s' % version_id_to_copy) s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current version_id: %s' % s3_obj.version_id) copy_response = s3_obj.copy_from(CopySource={'Bucket': bucket.name, 'Key': s3_object_name, 'VersionId': version_id_to_copy}) log.info('copy_response: %s' % copy_response) if copy_response is None: raise TestExecError("copy object from version id failed") # current_version_id = copy_response['VersionID'] log.info('current_version_id: %s' % s3_obj.version_id ) # delete the version_id_to_copy object s3_obj.delete(VersionId=version_id_to_copy) log.info('all versions for the object after the copy operation: %s\n' % s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) # log.info('downloading current s3object: %s' % s3_object_name) # s3_obj.download_file(s3_object_name + ".download") if config.test_ops['delete_object_versions'] is True: log.info('deleting s3_obj keys and its versions') s3_obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('deleting versions for s3 obj: %s' % s3_object_name) for version in versions: log.info('trying to delete obj version: %s' % version.version_id) del_obj_version = s3lib.resource_op({'obj': s3_obj, 'resource': 'delete', 'kwargs': dict(VersionId=version.version_id)}) log.info('response:\n%s' % del_obj_version) if del_obj_version is not None: response = HttpResponseParser(del_obj_version) if response.status_code == 204: log.info('version deleted ') else: raise TestExecError("version deletion failed") else: raise TestExecError("version deletion failed") if config.test_ops['suspend_version'] is True: # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') suspend_version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'suspend', 'args': None}) response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info('versioning suspended') else: raise TestExecError("version suspend failed") if config.test_ops['upload_after_suspend'] is True: log.info('trying to upload after suspending versioning on bucket') for s3_object_name in s3_object_names: # non versioning upload log.info('s3 object name: %s' % s3_object_name) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) non_version_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op="append", **{'message': '\nhello object for non version\n'}) if non_version_data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key']}, **non_version_data_info) object_uploaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'upload_file', 'args': [non_version_data_info['name'], s3_object_name], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name+".download") object_downloaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError("Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5) log.info('s3_object_uploaded_md5: %s' % non_version_data_info['md5']) test_info.success_status('test passed') sys.exit(0) except Exception,e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def upload_version_object(config, user_info, rgw_conn, s3_object_name, object_size, bucket, TEST_DATA_PATH): # versioning upload log.info("versioning count: %s" % config.version_count) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, object_size) if original_data_info is False: TestExecError("data creation failed") created_versions_count = 0 for vc in range(config.version_count): log.info("version count for %s is %s" % (s3_object_name, str(vc))) log.info("modifying data: %s" % s3_object_name) modified_data_info = manage_data.io_generator( s3_object_path, object_size, op="append", **{"message": "\nhello for version: %s\n" % str(vc)}, ) if modified_data_info is False: TestExecError("data modification failed") log.info("uploading s3 object: %s" % s3_object_path) upload_info = dict( { "access_key": user_info["access_key"], "versioning_status": "enabled", "version_count_no": vc, }, **modified_data_info, ) s3_obj = s3lib.resource_op({ "obj": bucket, "resource": "Object", "args": [s3_object_name], "extra_info": upload_info, }) object_uploaded_status = s3lib.resource_op({ "obj": s3_obj, "resource": "upload_file", "args": [modified_data_info["name"]], "extra_info": upload_info, }) if object_uploaded_status is False: raise TestExecError( "Resource execution failed: object upload failed") if object_uploaded_status is None: log.info("object uploaded") s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info("current_version_id: %s" % s3_obj.version_id) key_version_info = basic_io_structure.version_info( **{ "version_id": s3_obj.version_id, "md5_local": upload_info["md5"], "count_no": vc, "size": upload_info["size"], }) log.info("key_version_info: %s" % key_version_info) write_key_io_info.add_versioning_info(user_info["access_key"], bucket.name, s3_object_path, key_version_info) created_versions_count += 1 log.info("created_versions_count: %s" % created_versions_count) log.info("adding metadata") metadata1 = {"m_data1": "this is the meta1 for this obj"} s3_obj.metadata.update(metadata1) metadata2 = {"m_data2": "this is the meta2 for this obj"} s3_obj.metadata.update(metadata2) log.info("metadata for this object: %s" % s3_obj.metadata) log.info("metadata count for object: %s" % (len(s3_obj.metadata))) if not s3_obj.metadata: raise TestExecError( "metadata not created even adding metadata") versions = bucket.object_versions.filter(Prefix=s3_object_name) created_versions_count_from_s3 = len( [v.version_id for v in versions]) log.info("created versions count on s3: %s" % created_versions_count_from_s3) if created_versions_count is created_versions_count_from_s3: log.info("no new versions are created when added metadata") else: raise TestExecError( "version count mismatch, " "possible creation of version on adding metadata") s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed") if object_downloaded_status is None: log.info("object downloaded") # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info("downloaded_md5: %s" % s3_object_downloaded_md5) log.info("uploaded_md5: %s" % modified_data_info["md5"]) log.info("deleting downloaded version file") utils.exec_shell_cmd("sudo rm -rf %s" % s3_object_download_path) log.info("all versions for the object: %s\n" % s3_object_name)
def upload_version_object(config, user_info, rgw_conn, s3_object_name, object_size, bucket, TEST_DATA_PATH): # versioning upload log.info('versioning count: %s' % config.version_count) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, object_size) if original_data_info is False: TestExecError("data creation failed") created_versions_count = 0 for vc in range(config.version_count): log.info('version count for %s is %s' % (s3_object_name, str(vc))) log.info('modifying data: %s' % s3_object_name) modified_data_info = manage_data.io_generator( s3_object_path, object_size, op='append', **{'message': '\nhello for version: %s\n' % str(vc)}) if modified_data_info is False: TestExecError("data modification failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict( { 'access_key': user_info['access_key'], 'versioning_status': 'enabled', 'version_count_no': vc }, **modified_data_info) s3_obj = s3lib.resource_op({ 'obj': bucket, 'resource': 'Object', 'args': [s3_object_name], 'extra_info': upload_info, }) object_uploaded_status = s3lib.resource_op({ 'obj': s3_obj, 'resource': 'upload_file', 'args': [modified_data_info['name']], 'extra_info': upload_info }) if object_uploaded_status is False: raise TestExecError( "Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current_version_id: %s' % s3_obj.version_id) key_version_info = basic_io_structure.version_info( **{ 'version_id': s3_obj.version_id, 'md5_local': upload_info['md5'], 'count_no': vc, 'size': upload_info['size'] }) log.info('key_version_info: %s' % key_version_info) write_key_io_info.add_versioning_info(user_info['access_key'], bucket.name, s3_object_path, key_version_info) created_versions_count += 1 log.info('created_versions_count: %s' % created_versions_count) log.info('adding metadata') metadata1 = {"m_data1": "this is the meta1 for this obj"} s3_obj.metadata.update(metadata1) metadata2 = {"m_data2": "this is the meta2 for this obj"} s3_obj.metadata.update(metadata2) log.info('metadata for this object: %s' % s3_obj.metadata) log.info('metadata count for object: %s' % (len(s3_obj.metadata))) if not s3_obj.metadata: raise TestExecError( 'metadata not created even adding metadata') versions = bucket.object_versions.filter(Prefix=s3_object_name) created_versions_count_from_s3 = len( [v.version_id for v in versions]) log.info('created versions count on s3: %s' % created_versions_count_from_s3) if created_versions_count is created_versions_count_from_s3: log.info('no new versions are created when added metdata') else: raise TestExecError( "version count missmatch, " "possible creation of version on adding metadata") s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({ 'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('downloaded_md5: %s' % s3_object_downloaded_md5) log.info('uploaded_md5: %s' % modified_data_info['md5']) log.info('deleting downloaded version file') utils.exec_shell_cmd('sudo rm -rf %s' % s3_object_download_path) log.info('all versions for the object: %s\n' % s3_object_name)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) umgmt = UserMgmt() ceph_conf = CephConfOp() log.info(type(ceph_conf)) rgw_service = RGWService() # preparing data user_names = ["tuffy", "scooby", "max"] tenant = "tenant" tenant_user_info = umgmt.create_tenant_user(tenant_name=tenant, user_id=user_names[0], displayname=user_names[0]) user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0]) auth = Auth(user_info) rgw = auth.do_auth() for cc in range(config.container_count): if config.version_enable is True: log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_swift_versioning_enabled, "True") log.info("trying to restart services ") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") container_name_old = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=str(cc) + "old") log.info(container_name_old) container = swiftlib.resource_op({ "obj": rgw, "resource": "put_container", "kwargs": dict(container=container_name_old), }) container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=str(cc) + "new") log.info(container_name) container = swiftlib.resource_op({ "obj": rgw, "resource": "put_container", "args": [ container_name, { "X-Versions-Location": container_name_old }, ], }) if container is False: raise TestExecError( "Resource execution failed: container creation failed") ls = [] swift_object_name = "" for version_count in range(config.version_count): for oc, size in list(config.mapped_sizes.items()): swift_object_name = fill_container(rgw, container_name, user_names[0], oc, cc, size) ls = rgw.get_container(container_name_old) ls = list(ls) if config.copy_version_object is True: old_obj_name = ls[1][config.version_count - 2]["name"] log.info(old_obj_name) container = swiftlib.resource_op({ "obj": rgw, "resource": "copy_object", "kwargs": dict( container=container_name_old, obj=old_obj_name, destination=container_name + "/" + swift_object_name, ), }) if container is False: raise TestExecError("Resource execution failed") log.info("Successfully copied item") else: current_count = "radosgw-admin bucket stats --uid={uid} --tenant={tenant} --bucket='{bucket}' ".format( uid=user_names[0], tenant=tenant, bucket=container_name) num_obj_current = utils.exec_shell_cmd(current_count) num_obj_current = json.loads(num_obj_current) num_obj_current = (num_obj_current[0].get("usage").get( "rgw.main").get("num_objects")) old_count = "radosgw-admin bucket stats --uid={uid} --tenant={tenant} --bucket='{bucket}' ".format( uid=user_names[0], tenant=tenant, bucket=container_name_old) num_obj_old = utils.exec_shell_cmd(old_count) num_obj_old = json.loads(num_obj_old) num_obj_old = (num_obj_old[0].get("usage").get("rgw.main").get( "num_objects")) version_count_from_config = ( config.objects_count * config.version_count) - config.objects_count if (num_obj_current == config.objects_count) and ( num_obj_old == version_count_from_config): log.info("objects and versioned obbjects are correct") else: test_info.failed_status("test failed") elif config.object_expire is True: container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc) container = swiftlib.resource_op({ "obj": rgw, "resource": "put_container", "args": [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation failed") for oc, size in list(config.mapped_sizes.items()): swift_object_name = fill_container( rgw, container_name, user_names[0], oc, cc, size, header={"X-Delete-After": 5}, ) time.sleep(7) container_exists = swiftlib.resource_op({ "obj": rgw, "resource": "get_object", "args": [container_name, swift_object_name], }) log.info(container_exists) if container_exists: msg = "test failed as the objects are still present" test_info.failed_status(msg) raise TestExecError(msg) elif config.large_object_upload is True: container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc) container = swiftlib.resource_op({ "obj": rgw, "resource": "put_container", "args": [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation failed") for oc, size in list(config.mapped_sizes.items()): swift_object_name = fill_container( rgw, container_name, user_names[0], oc, cc, size, multipart=True, split_size=config.split_size, ) container_name_new = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=str(cc) + "New") container = swiftlib.resource_op({ "obj": rgw, "resource": "put_container", "kwargs": dict(container=container_name_new), }) if container is False: raise TestExecError( "Resource execution failed: container creation failed") container = swiftlib.resource_op({ "obj": rgw, "resource": "put_object", "kwargs": dict( container=container_name_new, obj=swift_object_name, contents=None, headers={ "X-Object-Manifest": container_name + "/" + swift_object_name + "/" }, ), }) if container is False: raise TestExecError( "Resource execution failed: container creation failed") if config.large_object_download is True: swift_old_object_path = os.path.join( TEST_DATA_PATH, swift_object_name) swift_object_download_fname = swift_object_name + ".download" log.info("download object name: %s" % swift_object_download_fname) swift_object_download_path = os.path.join( TEST_DATA_PATH, swift_object_download_fname) log.info("download object path: %s" % swift_object_download_path) swift_object_downloaded = rgw.get_object( container_name_new, swift_object_name) with open(swift_object_download_path, "wb") as fp: fp.write(swift_object_downloaded[1]) old_object = utils.get_md5(swift_old_object_path) downloaded_obj = utils.get_md5(swift_object_download_path) log.info("s3_object_downloaded_md5: %s" % old_object) log.info("s3_object_uploaded_md5: %s" % downloaded_obj) if str(old_object) == str(downloaded_obj): log.info("md5 match") utils.exec_shell_cmd("rm -rf %s" % swift_object_download_path) else: raise TestExecError("md5 mismatch") else: container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc) container = swiftlib.resource_op({ "obj": rgw, "resource": "put_container", "args": [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation failed") for oc, size in list(config.mapped_sizes.items()): swift_object_name = fill_container(rgw, container_name, user_names[0], oc, cc, size) # download object swift_object_download_fname = swift_object_name + ".download" log.info("download object name: %s" % swift_object_download_fname) swift_object_download_path = os.path.join( TEST_DATA_PATH, swift_object_download_fname) log.info("download object path: %s" % swift_object_download_path) swift_object_downloaded = rgw.get_object( container_name, swift_object_name) with open(swift_object_download_path, "w") as fp: fp.write(str(swift_object_downloaded[1])) # modify and re-upload log.info("appending new message to test_data") message_to_append = "adding new msg after download" fp = open(swift_object_download_path, "a+") fp.write(message_to_append) fp.close() with open(swift_object_download_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", ) # delete object log.info("deleting swift object") rgw.delete_object(container_name, swift_object_name) # delete container log.info("deleting swift container") rgw.delete_container(container_name) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!") reusable.remove_user(tenant_user_info, tenant=tenant)