def test_exec(config): test_info = AddTestInfo('Bucket Request Payer') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # create user all_users_info = s3lib.create_users(config.user_count, config.cluster_name) for each_user in all_users_info: # authenticate auth = Auth(each_user) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = resuables.create_bucket( bucket_name=bucket_name_to_create, rgw=rgw_conn, user_info=each_user) bucket_request_payer = s3lib.resource_op({ 'obj': rgw_conn, 'resource': 'BucketRequestPayment', 'args': [bucket.name] }) # change the bucket request payer to 'requester' payer = {'Payer': 'Requester'} response = s3lib.resource_op({ 'obj': bucket_request_payer, 'resource': 'put', 'kwargs': dict(RequestPaymentConfiguration=payer) }) log.info(response) test_info.success_status('test passed') sys.exit(0) except Exception, e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() config.rgw_lc_debug_interval = 30 config.rgw_lc_max_worker = 10 log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_lc_debug_interval, str(config.rgw_lc_debug_interval)) ceph_version = utils.exec_shell_cmd("ceph version") op = ceph_version.split() if "pacific" in op: commands = [ f"ceph config set client.rgw rgw_lc_max_worker {config.rgw_lc_max_worker}", "ceph config set client.rgw rgw_lc_debug_interval 30", ] for command in commands: utils.exec_shell_cmd(command) if "nautilus" in op: ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_lc_max_worker, str(config.rgw_lc_max_worker)) log.info("trying to restart services") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") config.user_count = 1 config.bucket_count = 1 # create user user_info = s3lib.create_users(config.user_count) user_info = user_info[0] auth = Auth(user_info, ssl=config.ssl) rgw_conn = auth.do_auth() rgw_conn2 = auth.do_auth_using_client() log.info("no of buckets to create: %s" % config.bucket_count) bucket_name = utils.gen_bucket_name_from_userid(user_info["user_id"], rand_no=1) obj_list = [] obj_tag = "suffix1=WMV1" bucket = reusable.create_bucket(bucket_name, rgw_conn, user_info) prefix = list( map( lambda x: x, [ rule["Filter"].get("Prefix") or rule["Filter"]["And"].get("Prefix") for rule in config.lifecycle_conf ], )) prefix = prefix if prefix else ["dummy1"] if config.test_ops["enable_versioning"] is True: reusable.enable_versioning(bucket, rgw_conn, user_info, write_bucket_io_info) if config.test_ops["create_object"] is True: for oc, size in list(config.mapped_sizes.items()): config.obj_size = size key = prefix.pop() prefix.insert(0, key) s3_object_name = key + "." + bucket.name + "." + str(oc) obj_list.append(s3_object_name) if config.test_ops["version_count"] > 0: for vc in range(config.test_ops["version_count"]): log.info("version count for %s is %s" % (s3_object_name, str(vc))) log.info("modifying data: %s" % s3_object_name) reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, user_info, append_data=True, append_msg="hello object for version: %s\n" % str(vc), ) else: log.info("s3 objects to create: %s" % config.objects_count) reusable.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, user_info) life_cycle_rule = {"Rules": config.lifecycle_conf} reusable.put_get_bucket_lifecycle_test(bucket, rgw_conn, rgw_conn2, life_cycle_rule, config) lc_ops.validate_prefix_rule(bucket, config) if config.test_ops["delete_marker"] is True: life_cycle_rule_new = {"Rules": config.delete_marker_ops} reusable.put_get_bucket_lifecycle_test(bucket, rgw_conn, rgw_conn2, life_cycle_rule_new, config) if config.test_ops["enable_versioning"] is False: if config.test_ops["create_object"] is True: for oc, size in list(config.mapped_sizes.items()): config.obj_size = size key = prefix.pop() prefix.insert(0, key) s3_object_name = key + "." + bucket.name + "." + str(oc) obj_list.append(s3_object_name) reusable.upload_object_with_tagging(s3_object_name, bucket, TEST_DATA_PATH, config, user_info, obj_tag) life_cycle_rule = {"Rules": config.lifecycle_conf} reusable.put_get_bucket_lifecycle_test(bucket, rgw_conn, rgw_conn2, life_cycle_rule, config) lc_ops.validate_and_rule(bucket, config) reusable.remove_user(user_info) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user if config.dbr_scenario == "brownfield": user_brownfiled = "brownfield_user" all_users_info = s3lib.create_users(config.user_count, user_brownfiled) else: all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get("encryption_algorithm", None) is not None: log.info("encryption enabled, making ceph config changes") ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{"signature_version": "s3v4"}) else: rgw_conn = auth.do_auth() # enabling sharding if config.test_ops["sharding"]["enable"] is True: log.info("enabling sharding on buckets") max_shards = config.test_ops["sharding"]["max_shards"] log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_override_bucket_index_max_shards, str(max_shards), ) log.info("trying to restart services ") srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.test_ops["compression"]["enable"] is True: compression_type = config.test_ops["compression"]["type"] log.info("enabling compression") cmd = "radosgw-admin zone get" out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = ("radosgw-admin zone placement modify --rgw-zone=%s " "--placement-id=default-placement --compression=%s" % (zone, compression_type)) out = utils.exec_shell_cmd(cmd) ceph_version = utils.exec_shell_cmd("ceph version").split()[4] try: data = json.loads(out) if ceph_version == "luminous": if (data["placement_pools"][0]["val"]["compression"] == compression_type): log.info("Compression enabled successfully") else: if ceph_version in ["nautilus", "octopus"]: if (data["placement_pools"][0]["val"] ["storage_classes"]["STANDARD"]["compression_type"] == compression_type): log.info("Compression enabled successfully") except ValueError as e: exit(str(e)) log.info("trying to restart rgw services ") srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.gc_verification is True: conf = config.ceph_conf reusable.set_gc_conf(ceph_conf, conf) if config.dynamic_resharding is True: if utils.check_dbr_support(): log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_max_objs_per_shard, str(config.max_objects_per_shard), ) srv_restarted = rgw_service.restart() # create buckets if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc) if config.bucket_sync_crash is True: is_primary = utils.is_cluster_primary() if is_primary: bucket_name_to_create = "bkt-crash-check" if config.dbr_scenario == "brownfield": bucket_name_to_create = ("brownfield-dynamic-bkt" if config.dynamic_resharding else "brownfield-manual-bkt") log.info("creating bucket with name: %s" % bucket_name_to_create) bucket = reusable.create_bucket(bucket_name_to_create, rgw_conn, each_user) if config.dynamic_resharding is True: reusable.check_sync_status() op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}") json_doc = json.loads(op) old_num_shards = json_doc["num_shards"] log.info(f"no_of_shards_created: {old_num_shards}") if config.test_ops["create_object"] is True: # uploading data log.info("s3 objects to create: %s" % config.objects_count) if utils.check_dbr_support(): if bucket_name_to_create in [ "brownfield-dynamic-bkt", "brownfield-manual-bkt", ]: op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) if bool(json_doc["usage"]): num_object = json_doc["usage"]["rgw.main"][ "num_objects"] config.objects_count = (num_object * 2 + config.objects_count) config.mapped_sizes = utils.make_mapped_sizes( config) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) if config.test_ops["download_object"] is True: log.info("trying to download object: %s" % s3_object_name) s3_object_download_name = s3_object_name + "." + "download" s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_download_name) log.info("s3_object_download_path: %s" % s3_object_download_path) log.info("downloading to filename: %s" % s3_object_download_name) if (config.test_ops.get("encryption_algorithm", None) is not None): log.info("encryption download") log.info( "encryption algorithm: %s" % config.test_ops["encryption_algorithm"]) object_downloaded_status = bucket.download_file( s3_object_name, s3_object_download_path, ExtraArgs={ "SSECustomerKey": encryption_key, "SSECustomerAlgorithm": config. test_ops["encryption_algorithm"], }, ) else: object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [ s3_object_name, s3_object_download_path, ], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info("object downloaded") s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path) s3_object_uploaded_md5 = utils.get_md5( s3_object_path) log.info("s3_object_downloaded_md5: %s" % s3_object_downloaded_md5) log.info("s3_object_uploaded_md5: %s" % s3_object_uploaded_md5) if str(s3_object_uploaded_md5) == str( s3_object_downloaded_md5): log.info("md5 match") utils.exec_shell_cmd("rm -rf %s" % s3_object_download_path) else: raise TestExecError("md5 mismatch") if config.local_file_delete is True: log.info( "deleting local file created after the upload") utils.exec_shell_cmd("rm -rf %s" % s3_object_path) if config.bucket_sync_crash is True: is_primary = utils.is_cluster_primary() if is_primary is False: crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!") realm, source_zone = utils.get_realm_source_zone_info( ) log.info(f"Realm name: {realm}") log.info(f"Source zone name: {source_zone}") for i in range( 600): # Running sync command for 600 times op = utils.exec_shell_cmd( f"radosgw-admin bucket sync run --bucket bkt-crash-check --rgw-curl-low-speed-time=0 --source-zone {source_zone} --rgw-realm {realm}" ) crash_info = reusable.check_for_crash() if crash_info: raise TestExecError( "ceph daemon crash found!") time.sleep(1) if config.dynamic_resharding is True: if utils.check_dbr_support(): reusable.check_sync_status() for i in range(10): time.sleep( 60 ) # Adding delay for processing reshard list op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) new_num_shards = json_doc["num_shards"] log.info( f"no_of_shards_created: {new_num_shards}") if new_num_shards > old_num_shards: break else: raise TestExecError( "num shards are same after processing resharding" ) if config.manual_resharding is True: if utils.check_dbr_support(): op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) old_num_shards = json_doc["num_shards"] log.info(f"no_of_shards_created: {old_num_shards}") op = utils.exec_shell_cmd( f"radosgw-admin reshard add --bucket {bucket.name} --num-shards {config.shards}" ) op = utils.exec_shell_cmd( "radosgw-admin reshard process") time.sleep(60) op = utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket {bucket.name}" ) json_doc = json.loads(op) new_num_shards = json_doc["num_shards"] log.info(f"no_of_shards_created: {new_num_shards}") if new_num_shards <= old_num_shards: raise TestExecError( "num shards are same after processing resharding" ) # verification of shards after upload if config.test_datalog_trim_command is True: shard_id, end_marker = reusable.get_datalog_marker() cmd = f"sudo radosgw-admin datalog trim --shard-id {shard_id} --end-marker {end_marker} --debug_ms=1 --debug_rgw=20" out, err = utils.exec_shell_cmd(cmd, debug_info=True) if "Segmentation fault" in err: raise TestExecError("Segmentation fault occured") if config.test_ops["sharding"]["enable"] is True: cmd = ( "radosgw-admin metadata get bucket:%s | grep bucket_id" % bucket.name) out = utils.exec_shell_cmd(cmd) b_id = (out.replace( '"', "").strip().split(":")[1].strip().replace(",", "")) cmd2 = "rados -p default.rgw.buckets.index ls | grep %s" % b_id out = utils.exec_shell_cmd(cmd2) log.info( "got output from sharing verification.--------") # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops["compression"]["enable"] is True: cmd = "radosgw-admin bucket stats --bucket=%s" % bucket.name out = utils.exec_shell_cmd(cmd) # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops["compression"]["enable"] is True: cmd = "radosgw-admin bucket stats --bucket=%s" % bucket.name out = utils.exec_shell_cmd(cmd) if config.test_ops["delete_bucket_object"] is True: reusable.delete_objects(bucket) time.sleep(10) reusable.check_sync_status() reusable.delete_bucket(bucket) # disable compression after test if config.test_ops["compression"]["enable"] is True: log.info("disable compression") cmd = "radosgw-admin zone get" out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = ("radosgw-admin zone placement modify --rgw-zone=%s " "--placement-id=default-placement --compression=none" % zone) out = utils.exec_shell_cmd(cmd) srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.gc_verification is True: final_op = reusable.verify_gc() if final_op != -1: test_info.failed_status("test failed") sys.exit(1) # check sync status if a multisite cluster reusable.check_sync_status() # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): test_info = AddTestInfo('RGW Dynamic Resharding test') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() try: test_info.started_info() log.info('starting IO') config.max_objects_per_shard = 10 config.no_of_shards = 10 config.user_count = 1 user_info = s3lib.create_users(config.user_count) user_info = user_info[0] auth = Auth(user_info) rgw_conn = auth.do_auth() config.bucket_count = 1 log.info('no of buckets to create: %s' % config.bucket_count) bucket_name = utils.gen_bucket_name_from_userid(user_info['user_id'], rand_no=1) bucket = create_bucket_with_versioning(rgw_conn, user_info, bucket_name) upload_objects(user_info, bucket, config) log.info('sharding configuration will be added now.') if config.sharding_type == 'online': log.info('sharding type is online') # for online, # the number of shards should be greater than [ (no of objects)/(max objects per shard) ] # example: objects = 500 ; max object per shard = 10 # then no of shards should be at least 50 or more time.sleep(15) log.info('making changes to ceph.conf') ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_max_objs_per_shard, config.max_objects_per_shard) ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_dynamic_resharding, True) num_shards_expected = config.objects_count / config.max_objects_per_shard log.info('num_shards_expected: %s' % num_shards_expected) log.info('trying to restart services ') srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') if config.sharding_type == 'offline': log.info('sharding type is offline') # for offline. # the number of shards will be the value set in the command. time.sleep(15) log.info('in offline sharding') cmd_exec = utils.exec_shell_cmd('radosgw-admin bucket reshard --bucket=%s --num-shards=%s' % (bucket.name, config.no_of_shards)) if cmd_exec is False: raise TestExecError("offline resharding command execution failed") # upload_objects(user_info, bucket, config) log.info('s3 objects to create: %s' % config.objects_count) for oc in range(config.objects_count): s3_object_name = utils.gen_s3_object_name(bucket.name, config.objects_count + oc) resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, user_info) time.sleep(300) log.info('verification starts') op = utils.exec_shell_cmd("radosgw-admin metadata get bucket:%s" % bucket.name) json_doc = json.loads(op) bucket_id = json_doc['data']['bucket']['bucket_id'] op2 = utils.exec_shell_cmd("radosgw-admin metadata get bucket.instance:%s:%s" % (bucket.name, bucket_id)) json_doc2 = json.loads((op2)) num_shards_created = json_doc2['data']['bucket_info']['num_shards'] log.info('no_of_shards_created: %s' % num_shards_created) log.info('no_of_shards_expected: %s' % num_shards_expected) if config.sharding_type == 'offline': if num_shards_expected != num_shards_created: raise TestExecError("expected number of shards not created") log.info('Expected number of shards created') if config.sharding_type == 'online': log.info('for online, ' 'number of shards created should be greater than or equal to number of expected shards') if int(num_shards_created) >= int(num_shards_expected): log.info('Expected number of shards created') else: raise TestExecError('Expected number of shards not created') read_io = ReadIOInfo() read_io.yaml_fname = 'io_info.yaml' read_io.verify_io() test_info.success_status('test passed') sys.exit(0) except Exception as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1) except TestExecError as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): test_info = AddTestInfo('test with acls') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() s3_ops = ResourceOps() # create user config.user_count = 2 all_users_info = s3lib.create_users(config.user_count, config.cluster_name) u1 = all_users_info[0] u2 = all_users_info[1] # authenticate u1_auth = Auth(u1) u1_rgw_conn = u1_auth.do_auth() u2_auth = Auth(u2) u2_rgw_conn = u2_auth.do_auth() no_of_buckets_to_create = 3 u1_buckets = [] u2_buckets = [] for i in range(no_of_buckets_to_create): u1_bucket = create_bucket(u1_rgw_conn, u1, rand_no=i) log.info('u1_bucket_name: %s' % u1_bucket.name) u1_buckets.append(u1_bucket) u2_bucket = create_bucket(u2_rgw_conn, u2, rand_no=i) log.info('u2_bucket_name: %s' % u2_bucket.name) u2_buckets.append(u2_bucket) # test_acls_private(u1_rgw_conn, u1, u2, u1_buckets[0], u2_buckets[0]) test_acls_public_write(u1_rgw_conn, u1, u2, u1_buckets[1], u2_buckets[1]) # test_acls_public_read(u1_rgw_conn, u1, u2, u1_buckets[2], u2_buckets[2]) # print u1_bucket_info.delete() test_info.success_status('test passed') sys.exit(0) except Exception, e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): test_info = AddTestInfo('test swift user key gen') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # preparing data user_names = ['tuffy', 'scooby', 'max'] tenant1 = 'tenant' cmd = 'radosgw-admin user create --uid=%s --display-name="%s" --tenant=%s --cluster %s' \ %(user_names[0], user_names[0], tenant1, config.cluster_name) out = utils.exec_shell_cmd(cmd) if out is False: raise TestExecError("RGW User creation error") log.info('output :%s' % out) v1_as_json = json.loads(out) log.info('creted user_id: %s' % v1_as_json['user_id']) cmd2 = 'radosgw-admin subuser create --uid=%s$%s --subuser=%s:swift --tenant=%s --access=full --cluster %s' \ % (tenant1, user_names[0], user_names[0], tenant1, config.cluster_name) out2 = utils.exec_shell_cmd(cmd2) if out2 is False: raise TestExecError("sub-user creation error") v2_as_json = json.loads(out2) log.info('created subuser: %s' % v2_as_json['subusers'][0]['id']) cmd3 = 'radosgw-admin key create --subuser=%s:swift --uid=%s$%s --tenant=%s --key-type=swift --gen-secret ' \ '--cluster %s' %(user_names[0], user_names[0], tenant1, tenant1, config.cluster_name) out3 = utils.exec_shell_cmd(cmd3) if out3 is False: raise TestExecError("secret_key gen error") v3_as_json = json.loads(out3) log.info('created subuser: %s\nsecret_key generated: %s' % (v3_as_json['swift_keys'][0]['user'], v3_as_json['swift_keys'][0]['secret_key'])) user_info = { 'user_id': v3_as_json['swift_keys'][0]['user'], 'key': v3_as_json['swift_keys'][0]['secret_key'] } auth = Auth(user_info) rgw = auth.do_auth() for cc in range(config.container_count): container_name = utils.gen_bucket_name_from_userid( user_info['user_id'], rand_no=cc) container = swiftlib.resource_op({ 'obj': rgw, 'resource': 'put_container', 'args': [container_name] }) if container is False: raise TestExecError( "Resource execution failed: container creation faield") for oc in range(config.objects_count): swift_object_name = utils.gen_s3_object_name( '%s.container.%s' % (user_names[0], cc), oc) log.info('object name: %s' % swift_object_name) object_path = os.path.join(TEST_DATA_PATH, swift_object_name) log.info('object path: %s' % object_path) object_size = utils.get_file_size( config.objects_size_range['min'], config.objects_size_range['max']) data_info = manage_data.io_generator(object_path, object_size) if data_info is False: TestExecError("data creation failed") log.info('uploading object: %s' % object_path) with open(object_path, 'r') as fp: rgw.put_object(container_name, swift_object_name, contents=fp.read(), content_type='text/plain') test_info.success_status('test passed') sys.exit(0) except Exception, e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): """ Executes test based on configuration passed Args: config(object): Test configuration """ io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) umgmt = UserMgmt() ceph_conf = CephConfOp() rgw_service = RGWService() # preparing data user_names = ["tom", "ram", "sam"] tenant = "tenant" tenant_user_info = umgmt.create_tenant_user( tenant_name=tenant, user_id=user_names[1], displayname=user_names[1] ) user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[1]) auth = Auth(user_info, config.ssl) rgw = auth.do_auth() container_name = utils.gen_bucket_name_from_userid(user_info["user_id"], rand_no=0) container = swiftlib.resource_op( {"obj": rgw, "resource": "put_container", "args": [container_name]} ) if container is False: raise TestExecError("Resource execution failed: container creation faield") for oc, size in list(config.mapped_sizes.items()): # upload objects to the container swift_object_name = fill_container( rgw, container_name, user_names[1], oc, 0, size ) # delete all uploaded objects log.info("deleting all swift objects") auth_response = rgw.get_auth() token = auth_response[1] # test.txt file should contain container_name with open("test.txt", "w") as f: f.write(container_name) ip_and_port = rgw.authurl.split("/")[2] url = "http://{}/swift/v1/?bulk-delete".format(ip_and_port) test_file = open("test.txt", "r") headers = { "Accept": "application/json", "Content-Type": "text/plain", "X-Auth-Token": token, } response = requests.delete( url, headers=headers, files={"form_field_name": test_file} ) if response.status_code == 200: log.info("Bulk delete succeeded") else: raise TestExecError( "Bulk delete failed with status code: %d" % response.status_code ) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) umgmt = UserMgmt() host, ip = utils.get_hostname_ip() port = utils.get_radosgw_port_no() hostname = str(ip) + ":" + str(port) log.info(hostname) # preparing data admin_api_user = "******" + randomString() log.info(admin_api_user) user_info = umgmt.create_rest_admin_user(user_id=admin_api_user, displayname=admin_api_user) rgw = RGWAdmin(access_key=user_info['access_key'], secret_key=user_info['secret_key'], server=hostname, secure=False, verify=False) api_user = "******" + randomString() log.info(api_user) for uc in range(config.user_count): #Create User data = rgw.create_user(uid=api_user, display_name=api_user, email=api_user + '@abc.xyz') log.info("User created successfully") log.info(data) log.info('verification starts') op = utils.exec_shell_cmd("radosgw-admin user info --uid %s" % api_user) json_doc = json.loads(op) log.info(json_doc) v = verify_user(data, json_doc) if v is False: test_info.failed_status('test failed') sys.exit(1) log.info("Verification for create operation completed") #Update User data = rgw.modify_user(uid=api_user, display_name=api_user + "_11", email=api_user + '*****@*****.**') log.info("User Updated successfully") log.info(data) log.info('verification starts') op = utils.exec_shell_cmd("radosgw-admin user info --uid %s" % api_user) json_doc = json.loads(op) log.info(json_doc) v = verify_user(data, json_doc) if v is False: test_info.failed_status('test failed') sys.exit(1) log.info("Verification for Update operation completed") #delete User data = rgw.remove_user(uid=api_user, purge_data=False) log.info(data) log.info("User removed") op = utils.exec_shell_cmd("radosgw-admin user list") json_doc = json.loads(op) if api_user in json_doc: test_info.failed_status('test failed') sys.exit(1) log.info("Verification for Delete operation completed")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() objects_created_list = [] # create user all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get('encryption_algorithm', None) is not None: log.info('encryption enabled, making ceph config changes') ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') # making changes to max_objects_per_shard and rgw_gc_obj_min_wait to ceph.conf log.info('making changes to ceph.conf') log.info(f'rgw_max_objs_per_shard parameter set to {str(config.max_objects_per_shard)}') ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_max_objs_per_shard, str(config.max_objects_per_shard)) ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_dynamic_resharding, 'True') log.info(f'rgw gc obj min wait configuration parameter set to {str(config.rgw_gc_obj_min_wait)}') ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_gc_obj_min_wait,str(config.rgw_gc_obj_min_wait)) sleep_time = 10 log.info(f'Restarting RGW service and waiting for {sleep_time} seconds') srv_restarted = rgw_service.restart() time.sleep(sleep_time) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{'signature_version': 's3v4'}) else: rgw_conn = auth.do_auth() objects_created_list = [] if config.test_ops['create_bucket'] is True: log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): log.info(f'creating {str(bc)} bucket') bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) bucket = reusable.create_bucket(bucket_name_to_create, rgw_conn, each_user) if config.test_ops.get('enable_version', False): log.info('enable bucket version') reusable.enable_versioning(bucket, rgw_conn, each_user, write_bucket_io_info) if config.test_ops['create_object'] is True: log.info('s3 objects to create: %s' % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name(bucket.name, oc) log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) if config.test_ops.get('enable_version', False): log.info('upload versioned objects') reusable.upload_version_object(config, each_user, rgw_conn, s3_object_name, config.obj_size, bucket, TEST_DATA_PATH) else: log.info('upload type: normal') reusable.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user) objects_created_list.append((s3_object_name, s3_object_path)) #deleting the local file created after upload if config.local_file_delete is True: log.info('deleting local file created after the upload') utils.exec_shell_cmd('rm -rf %s' % s3_object_path) # listing the objects if config.test_ops.get('list_objects', False): if config.test_ops.get('enable_version', False): for name,path in objects_created_list: reusable.list_versioned_objects(bucket,name,path,rgw_conn) else: reusable.list_objects(bucket) if config.test_ops.get('delete_bucket_object', False): if config.test_ops.get('enable_version', False): for name, path in objects_created_list: print("name, path",name,path) versions = bucket.object_versions.filter(Prefix=name) log.info('deleting s3_obj keys and its versions') s3_obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, name]}) log.info('deleting versions for s3 obj: %s' % name) for version in versions: log.info('trying to delete obj version: %s' % version.version_id) del_obj_version = s3lib.resource_op({'obj': s3_obj, 'resource': 'delete', 'kwargs': dict(VersionId=version.version_id)}) log.info('response:\n%s' % del_obj_version) if del_obj_version is not None: response = HttpResponseParser(del_obj_version) if response.status_code == 204: log.info('version deleted ') reusable.delete_version_object(bucket,version.version_id, path, rgw_conn, each_user) else: raise TestExecError("version deletion failed") else: raise TestExecError("version deletion failed") else: reusable.delete_objects(bucket) log.info(f'deleting the bucket {bucket_name_to_create}') reusable.delete_bucket(bucket) # check for any crashes during the execution crash_info=reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!") #remove the user reusable.remove_user(each_user)
def test_exec(config): test_info = AddTestInfo('test bucket policy') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # preparing data user_names = ['user1', 'user2', 'user3'] Bucket_names = ['bucket1', 'bucket2', 'bucket3'] object_names = ['o1', 'o2'] tenant1 = 'tenant1' tenant2 = 'tenant2' t1_u1_info = create_tenant_user(tenant_name=tenant1, user_id=user_names[0]) t1_u1_auth = Auth(t1_u1_info) t1_u1 = t1_u1_auth.do_auth() t2_u1_info = create_tenant_user(tenant_name=tenant2, user_id=user_names[0]) t2_u1_auth = Auth(t2_u1_info) t2_u1 = t2_u1_auth.do_auth() t1_u1_b1 = resuables.create_bucket(bucket_name=Bucket_names[0], rgw=t1_u1, user_info=t1_u1_info) t2_u1_b1 = resuables.create_bucket(bucket_name=Bucket_names[0], rgw=t2_u1, user_info=t2_u1_info) obj_sizes = config.mapped_sizes.values() config.obj_size = obj_sizes[0] resuables.upload_object(s3_object_name=object_names[0], bucket=t1_u1_b1, TEST_DATA_PATH=TEST_DATA_PATH, config=config, user_info=t1_u1_info) config.obj_size = obj_sizes[1] resuables.upload_object(s3_object_name=object_names[0], bucket=t2_u1_b1, TEST_DATA_PATH=TEST_DATA_PATH, config=config, user_info=t1_u1_info) t2_u2_info = create_tenant_user(tenant_name=tenant2, user_id=user_names[1]) t2_u2_auth = Auth(t2_u2_info) t2_u2 = t2_u2_auth.do_auth() # will try to access the bucket and objects in both tenants # access t1_u1_b1 log.info('trying to access tenant1->user1->bucket1') t1_u1_b1_from_t2_u2 = s3lib.resource_op({ 'obj': t2_u2, 'resource': 'Bucket', 'args': [Bucket_names[0]] }) log.info( 'trying to download tenant1->user1->bucket1->object1 from tenant2->user2' ) download_path1 = TEST_DATA_PATH + "/t1_u1_b1_%s.download" % object_names[ 0] t1_u1_b1_o1_download = s3lib.resource_op({ 'obj': t1_u1_b1_from_t2_u2, 'resource': 'download_file', 'args': [object_names[0], download_path1] }) if t1_u1_b1_o1_download is False: log.info('object not downloaded\n') if t1_u1_b1_o1_download is None: raise TestExecError( "object downloaded for tenant1->user1->bucket1->object1, this should not happen" ) log.info( 'trying to access tenant2->user1->bucket1 from user2 in tenant 2') t2_u1_b1_from_t2_u2 = s3lib.resource_op({ 'obj': t2_u2, 'resource': 'Bucket', 'args': [Bucket_names[0]] }) log.info( 'trying to download tenant2->user1->bucket1->object1 from tenant2->user2' ) download_path2 = TEST_DATA_PATH + "/t2_u1_b1_%s.download" % object_names[ 0] t2_u1_b1_o1_download = s3lib.resource_op({ 'obj': t2_u1_b1_from_t2_u2, 'resource': 'download_file', 'args': [object_names[0], download_path2] }) if t2_u1_b1_o1_download is False: log.info('object did not download, worked as expected') if t1_u1_b1_o1_download is None: raise TestExecError( 'object downloaded\n' 'downloaded tenant2->user1->bucket1->object1, this should not happen' ) test_info.success_status('test passed') sys.exit(0) except Exception, e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get('encryption_algorithm', None) is not None: log.info('encryption enabled, making ceph config changes') ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{'signature_version': 's3v4'}) else: rgw_conn = auth.do_auth() # enabling sharding if config.test_ops['sharding']['enable'] is True: log.info('enabling sharding on buckets') max_shards = config.test_ops['sharding']['max_shards'] log.info('making changes to ceph.conf') ceph_conf.set_to_ceph_conf( 'global', ConfigOpts.rgw_override_bucket_index_max_shards, str(max_shards)) log.info('trying to restart services ') srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') if config.test_ops['compression']['enable'] is True: compression_type = config.test_ops['compression']['type'] log.info('enabling compression') cmd = 'radosgw-admin zone get' out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = 'radosgw-admin zone placement modify --rgw-zone=%s ' \ '--placement-id=default-placement --compression=%s' % (zone,compression_type) out = utils.exec_shell_cmd(cmd) try: data = json.loads(out) if data['placement_pools'][0]['val']['storage_classes'][ 'STANDARD']['compression_type'] == compression_type: log.info('Compression enabled successfully') else: raise ValueError('failed to enable compression') except ValueError as e: exit(str(e)) log.info('trying to restart rgw services ') srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') # create buckets if config.test_ops['create_bucket'] is True: log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) bucket = resuables.create_bucket(bucket_name_to_create, rgw_conn, each_user) if config.test_ops['create_object'] is True: # uploading data log.info('s3 objects to create: %s' % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) if config.test_ops.get('upload_type') == 'multipart': log.info('upload type: multipart') resuables.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user) else: log.info('upload type: normal') resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user) if config.test_ops['download_object'] is True: log.info('trying to download object: %s' % s3_object_name) s3_object_download_name = s3_object_name + "." + "download" s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_download_name) log.info('s3_object_download_path: %s' % s3_object_download_path) log.info('downloading to filename: %s' % s3_object_download_name) if config.test_ops.get('encryption_algorithm', None) is not None: log.info('encryption download') log.info( 'encryption algorithm: %s' % config.test_ops['encryption_algorithm']) object_downloaded_status = bucket.download_file( s3_object_name, s3_object_download_path, ExtraArgs={ 'SSECustomerKey': encryption_key, 'SSECustomerAlgorithm': config.test_ops['encryption_algorithm'] }) else: object_downloaded_status = s3lib.resource_op({ 'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info('object downloaded') s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path) s3_object_uploaded_md5 = utils.get_md5( s3_object_path) log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5) log.info('s3_object_uploaded_md5: %s' % s3_object_uploaded_md5) if str(s3_object_uploaded_md5) == str( s3_object_downloaded_md5): log.info('md5 match') utils.exec_shell_cmd('rm -rf %s' % s3_object_download_path) else: raise TestExecError('md5 mismatch') if config.local_file_delete is True: log.info( 'deleting local file created after the upload') utils.exec_shell_cmd('rm -rf %s' % s3_object_path) # verification of shards after upload if config.test_ops['sharding']['enable'] is True: cmd = 'radosgw-admin metadata get bucket:%s | grep bucket_id' % bucket.name out = utils.exec_shell_cmd(cmd) b_id = out.replace( '"', '').strip().split(":")[1].strip().replace(',', '') cmd2 = 'rados -p default.rgw.buckets.index ls | grep %s' % b_id out = utils.exec_shell_cmd(cmd2) log.info( 'got output from sharing verification.--------') # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops['compression']['enable'] is True: cmd = 'radosgw-admin bucket stats --bucket=%s' % bucket.name out = utils.exec_shell_cmd(cmd) # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops['compression']['enable'] is True: cmd = 'radosgw-admin bucket stats --bucket=%s' % bucket.name out = utils.exec_shell_cmd(cmd) if config.test_ops['delete_bucket_object'] is True: log.info('listing all objects in bucket: %s' % bucket.name) objects = s3lib.resource_op({ 'obj': bucket, 'resource': 'objects', 'args': None }) log.info('objects :%s' % objects) all_objects = s3lib.resource_op({ 'obj': objects, 'resource': 'all', 'args': None }) log.info('all objects: %s' % all_objects) for obj in all_objects: log.info('object_name: %s' % obj.key) log.info('deleting all objects in bucket') objects_deleted = s3lib.resource_op({ 'obj': objects, 'resource': 'delete', 'args': None }) log.info('objects_deleted: %s' % objects_deleted) if objects_deleted is False: raise TestExecError( 'Resource execution failed: Object deletion failed' ) if objects_deleted is not None: response = HttpResponseParser(objects_deleted[0]) if response.status_code == 200: log.info('objects deleted ') else: raise TestExecError("objects deletion failed") else: raise TestExecError("objects deletion failed") log.info('deleting bucket: %s' % bucket.name) # bucket_deleted_status = s3_ops.resource_op(bucket, 'delete') bucket_deleted_status = s3lib.resource_op({ 'obj': bucket, 'resource': 'delete', 'args': None }) log.info('bucket_deleted_status: %s' % bucket_deleted_status) if bucket_deleted_status is not None: response = HttpResponseParser( bucket_deleted_status) if response.status_code == 204: log.info('bucket deleted ') else: raise TestExecError("bucket deletion failed") else: raise TestExecError("bucket deletion failed") # disable compression after test if config.test_ops['compression']['enable'] is True: log.info('disable compression') cmd = 'radosgw-admin zone get' out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = 'radosgw-admin zone placement modify --rgw-zone=%s ' \ '--placement-id=default-placement --compression=none' % zone out = utils.exec_shell_cmd(cmd) srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted')
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) write_user_info = AddUserInfo() test_info = AddTestInfo('create m buckets') conf_path = '/etc/ceph/%s.conf' % config.cluster_name ceph_conf = CephConfOp(conf_path) rgw_service = RGWService() try: test_info.started_info() # get user with open('user_details') as fout: all_users_info = simplejson.load(fout) for each_user in all_users_info: user_info = basic_io_structure.user(**{'user_id': each_user['user_id'], 'access_key': each_user['access_key'], 'secret_key': each_user['secret_key']}) write_user_info.add_user_info(user_info) for each_user in all_users_info: # authenticate auth = Auth(each_user) rgw_conn = auth.do_auth() # enabling sharding if config.test_ops['sharding']['enable'] is True: log.info('enabling sharding on buckets') max_shards = config.test_ops['sharding']['max_shards'] log.info('making changes to ceph.conf') ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_override_bucket_index_max_shards, max_shards) log.info('trying to restart services ') srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') # create buckets if config.test_ops['create_bucket'] is True: log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Bucket', 'args': [bucket_name_to_create]}) created = s3lib.resource_op({'obj': bucket, 'resource': 'create', 'args': None, 'extra_info': {'access_key': each_user['access_key']}}) if created is False: raise TestExecError("Resource execution failed: bucket creation failed") if created is not None: response = HttpResponseParser(created) if response.status_code == 200: log.info('bucket created') else: raise TestExecError("bucket creation failed") else: raise TestExecError("bucket creation failed") if config.test_ops['sharding']['enable'] is True: cmd = 'radosgw-admin metadata get bucket:%s --cluster %s | grep bucket_id' \ % (bucket.name, config.cluster_name) out = utils.exec_shell_cmd(cmd) b_id = out.replace('"', '').strip().split(":")[1].strip().replace(',', '') cmd2 = 'rados -p default.rgw.buckets.index ls --cluster %s | grep %s' \ % (config.cluster_name, b_id) out = utils.exec_shell_cmd(cmd2) log.info('got output from sharing verification.--------') test_info.success_status('test passed') sys.exit(0) except Exception,e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() config.rgw_lc_debug_interval = 1 config.rgw_lifecycle_work_time = "00:00-23:59" log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_lc_debug_interval, str(config.rgw_lc_debug_interval)) log.info("trying to restart services") srv_restarted = rgw_service.restart() if srv_restarted is False: raise TestExecError("RGW service restart failed") rgw_service.status() # create user user_info = s3lib.create_users(config.user_count) for each_user in user_info: auth = Auth(each_user, ssl=config.ssl) rgw_conn = auth.do_auth() rgw_conn2 = auth.do_auth_using_client() if config.test_ops["create_bucket"]: log.info("no of buckets to create: %s" % config.bucket_count) # create bucket for bc in range(config.bucket_count): bucket_name = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=1) bucket = reusable.create_bucket(bucket_name, rgw_conn, each_user) life_cycle_rule = {"Rules": config.lifecycle_conf} reusable.put_bucket_lifecycle(bucket, rgw_conn, rgw_conn2, life_cycle_rule) if config.test_ops["create_object"]: for oc, size in list(config.mapped_sizes.items()): config.obj_size = size log.info( f"s3 objects to create of size {config.obj_size}") s3_object_name = config.lifecycle_conf[0]["Filter"][ "Prefix"] + str(oc) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info( f"s3 object path: {s3_object_path}, name: {s3_object_name}" ) reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user) for i in (1, 100): time.sleep(60) bucket_details = json.loads( utils.exec_shell_cmd( f"radosgw-admin bucket stats --bucket={bucket.name}" )) if bucket_details["usage"]["rgw.main"]["num_objects"] == 0: break else: raise TestExecError( "Bucket object expiration taking longer than expected") gc_list_output = json.loads( utils.exec_shell_cmd( "radosgw-admin gc list --include-all")) if gc_list_output: log.info("Removing shadow objects found") utils.exec_shell_cmd( "radosgw-admin gc process --include-all") bucket_id = (bucket_details["id"] + "_" + config.lifecycle_conf[0]["Filter"]["Prefix"]) log.info( f"check for all the entry {bucket_id} for the bucket in data pool" ) obj_pool = utils.exec_shell_cmd( f"rados ls -p default.rgw.buckets.data | grep {bucket_id}") if obj_pool: for obj in obj_pool: object_name = obj.split("_")[-1] log.info(f"s3 object name to download: {object_name}") object_name_downloaded = object_name + "." + "download" object_download_path = os.path.join( TEST_DATA_PATH, object_name_downloaded) object_downloaded_status = s3lib.resource_op({ "obj": bucket, "resource": "download_file", "args": [object_name, object_download_path], }) if object_downloaded_status is False: log.info("As expected object is not Downloadable") if object_downloaded_status is None: raise TestExecError( "Objects are not listed but can be downloadable" ) if config.local_file_delete: log.info("deleting local file created after the upload") utils.exec_shell_cmd(f"rm -rf {TEST_DATA_PATH}") reusable.delete_bucket(bucket) reusable.remove_user(each_user) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) rgw_service = RGWService() log.info( "adding indexless placement to placement target of default zonegroup") zonegroup_set = utils.exec_shell_cmd( 'radosgw-admin zonegroup placement add --rgw-zonegroup="default" --placement-id="indexless-placement"' ) log.info("adding indexless placement to placement pool of default zone") zone_set = utils.exec_shell_cmd( 'radosgw-admin zone placement add --rgw-zone="default" --placement-id="indexless-placement" --data-pool="default.rgw.buckets.data" --index-pool="default.rgw.buckets.index" --data_extra_pool="default.rgw.buckets.non-ec" --placement-index-type="indexless"' ) log.info("making indexless-placement as default") indexless_default = utils.exec_shell_cmd( 'radosgw-admin zonegroup placement default --placement-id="indexless-placement"' ) log.info("restart the rgw daemons") restart_service = rgw_service.restart() if restart_service is False: raise TestExecError("RGW service restart failed") log.info("sleep for 20 seconds after RGW service restart") time.sleep(20) # perform s3 operations all_users_info = s3lib.create_users(config.user_count) for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{"signature_version": "s3v4"}) else: rgw_conn = auth.do_auth() if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc) log.info("creating bucket with name: %s" % bucket_name_to_create) bucket = reusable.create_bucket(bucket_name_to_create, rgw_conn, each_user) if config.test_ops["create_object"] is True: # uploading data log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) log.info("upload type: normal") reusable.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user) # verify the bucket created has index_type = Indexless log.info("verify the default placement is Indexless placement") zonegroup_get = utils.exec_shell_cmd("radosgw-admin zonegroup get") zonegroup_get_json = json.loads(zonegroup_get) default_place = zonegroup_get_json["default_placement"] if default_place == "indexless-placement": log.info("default placement is Indexless placement") else: raise TestExecError("default placement is not Indexless placement") # delete bucket and objects if config.test_ops["delete_bucket"] is True: log.info("Deleting buckets and objects") reusable.delete_bucket(bucket) # reverting to default placement group log.info("revert changes to zone, zonegroup and default placement target") zone_set = utils.exec_shell_cmd( 'radosgw-admin zone placement rm --rgw-zone="default" --placement-id="indexless-placement" ' ) if "indexless" in zone_set: raise TestExecError( "Indexless placement present in zone even after revert") zonegroup_set = utils.exec_shell_cmd( 'radosgw-admin zonegroup placement rm --rgw-zonegroup="default" --placement-id="indexless-placement"' ) if "indexless" in zonegroup_set: raise TestExecError( "Indexless placement present in zonegroup even after revert") default_placement = utils.exec_shell_cmd( 'radosgw-admin zonegroup placement default --placement-id="default-placement"' ) log.info("restart the rgw daemons") restart_service = rgw_service.restart() if restart_service is False: raise TestExecError("RGW service restart failed") log.info("sleep for 20 seconds after RGW service restart") time.sleep(20) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get("encryption_algorithm", None) is not None: log.info("encryption enabled, making ceph config changes") ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{"signature_version": "s3v4"}) else: rgw_conn = auth.do_auth() objects_created_list = [] if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc) log.info("creating bucket with name: %s" % bucket_name_to_create) bucket = reusable.create_bucket(bucket_name_to_create, rgw_conn, each_user) if config.test_ops.get("enable_version", False): log.info("enable bucket version") reusable.enable_versioning(bucket, rgw_conn, each_user, write_bucket_io_info) if config.test_ops["create_object"] is True: if config.test_ops["object_structure"] == "flat": # uploading data log.info("top level s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join( TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get( "upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) objects_created_list.append( (s3_object_name, s3_object_path)) # deleting the local file created after upload if config.local_file_delete is True: log.info( "deleting local file created after the upload" ) utils.exec_shell_cmd("rm -rf %s" % s3_object_path) # this covers listing of a bucket with pseudo directories and objects in it ; Unable to list contents of large buckets https://bugzilla.redhat.com/show_bug.cgi?id=1874645#c72 if config.test_ops["object_structure"] == "pseudo": log.info( f"pseudo directories to create {config.pseudo_dir_count} with {config.objects_count} objects in each" ) for count in range(config.pseudo_dir_count): s3_pseudo_dir_name = utils.gen_s3_object_name( bucket_name_to_create, count) s3_object_path = os.path.join( TEST_DATA_PATH, s3_pseudo_dir_name) manage_data.pseudo_dir_generator(s3_object_path) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_pseudo_object_name( s3_pseudo_dir_name, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join( TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get( "upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) # deleting the local file created after upload if config.local_file_delete is True: log.info( "deleting local file created after the upload" ) utils.exec_shell_cmd("rm -rf %s" % s3_object_path) # listing bucket with only pseudo directories ; Bug allows ordered bucket listing to get stuck -- 4.1 https://bugzilla.redhat.com/show_bug.cgi?id=1853052#c0 if config.test_ops["create_object"] is False: if config.test_ops[ "object_structure"] == "pseudo-dir-only": log.info( f"pseudo directories to create {config.pseudo_dir_count}" ) for count in range(config.pseudo_dir_count): s3_pseudo_dir_name = utils.gen_s3_object_name( bucket_name_to_create, count) utils.create_psuedo_dir(s3_pseudo_dir_name, bucket) # radoslist listing of the bucket if config.test_ops["radoslist"] is True: log.info( "executing the command radosgw-admin bucket radoslist " ) radoslist = utils.exec_shell_cmd( "radosgw-admin bucket radoslist --bucket %s" % bucket_name_to_create) if radoslist is False: raise TestExecError( "Radoslist command execution failed") # get the configuration parameter - rgw_bucket_index_max_aio ceph_version_id, ceph_version_name = utils.get_ceph_version() if ceph_version_name in ["luminous", "nautilus"]: cmd = "ceph daemon `ls -t /var/run/ceph/ceph-client.rgw.*.asok|head -1` config show |grep rgw_bucket_index_max_aio" max_aio_output = utils.exec_shell_cmd(cmd) max_aio = max_aio_output.split()[1] else: cmd = "ceph config get mon rgw_bucket_index_max_aio" max_aio_output = utils.exec_shell_cmd(cmd) max_aio = max_aio_output.rstrip("\n") # bucket stats to get the num_objects of the bucket bucket_stats = utils.exec_shell_cmd( "radosgw-admin bucket stats --bucket %s" % bucket_name_to_create) bucket_stats_json = json.loads(bucket_stats) bkt_num_objects = bucket_stats_json["usage"]["rgw.main"][ "num_objects"] # ordered listing via radosgw-admin command and noting time taken log.info( "measure the execution time taken to list via radosgw-admin command" ) if config.test_ops["radosgw_listing_ordered"] is True: log.info("ordered listing via radosgw-admin command") rgw_cmd_time = reusable.time_to_list_via_radosgw( bucket_name_to_create, "ordered") if rgw_cmd_time > 0: rgw_cmd_time_secs = "{:.4f}".format(rgw_cmd_time) rgw_cmd_time_mins = "{:.4f}".format(rgw_cmd_time / 60) log.info( f"with rgw_bucket_index_max_aio = {max_aio} time taken for ordered listing of {bkt_num_objects} objects is : {rgw_cmd_time_secs} secs ; {rgw_cmd_time_mins} mins" ) else: raise TestExecError( "object listing via radosgw-admin command failed") # unordered listing via radosgw-admin command and noting time taken if config.test_ops["radosgw_listing_ordered"] is False: log.info("unordered listing via radosgw-admin command") rgw_time = reusable.time_to_list_via_radosgw( bucket_name_to_create, "unordered") if rgw_time > 0: rgw_time_secs = "{:.4f}".format(rgw_time) rgw_time_mins = "{:.4f}".format(rgw_time / 60) log.info( f"with rgw_bucket_index_max_aio = {max_aio} time taken for unordered listing of {bkt_num_objects} objects is : {rgw_time_secs} secs ; {rgw_time_mins} mins" ) else: raise TestExecError( "object listing via radosgw-admin command failed") # listing via boto and noting the time taken log.info("measure the execution time taken to list via boto") boto_time = reusable.time_to_list_via_boto( bucket_name_to_create, rgw_conn) if boto_time > 0: boto_time_secs = "{:.4f}".format(boto_time) boto_time_mins = "{:.4f}".format(boto_time / 60) log.info( f"with rgw_bucket_index_max_aio = {max_aio} time taken to list {bkt_num_objects} objects via boto : {boto_time_secs} secs ; {boto_time_mins} mins" ) else: raise TestExecError("object listing via boto failed") # radoslist on all buckets. BZ:https://bugzilla.redhat.com/show_bug.cgi?id=1892265 if config.radoslist_all is True: log.info( "Executing the command radosgw-admin bucket radoslist on all buckets" ) cmd = "radosgw-admin bucket radoslist | grep ERROR" radoslist_all_error = utils.exec_shell_cmd(cmd) print(radoslist_all_error) if radoslist_all_error is False: raise TestExecError("ERROR in radoslist command") if config.test_ops.get("delete_bucket_object", False): if config.test_ops.get("enable_version", False): for name, path in objects_created_list: reusable.delete_version_object(bucket, name, path, rgw_conn, each_user) else: reusable.delete_objects(bucket) time.sleep(30) reusable.delete_bucket(bucket) # check sync status if a multisite cluster reusable.check_sync_status() # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!") if config.user_remove is True: reusable.remove_user(each_user)
def test_exec(config): test_info = AddTestInfo('test versioning with objects') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # create user all_users_info = s3lib.create_users(config.user_count, config.cluster_name) for each_user in all_users_info: # authenticate auth = Auth(each_user) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid(each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Bucket', 'args': [bucket_name_to_create]}) # created = s3_ops.resource_op(bucket, 'create', None, **{'access_key': each_user['access_key']}) created = s3lib.resource_op({'obj': bucket, 'resource': 'create', 'args': None, 'extra_info': {'access_key': each_user['access_key']}}) if created is False: raise TestExecError("Resource execution failed: bucket creation faield") if created is not None: response = HttpResponseParser(created) if response.status_code == 200: log.info('bucket created') else: raise TestExecError("bucket creation failed") else: raise TestExecError("bucket creation failed") # getting bucket version object if config.test_ops['enable_version'] is True: log.info('bucket versionig test on bucket: %s' % bucket.name) # bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name) bucket_versioning = s3lib.resource_op({'obj': rgw_conn, 'resource': 'BucketVersioning', 'args': [bucket.name]}) # checking the versioning status # version_status = s3_ops.resource_op(bucket_versioning, 'status') version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'status', 'args': None }) if version_status is None: log.info('bucket versioning still not enabled') # enabling bucket versioning # version_enable_status = s3_ops.resource_op(bucket_versioning, 'enable') version_enable_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'enable', 'args': None}) response = HttpResponseParser(version_enable_status) if response.status_code == 200: log.info('version enabled') else: raise TestExecError("version enable failed") if config.objects_count > 0: log.info('s3 objects to create: %s' % config.objects_count) for oc in range(config.objects_count): # versioning upload s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc)) s3_object_names.append(s3_object_name) log.info('s3 object name: %s' % s3_object_name) log.info('versioning count: %s' % config.version_count) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) s3_object_name = utils.gen_s3_object_name(bucket_name_to_create, str(oc)) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) original_data_info = manage_data.io_generator(s3_object_path, s3_object_size) if original_data_info is False: TestExecError("data creation failed") for vc in range(config.version_count): log.info('version count for %s is %s' % (s3_object_name, str(vc))) log.info('modifying data: %s' % s3_object_name) modified_data_info = manage_data.io_generator(s3_object_path, s3_object_size, data='append', **{ 'message': '\nhello object for version: %s\n' % str( vc)}) if modified_data_info is False: TestExecError("data modification failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key']}, **modified_data_info) object_uploaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'upload_file', 'args': [modified_data_info['name'], s3_object_name], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') log.info('all versions for the object: %s\n' % s3_object_name) versions = bucket.object_versions.filter(Prefix=s3_object_name) for version in versions: log.info('key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) if config.test_ops['copy_to_version'] is True: # reverting object to one of the versions ( randomly chosen ) version_id_to_copy = random.choice([v.version_id for v in versions]) log.info('version_id_to_copy: %s' % version_id_to_copy) s3_obj = rgw_conn.Object(bucket.name, s3_object_name) log.info('current version_id: %s' % s3_obj.version_id) copy_response = s3_obj.copy_from(CopySource={'Bucket': bucket.name, 'Key': s3_object_name, 'VersionId': version_id_to_copy}) log.info('copy_response: %s' % copy_response) if copy_response is None: raise TestExecError("copy object from version id failed") # current_version_id = copy_response['VersionID'] log.info('current_version_id: %s' % s3_obj.version_id) # delete the version_id_to_copy object s3_obj.delete(VersionId=version_id_to_copy) log.info('all versions for the object after the copy operation: %s\n' % s3_object_name) for version in versions: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) # log.info('downloading current s3object: %s' % s3_object_name) # s3_obj.download_file(s3_object_name + ".download") if config.test_ops['delete_object_versions'] is True: log.info('deleting s3_obj keys and its versions') s3_obj = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name]}) log.info('deleting versions for s3 obj: %s' % s3_object_name) for version in versions: log.info('trying to delete obj version: %s' % version.version_id) del_obj_version = s3lib.resource_op({'obj': s3_obj, 'resource': 'delete', 'kwargs': dict(VersionId=version.version_id)}) log.info('response:\n%s' % del_obj_version) if del_obj_version is not None: response = HttpResponseParser(del_obj_version) if response.status_code == 204: log.info('version deleted ') else: raise TestExecError("version deletion failed") else: raise TestExecError("version deletion failed") if config.test_ops['suspend_version'] is True: # suspend_version_status = s3_ops.resource_op(bucket_versioning, 'suspend') suspend_version_status = s3lib.resource_op({'obj': bucket_versioning, 'resource': 'suspend', 'args': None}) response = HttpResponseParser(suspend_version_status) if response.status_code == 200: log.info('versioning suspended') else: raise TestExecError("version suspend failed") if config.test_ops['upload_after_suspend'] is True: log.info('trying to upload after suspending versioning on bucket') for s3_object_name in s3_object_names: # non versioning upload log.info('s3 object name: %s' % s3_object_name) s3_object_size = utils.get_file_size(config.objects_size_range['min'], config.objects_size_range['max']) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) non_version_data_info = manage_data.io_generator(s3_object_path, s3_object_size, op="append", **{ 'message': '\nhello object for non version\n'}) if non_version_data_info is False: TestExecError("data creation failed") log.info('uploading s3 object: %s' % s3_object_path) upload_info = dict({'access_key': each_user['access_key']}, **non_version_data_info) object_uploaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'upload_file', 'args': [non_version_data_info['name'], s3_object_name], 'extra_info': upload_info}) if object_uploaded_status is False: raise TestExecError("Resource execution failed: object upload failed") if object_uploaded_status is None: log.info('object uploaded') s3_object_download_path = os.path.join(TEST_DATA_PATH, s3_object_name + ".download") object_downloaded_status = s3lib.resource_op({'obj': bucket, 'resource': 'download_file', 'args': [s3_object_name, s3_object_download_path], }) if object_downloaded_status is False: raise TestExecError("Resource execution failed: object download failed") if object_downloaded_status is None: log.info('object downloaded') # checking md5 of the downloaded file s3_object_downloaded_md5 = utils.get_md5(s3_object_download_path) log.info('s3_object_downloaded_md5: %s' % s3_object_downloaded_md5) log.info('s3_object_uploaded_md5: %s' % non_version_data_info['md5']) test_info.success_status('test passed') sys.exit(0) except Exception as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1) except TestExecError as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() log.info("starting IO") config.user_count = 1 user_info = s3lib.create_users(config.user_count) user_info = user_info[0] auth = Auth(user_info, ssl=config.ssl) rgw_conn = auth.do_auth() log.info("sharding configuration will be added now.") if config.sharding_type == "dynamic": log.info("sharding type is dynamic") # for dynamic, # the number of shards should be greater than [ (no of objects)/(max objects per shard) ] # example: objects = 500 ; max object per shard = 10 # then no of shards should be at least 50 or more time.sleep(15) log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_max_objs_per_shard, str(config.max_objects_per_shard), ) ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_dynamic_resharding, "True") num_shards_expected = config.objects_count / config.max_objects_per_shard log.info("num_shards_expected: %s" % num_shards_expected) log.info("trying to restart services ") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") config.bucket_count = 1 objects_created_list = [] log.info("no of buckets to create: %s" % config.bucket_count) bucket_name = utils.gen_bucket_name_from_userid(user_info["user_id"], rand_no=1) bucket = reusable.create_bucket(bucket_name, rgw_conn, user_info) if config.test_ops.get("enable_version", False): log.info("enable bucket version") reusable.enable_versioning(bucket, rgw_conn, user_info, write_bucket_io_info) log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name(bucket.name, oc) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) if config.test_ops.get("enable_version", False): reusable.upload_version_object( config, user_info, rgw_conn, s3_object_name, config.obj_size, bucket, TEST_DATA_PATH, ) else: reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, user_info ) objects_created_list.append((s3_object_name, s3_object_path)) if config.sharding_type == "manual": log.info("sharding type is manual") # for manual. # the number of shards will be the value set in the command. time.sleep(15) log.info("in manual sharding") cmd_exec = utils.exec_shell_cmd( "radosgw-admin bucket reshard --bucket=%s --num-shards=%s " "--yes-i-really-mean-it" % (bucket.name, config.shards) ) if cmd_exec is False: raise TestExecError("manual resharding command execution failed") sleep_time = 600 log.info(f"verification starts after waiting for {sleep_time} seconds") time.sleep(sleep_time) op = utils.exec_shell_cmd("radosgw-admin metadata get bucket:%s" % bucket.name) json_doc = json.loads(op) bucket_id = json_doc["data"]["bucket"]["bucket_id"] op2 = utils.exec_shell_cmd( "radosgw-admin metadata get bucket.instance:%s:%s" % (bucket.name, bucket_id) ) json_doc2 = json.loads((op2)) num_shards_created = json_doc2["data"]["bucket_info"]["num_shards"] log.info("no_of_shards_created: %s" % num_shards_created) if config.sharding_type == "manual": if config.shards != num_shards_created: raise TestExecError("expected number of shards not created") log.info("Expected number of shards created") if config.sharding_type == "dynamic": log.info("Verify if resharding list is empty") reshard_list_op = json.loads(utils.exec_shell_cmd("radosgw-admin reshard list")) if not reshard_list_op: log.info( "for dynamic number of shards created should be greater than or equal to number of expected shards" ) log.info("no_of_shards_expected: %s" % num_shards_expected) if int(num_shards_created) >= int(num_shards_expected): log.info("Expected number of shards created") else: raise TestExecError("Expected number of shards not created") if config.test_ops.get("delete_bucket_object", False): if config.test_ops.get("enable_version", False): for name, path in objects_created_list: reusable.delete_version_object(bucket, name, path, rgw_conn, user_info) else: reusable.delete_objects(bucket) reusable.delete_bucket(bucket) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get("encryption_algorithm", None) is not None: log.info("encryption enabled, making ceph config changes") ceph_conf.set_to_ceph_conf("global", ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{"signature_version": "s3v4"}) else: rgw_conn = auth.do_auth() # enabling sharding if config.test_ops["sharding"]["enable"] is True: log.info("enabling sharding on buckets") max_shards = config.test_ops["sharding"]["max_shards"] log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_override_bucket_index_max_shards, str(max_shards), ) log.info("trying to restart services ") srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.test_ops["compression"]["enable"] is True: compression_type = config.test_ops["compression"]["type"] log.info("enabling compression") cmd = "radosgw-admin zone get" out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = ( "radosgw-admin zone placement modify --rgw-zone=%s " "--placement-id=default-placement --compression=%s" % (zone, compression_type) ) out = utils.exec_shell_cmd(cmd) ceph_version = utils.exec_shell_cmd("ceph version").split()[4] try: data = json.loads(out) if ceph_version == "luminous": if ( data["placement_pools"][0]["val"]["compression"] == compression_type ): log.info("Compression enabled successfully") else: if ceph_version in ["nautilus", "octopus"]: if ( data["placement_pools"][0]["val"]["storage_classes"][ "STANDARD" ]["compression_type"] == compression_type ): log.info("Compression enabled successfully") except ValueError as e: exit(str(e)) log.info("trying to restart rgw services ") srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.gc_verification is True: conf = config.ceph_conf reusable.set_gc_conf(ceph_conf, conf) # create buckets if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc ) log.info("creating bucket with name: %s" % bucket_name_to_create) bucket = reusable.create_bucket( bucket_name_to_create, rgw_conn, each_user ) if config.test_ops["create_object"] is True: # uploading data log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc ) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, ) if config.test_ops["download_object"] is True: log.info("trying to download object: %s" % s3_object_name) s3_object_download_name = s3_object_name + "." + "download" s3_object_download_path = os.path.join( TEST_DATA_PATH, s3_object_download_name ) log.info( "s3_object_download_path: %s" % s3_object_download_path ) log.info( "downloading to filename: %s" % s3_object_download_name ) if ( config.test_ops.get("encryption_algorithm", None) is not None ): log.info("encryption download") log.info( "encryption algorithm: %s" % config.test_ops["encryption_algorithm"] ) object_downloaded_status = bucket.download_file( s3_object_name, s3_object_download_path, ExtraArgs={ "SSECustomerKey": encryption_key, "SSECustomerAlgorithm": config.test_ops[ "encryption_algorithm" ], }, ) else: object_downloaded_status = s3lib.resource_op( { "obj": bucket, "resource": "download_file", "args": [ s3_object_name, s3_object_download_path, ], } ) if object_downloaded_status is False: raise TestExecError( "Resource execution failed: object download failed" ) if object_downloaded_status is None: log.info("object downloaded") s3_object_downloaded_md5 = utils.get_md5( s3_object_download_path ) s3_object_uploaded_md5 = utils.get_md5(s3_object_path) log.info( "s3_object_downloaded_md5: %s" % s3_object_downloaded_md5 ) log.info( "s3_object_uploaded_md5: %s" % s3_object_uploaded_md5 ) if str(s3_object_uploaded_md5) == str( s3_object_downloaded_md5 ): log.info("md5 match") utils.exec_shell_cmd( "rm -rf %s" % s3_object_download_path ) else: raise TestExecError("md5 mismatch") if config.local_file_delete is True: log.info("deleting local file created after the upload") utils.exec_shell_cmd("rm -rf %s" % s3_object_path) # verification of shards after upload if config.test_datalog_trim_command is True: cmd = "sudo radosgw-admin datalog trim --shard-id 117 --end-marker 1_1626169668.769402_510233116.1 --debug_ms=1 --debug_rgw=20" out, err = utils.exec_shell_cmd(cmd, debug_info=True) if "Segmentation fault" in err: log.info("Segmentation fault occured") test_info.failed_status("test failed") sys.exit(1) if config.test_ops["sharding"]["enable"] is True: cmd = ( "radosgw-admin metadata get bucket:%s | grep bucket_id" % bucket.name ) out = utils.exec_shell_cmd(cmd) b_id = ( out.replace('"', "") .strip() .split(":")[1] .strip() .replace(",", "") ) cmd2 = "rados -p default.rgw.buckets.index ls | grep %s" % b_id out = utils.exec_shell_cmd(cmd2) log.info("got output from sharing verification.--------") # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops["compression"]["enable"] is True: cmd = "radosgw-admin bucket stats --bucket=%s" % bucket.name out = utils.exec_shell_cmd(cmd) # print out bucket stats and verify in logs for compressed data by # comparing size_kb_utilized and size_kb_actual if config.test_ops["compression"]["enable"] is True: cmd = "radosgw-admin bucket stats --bucket=%s" % bucket.name out = utils.exec_shell_cmd(cmd) if config.test_ops["delete_bucket_object"] is True: reusable.delete_objects(bucket) time.sleep(10) reusable.check_sync_status() reusable.delete_bucket(bucket) # disable compression after test if config.test_ops["compression"]["enable"] is True: log.info("disable compression") cmd = "radosgw-admin zone get" out = utils.exec_shell_cmd(cmd) zone = json.loads(out) zone = zone.get("name") cmd = ( "radosgw-admin zone placement modify --rgw-zone=%s " "--placement-id=default-placement --compression=none" % zone ) out = utils.exec_shell_cmd(cmd) srv_restarted = rgw_service.restart() time.sleep(10) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") if config.gc_verification is True: final_op = reusable.verify_gc() if final_op != -1: test_info.failed_status("test failed") sys.exit(1) # check sync status if a multisite cluster reusable.check_sync_status() # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): test_info = AddTestInfo( 'create m buckets with n objects with bucket life cycle') io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # create user all_users_info = s3lib.create_users(config.user_count, config.cluster_name) for each_user in all_users_info: # authenticate auth = Auth(each_user) rgw_conn = auth.do_auth() rgw_conn2 = auth.do_auth_using_client() # create buckets if config.test_ops['create_bucket'] is True: log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name = utils.gen_bucket_name_from_userid( each_user['user_id'], rand_no=1) bucket = resuables.create_bucket(bucket_name, rgw_conn, each_user) if config.test_ops['create_object'] is True: # uploading data log.info('s3 objects to create: %s' % config.objects_count) for oc in range(config.objects_count): s3_object_name = utils.gen_s3_object_name( bucket.name, oc) resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user) bucket_life_cycle = s3lib.resource_op({ 'obj': rgw_conn, 'resource': 'BucketLifecycleConfiguration', 'args': [bucket.name] }) life_cycle = basic_lifecycle_config(prefix="key", days=20, id="rul1") put_bucket_life_cycle = s3lib.resource_op({ "obj": bucket_life_cycle, "resource": "put", "kwargs": dict(LifecycleConfiguration=life_cycle) }) log.info('put bucket life cycle:\n%s' % put_bucket_life_cycle) if put_bucket_life_cycle is False: raise TestExecError( "Resource execution failed: bucket creation faield" ) if put_bucket_life_cycle is not None: response = HttpResponseParser(put_bucket_life_cycle) if response.status_code == 200: log.info('bucket life cycle added') else: raise TestExecError( "bucket lifecycle addition failed") else: raise TestExecError("bucket lifecycle addition failed") log.info('trying to retrieve bucket lifecycle config') get_bucket_life_cycle_config = s3lib.resource_op({ "obj": rgw_conn2, "resource": 'get_bucket_lifecycle_configuration', "kwargs": dict(Bucket=bucket.name) }) if get_bucket_life_cycle_config is False: raise TestExecError( "bucket lifecycle config retrieval failed") if get_bucket_life_cycle_config is not None: response = HttpResponseParser( get_bucket_life_cycle_config) if response.status_code == 200: log.info('bucket life cycle retrieved') else: raise TestExecError( "bucket lifecycle config retrieval failed") else: raise TestExecError("bucket life cycle retrieved") test_info.success_status('test passed') sys.exit(0) except Exception, e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status('test failed') sys.exit(1)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) umgmt = UserMgmt() ceph_conf = CephConfOp() log.info(type(ceph_conf)) rgw_service = RGWService() # preparing data user_names = ["tuffy", "scooby", "max"] tenant = "tenant" tenant_user_info = umgmt.create_tenant_user( tenant_name=tenant, user_id=user_names[0], displayname=user_names[0] ) user_info = umgmt.create_subuser(tenant_name=tenant, user_id=user_names[0]) auth = Auth(user_info, config.ssl) rgw = auth.do_auth() for cc in range(config.container_count): if config.version_enable is True: log.info("making changes to ceph.conf") ceph_conf.set_to_ceph_conf( "global", ConfigOpts.rgw_swift_versioning_enabled, "True" ) log.info("trying to restart services ") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") container_name_old = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=str(cc) + "old" ) log.info(container_name_old) container = swiftlib.resource_op( { "obj": rgw, "resource": "put_container", "kwargs": dict(container=container_name_old), } ) container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=str(cc) + "new" ) log.info(container_name) container = swiftlib.resource_op( { "obj": rgw, "resource": "put_container", "args": [ container_name, {"X-Versions-Location": container_name_old}, ], } ) if container is False: raise TestExecError( "Resource execution failed: container creation failed" ) ls = [] swift_object_name = "" for version_count in range(config.version_count): for oc, size in list(config.mapped_sizes.items()): swift_object_name = fill_container( rgw, container_name, user_names[0], oc, cc, size ) ls = rgw.get_container(container_name_old) ls = list(ls) if config.copy_version_object is True: old_obj_name = ls[1][config.version_count - 2]["name"] log.info(old_obj_name) container = swiftlib.resource_op( { "obj": rgw, "resource": "copy_object", "kwargs": dict( container=container_name_old, obj=old_obj_name, destination=container_name + "/" + swift_object_name, ), } ) if container is False: raise TestExecError("Resource execution failed") log.info("Successfully copied item") else: current_count = "radosgw-admin bucket stats --uid={uid} --tenant={tenant} --bucket='{bucket}' ".format( uid=user_names[0], tenant=tenant, bucket=container_name ) num_obj_current = utils.exec_shell_cmd(current_count) num_obj_current = json.loads(num_obj_current) num_obj_current = ( num_obj_current[0].get("usage").get("rgw.main").get("num_objects") ) old_count = "radosgw-admin bucket stats --uid={uid} --tenant={tenant} --bucket='{bucket}' ".format( uid=user_names[0], tenant=tenant, bucket=container_name_old ) num_obj_old = utils.exec_shell_cmd(old_count) num_obj_old = json.loads(num_obj_old) num_obj_old = ( num_obj_old[0].get("usage").get("rgw.main").get("num_objects") ) version_count_from_config = ( config.objects_count * config.version_count ) - config.objects_count if (num_obj_current == config.objects_count) and ( num_obj_old == version_count_from_config ): log.info("objects and versioned obbjects are correct") else: test_info.failed_status("test failed") elif config.object_expire is True: container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc ) container = swiftlib.resource_op( {"obj": rgw, "resource": "put_container", "args": [container_name]} ) if container is False: raise TestExecError( "Resource execution failed: container creation failed" ) for oc, size in list(config.mapped_sizes.items()): swift_object_name = fill_container( rgw, container_name, user_names[0], oc, cc, size, header={"X-Delete-After": 5}, ) time.sleep(7) container_exists = swiftlib.resource_op( { "obj": rgw, "resource": "get_object", "args": [container_name, swift_object_name], } ) log.info(container_exists) if container_exists: msg = "test failed as the objects are still present" test_info.failed_status(msg) raise TestExecError(msg) elif config.large_object_upload is True: container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc ) container = swiftlib.resource_op( {"obj": rgw, "resource": "put_container", "args": [container_name]} ) if container is False: raise TestExecError( "Resource execution failed: container creation failed" ) for oc, size in list(config.mapped_sizes.items()): swift_object_name = fill_container( rgw, container_name, user_names[0], oc, cc, size, multipart=True, split_size=config.split_size, ) container_name_new = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=str(cc) + "New" ) container = swiftlib.resource_op( { "obj": rgw, "resource": "put_container", "kwargs": dict(container=container_name_new), } ) if container is False: raise TestExecError( "Resource execution failed: container creation failed" ) container = swiftlib.resource_op( { "obj": rgw, "resource": "put_object", "kwargs": dict( container=container_name_new, obj=swift_object_name, contents=None, headers={ "X-Object-Manifest": container_name + "/" + swift_object_name + "/" }, ), } ) if container is False: raise TestExecError( "Resource execution failed: container creation failed" ) if config.large_object_download is True: swift_old_object_path = os.path.join( TEST_DATA_PATH, swift_object_name ) swift_object_download_fname = swift_object_name + ".download" log.info("download object name: %s" % swift_object_download_fname) swift_object_download_path = os.path.join( TEST_DATA_PATH, swift_object_download_fname ) log.info("download object path: %s" % swift_object_download_path) swift_object_downloaded = rgw.get_object( container_name_new, swift_object_name ) with open(swift_object_download_path, "wb") as fp: fp.write(swift_object_downloaded[1]) old_object = utils.get_md5(swift_old_object_path) downloaded_obj = utils.get_md5(swift_object_download_path) log.info("s3_object_downloaded_md5: %s" % old_object) log.info("s3_object_uploaded_md5: %s" % downloaded_obj) if str(old_object) == str(downloaded_obj): log.info("md5 match") utils.exec_shell_cmd("rm -rf %s" % swift_object_download_path) else: raise TestExecError("md5 mismatch") else: container_name = utils.gen_bucket_name_from_userid( user_info["user_id"], rand_no=cc ) container = swiftlib.resource_op( {"obj": rgw, "resource": "put_container", "args": [container_name]} ) if container is False: raise TestExecError( "Resource execution failed: container creation failed" ) for oc, size in list(config.mapped_sizes.items()): swift_object_name = fill_container( rgw, container_name, user_names[0], oc, cc, size ) # download object swift_object_download_fname = swift_object_name + ".download" log.info("download object name: %s" % swift_object_download_fname) swift_object_download_path = os.path.join( TEST_DATA_PATH, swift_object_download_fname ) log.info("download object path: %s" % swift_object_download_path) swift_object_downloaded = rgw.get_object( container_name, swift_object_name ) with open(swift_object_download_path, "w") as fp: fp.write(str(swift_object_downloaded[1])) # modify and re-upload log.info("appending new message to test_data") message_to_append = "adding new msg after download" fp = open(swift_object_download_path, "a+") fp.write(message_to_append) fp.close() with open(swift_object_download_path, "r") as fp: rgw.put_object( container_name, swift_object_name, contents=fp.read(), content_type="text/plain", ) # delete object log.info("deleting swift object") rgw.delete_object(container_name, swift_object_name) # delete container log.info("deleting swift container") rgw.delete_container(container_name) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!") reusable.remove_user(tenant_user_info, tenant=tenant)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) write_bucket_io_info = BucketIoInfo() write_key_io_info = KeyIoInfo() version_count = 3 # create user s3_user = s3lib.create_users(1)[0] # authenticate auth = Auth(s3_user, ssl=config.ssl) rgw_conn = auth.do_auth() b1_name = 'bucky.1e' # bucket 1 b1_k1_name = b1_name + ".key.1" # key1 b1_k2_name = b1_name + ".key.2" # key2 b2_name = 'bucky.2e' # bucket 2 b2_k1_name = b2_name + ".key.1" # key1 b2_k2_name = b2_name + ".key.2" # key2 b1 = resuables.create_bucket(b1_name, rgw_conn, s3_user) b2 = resuables.create_bucket(b2_name, rgw_conn, s3_user) # enable versioning on b1 resuables.enable_versioning(b1, rgw_conn, s3_user, write_bucket_io_info) # upload object to version enabled bucket b1 obj_sizes = list(config.mapped_sizes.values()) config.obj_size = obj_sizes[0] for vc in range(version_count): resuables.upload_object(b1_k1_name, b1, TEST_DATA_PATH, config, s3_user, append_data=True, append_msg='hello vc count: %s' % str(vc)) # upload object to non version bucket b2 config.obj_size = obj_sizes[1] resuables.upload_object(b2_k1_name, b2, TEST_DATA_PATH, config, s3_user) # copy b2_k1 to b1 and check if version id is created, expectation: version id should be created # copy b1_k1 to b2 and check if version id is created, expectation: version id should not be present b1_k2 = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [b1.name, b1_k2_name]}) b2_k2 = s3lib.resource_op({'obj': rgw_conn, 'resource': 'Object', 'args': [b2.name, b2_k2_name]}) log.info('copy from b2_k1 key to b1_k2 key to bucket 1 -> version enabled bucket') copy_response = b1_k2.copy_from(CopySource={'Bucket': b2.name, 'Key': b2_k1_name, }) log.info('copy_response: %s' % copy_response) if copy_response is None: raise TestExecError("copy object failed") log.info('checking if copies object has version id created') b1_k2_version_id = b1_k2.version_id log.info('version id: %s' % b1_k2_version_id) if b1_k2_version_id is None: raise TestExecError('Version ID not created for the copied object on to the versioned enabled bucket') else: log.info('Version ID created for the copied object on to the versioned bucket') all_objects_in_b1 = b1.objects.all() log.info('all objects in bucket 1') for obj in all_objects_in_b1: log.info('object_name: %s' % obj.key) versions = b1.object_versions.filter(Prefix=obj.key) log.info('displaying all versions of the object') for version in versions: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) log.info('-------------------------------------------') log.info('copy from b1_k1 key to b2_k2 to bucket 2 -> non version bucket') copy_response = b2_k2.copy_from(CopySource={'Bucket': b1.name, 'Key': b1_k1_name, }) log.info('copy_response: %s' % copy_response) if copy_response is None: raise TestExecError("copy object failed") log.info('checking if copies object has version id created') b2_k2_version_id = b2_k2.version_id log.info('version id: %s' % b2_k2_version_id) if b2_k2_version_id is None: log.info('Version ID not created for the copied object on to the non versioned bucket') else: raise TestExecError('Version ID created for the copied object on to the non versioned bucket') all_objects_in_b2 = b2.objects.all() log.info('all objects in bucket 2') for obj in all_objects_in_b2: log.info('object_name: %s' % obj.key) versions = b2.object_versions.filter(Prefix=obj.key) log.info('displaying all versions of the object') for version in versions: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id))
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_config_set = CephConfOp() rgw_service = RGWService() # create users config.user_count = 2 users_info = s3lib.create_users(config.user_count) # user1 is the owner user1, user2 = users_info[0], users_info[1] log.info("adding sts config to ceph.conf") sesison_encryption_token = "abcdefghijklmnoq" ceph_config_set.set_to_ceph_conf("global", ConfigOpts.rgw_sts_key, sesison_encryption_token) ceph_config_set.set_to_ceph_conf("global", ConfigOpts.rgw_s3_auth_use_sts, True) srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") auth = Auth(user1) iam_client = auth.do_auth_iam_client() """ TODO: policy_document and role_policy can be used valid dict types. need to explore on this. """ policy_document = ( '{"Version":"2012-10-17",' '"Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/%s"]},' '"Action":["sts:AssumeRole"]}]}' % (user2["user_id"])) role_policy = ('{"Version":"2012-10-17",' '"Statement":{"Effect":"Allow",' '"Action":"s3:*",' '"Resource":"arn:aws:s3:::*"}}') add_caps_cmd = ( 'sudo radosgw-admin caps add --uid="{user_id}" --caps="roles=*"'. format(user_id=user1["user_id"])) utils.exec_shell_cmd(add_caps_cmd) # log.info(policy_document) role_name = f"S3RoleOf.{user1['user_id']}" log.info(f"role_name: {role_name}") log.info("creating role") create_role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, Path="/", RoleName=role_name, ) log.info("create_role_response") log.info(create_role_response) policy_name = f"policy.{user1['user_id']}" log.info(f"policy_name: {policy_name}") log.info("putting role policy") put_policy_response = iam_client.put_role_policy( RoleName=role_name, PolicyName=policy_name, PolicyDocument=role_policy) log.info("put_policy_response") log.info(put_policy_response) auth = Auth(user2) sts_client = auth.do_auth_sts_client() log.info("assuming role") assume_role_response = sts_client.assume_role( RoleArn=create_role_response["Role"]["Arn"], RoleSessionName=user1["user_id"], DurationSeconds=3600, ) log.info(assume_role_response) assumed_role_user_info = { "access_key": assume_role_response["Credentials"]["AccessKeyId"], "secret_key": assume_role_response["Credentials"]["SecretAccessKey"], "session_token": assume_role_response["Credentials"]["SessionToken"], "user_id": user2["user_id"], } log.info("got the credentials after assume role") s3client = Auth(assumed_role_user_info) s3_client_rgw = s3client.do_auth() io_info_initialize.initialize(basic_io_structure.initial()) write_user_info = AddUserInfo() basic_io_structure = BasicIOInfoStructure() user_info = basic_io_structure.user( **{ "user_id": assumed_role_user_info["user_id"], "access_key": assumed_role_user_info["access_key"], "secret_key": assumed_role_user_info["secret_key"], }) write_user_info.add_user_info(user_info) if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( assumed_role_user_info["user_id"], rand_no=bc) log.info("creating bucket with name: %s" % bucket_name_to_create) bucket = reusable.create_bucket(bucket_name_to_create, s3_client_rgw, assumed_role_user_info) if config.test_ops["create_object"] is True: # uploading data log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, assumed_role_user_info, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, assumed_role_user_info, ) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): test_info = AddTestInfo("Bucket Request Payer") io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) try: test_info.started_info() # create user all_users_info = s3lib.create_users(config.user_count) for each_user in all_users_info: # authenticate auth = Auth(each_user) rgw_conn = auth.do_auth() s3_object_names = [] # create buckets log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user["user_id"], rand_no=bc) log.info("creating bucket with name: %s" % bucket_name_to_create) # bucket = s3_ops.resource_op(rgw_conn, 'Bucket', bucket_name_to_create) bucket = resuables.create_bucket( bucket_name=bucket_name_to_create, rgw=rgw_conn, user_info=each_user) bucket_request_payer = s3lib.resource_op({ "obj": rgw_conn, "resource": "BucketRequestPayment", "args": [bucket.name], }) # change the bucket request payer to 'requester' payer = {"Payer": "Requester"} response = s3lib.resource_op({ "obj": bucket_request_payer, "resource": "put", "kwargs": dict(RequestPaymentConfiguration=payer), }) log.info(response) if response is not None: response = HttpResponseParser(response) if response.status_code == 200: log.info("bucket created") else: raise TestExecError( "bucket request payer modification failed") else: raise TestExecError( "bucket request payer modification failed") payer = bucket_request_payer.payer log.info("bucket request payer: %s" % payer) if payer != "Requester": TestExecError( "Request payer is not set or changed properly ") log.info("s3 objects to create: %s" % config.objects_count) if config.objects_count is not None: log.info("objects size range:\n%s" % config.objects_size_range) for oc in range(config.objects_count): s3_object_name = utils.gen_s3_object_name( bucket.name, oc) resuables.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user) test_info.success_status("test passed") sys.exit(0) except Exception as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status("test failed") sys.exit(1) except TestExecError as e: log.info(e) log.info(traceback.format_exc()) test_info.failed_status("test failed") sys.exit(1)
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_config_set = CephConfOp() rgw_service = RGWService() if config.sts is None: raise TestExecError("sts policies are missing in yaml config") # create users config.user_count = 2 users_info = s3lib.create_users(config.user_count) user1, user2 = users_info[0], users_info[1] log.info("adding sts config to ceph.conf") sesison_encryption_token = "abcdefghijklmnoq" ceph_config_set.set_to_ceph_conf( "global", ConfigOpts.rgw_sts_key, sesison_encryption_token ) ceph_config_set.set_to_ceph_conf("global", ConfigOpts.rgw_s3_auth_use_sts, "True") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") # Adding caps for user1 add_caps_cmd = ( 'sudo radosgw-admin caps add --uid="{user_id}" --caps="roles=*"'.format( user_id=user1["user_id"] ) ) utils.exec_shell_cmd(add_caps_cmd) # user1 auth with iam_client auth = Auth(user1, ssl=config.ssl) iam_client = auth.do_auth_iam_client() # policy document policy_document = json.dumps(config.sts["policy_document"]).replace(" ", "") policy_document = policy_document.replace("<user_name>", user2["user_id"]) print(policy_document) # role policy role_policy = json.dumps(config.sts["role_policy"]).replace(" ", "") print(role_policy) role_name = f"S3RoleOf.{user1['user_id']}" log.info(f"role_name: {role_name}") # role creation happens here log.info("creating role") create_role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, Path="/", RoleName=role_name, ) log.info("create_role_response") log.info(create_role_response) # Put role policy happening here policy_name = f"policy.{user1['user_id']}" log.info(f"policy_name: {policy_name}") log.info("putting role policy") put_policy_response = iam_client.put_role_policy( RoleName=role_name, PolicyName=policy_name, PolicyDocument=role_policy ) log.info("put_policy_response") log.info(put_policy_response) # bucket creation operations now bucket_name = "testbucket" + user1["user_id"] # authenticating user1 for bucket creation operation auth = Auth(user1, ssl=config.ssl) user1_info = { "access_key": user1["access_key"], "secret_key": user1["secret_key"], "user_id": user1["user_id"], } s3_client_u1 = auth.do_auth() # bucket creation operation bucket = reusable.create_bucket(bucket_name, s3_client_u1, user1_info) # uploading objects to the bucket if config.test_ops["create_object"]: # uploading data log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name(bucket_name, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, user1_info, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, user1_info, ) auth = Auth(user2, ssl=config.ssl) sts_client = auth.do_auth_sts_client() log.info("assuming role") assume_role_response = sts_client.assume_role( RoleArn=create_role_response["Role"]["Arn"], RoleSessionName=user1["user_id"], DurationSeconds=3600, ) log.info(assume_role_response) assumed_role_user_info = { "access_key": assume_role_response["Credentials"]["AccessKeyId"], "secret_key": assume_role_response["Credentials"]["SecretAccessKey"], "session_token": assume_role_response["Credentials"]["SessionToken"], "user_id": user2["user_id"], } log.info("got the credentials after assume role") s3client = Auth(assumed_role_user_info, ssl=config.ssl) s3_client = s3client.do_auth_using_client() io_info_initialize.initialize(basic_io_structure.initial()) write_user_info = AddUserInfo() basic_io_structure = BasicIOInfoStructure() user_info = basic_io_structure.user( **{ "user_id": assumed_role_user_info["user_id"], "access_key": assumed_role_user_info["access_key"], "secret_key": assumed_role_user_info["secret_key"], } ) write_user_info.add_user_info(user_info) unexisting_object = bucket_name + "_unexisting_object" try: response = s3_client.head_object(Bucket=bucket_name, Key=unexisting_object) except botocore.exceptions.ClientError as e: response_code = e.response["Error"]["Code"] log.info(response_code) if e.response["Error"]["Code"] == "404": log.info("404 Unexisting Object Not Found") elif e.response["Error"]["Code"] == "403": raise TestExecError("Error code : 403 - HeadObject operation: Forbidden")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) if config.test_ops.get("upload_type") == "multipart": srv_time_pre_op = get_svc_time() # create user config.user_count = 1 tenant1 = "MountEverest" tenant2 = "Himalayas" tenant1_user_info = s3lib.create_tenant_users( tenant_name=tenant1, no_of_users_to_create=config.user_count) tenant1_user1_info = tenant1_user_info[0] for each_user in tenant1_user_info: tenant1_user1_information = each_user tenant2_user_info = s3lib.create_tenant_users( tenant_name=tenant2, no_of_users_to_create=config.user_count) tenant2_user1_info = tenant2_user_info[0] tenant1_user1_auth = Auth(tenant1_user1_info, ssl=config.ssl) tenant2_user1_auth = Auth(tenant2_user1_info, ssl=config.ssl) rgw_tenant1_user1 = tenant1_user1_auth.do_auth() rgw_tenant1_user1_c = tenant1_user1_auth.do_auth_using_client() rgw_tenant2_user1 = tenant2_user1_auth.do_auth() rgw_tenant2_user1_c = tenant2_user1_auth.do_auth_using_client() bucket_name1 = utils.gen_bucket_name_from_userid( tenant1_user1_info["user_id"], rand_no=1) t1_u1_bucket1 = reusable.create_bucket( bucket_name1, rgw_tenant1_user1, tenant1_user1_info, ) bucket_name2 = utils.gen_bucket_name_from_userid( tenant1_user1_info["user_id"], rand_no=2) t1_u1_bucket2 = reusable.create_bucket( bucket_name2, rgw_tenant1_user1, tenant1_user1_info, ) bucket_policy_generated = s3_bucket_policy.gen_bucket_policy( tenants_list=[tenant1], userids_list=[tenant2_user1_info["user_id"]], actions_list=["CreateBucket"], resources=[t1_u1_bucket1.name], ) bucket_policy = json.dumps(bucket_policy_generated) log.info("jsoned policy:%s\n" % bucket_policy) log.info("bucket_policy_generated:%s\n" % bucket_policy_generated) bucket_policy_obj = s3lib.resource_op({ "obj": rgw_tenant1_user1, "resource": "BucketPolicy", "args": [t1_u1_bucket1.name], }) put_policy = s3lib.resource_op({ "obj": bucket_policy_obj, "resource": "put", "kwargs": dict(ConfirmRemoveSelfBucketAccess=True, Policy=bucket_policy), }) log.info("put policy response:%s\n" % put_policy) if put_policy is False: raise TestExecError( "Resource execution failed: bucket creation faield") if put_policy is not None: response = HttpResponseParser(put_policy) if response.status_code == 200 or response.status_code == 204: log.info("bucket policy created") else: raise TestExecError("bucket policy creation failed") else: raise TestExecError("bucket policy creation failed") if config.test_ops.get("upload_type") == "multipart": for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name(t1_u1_bucket1.name, oc) log.info("s3 objects to create: %s" % config.objects_count) reusable.upload_mutipart_object( s3_object_name, t1_u1_bucket1, TEST_DATA_PATH, config, tenant1_user1_information, ) srv_time_post_op = get_svc_time() log.info(srv_time_pre_op) log.info(srv_time_post_op) if srv_time_post_op > srv_time_pre_op: log.info("Service is running without crash") else: raise TestExecError("Service got crashed") # get policy get_policy = rgw_tenant1_user1_c.get_bucket_policy( Bucket=t1_u1_bucket1.name) log.info("got bucket policy:%s\n" % get_policy["Policy"]) # modifying bucket policy to take new policy if config.bucket_policy_op == "modify": # adding new action list: ListBucket to existing action: CreateBucket log.info("modifying buckey policy") actions_list = ["ListBucket", "CreateBucket"] actions = list(map(s3_bucket_policy.gen_action, actions_list)) bucket_policy2_generated = s3_bucket_policy.gen_bucket_policy( tenants_list=[tenant1], userids_list=[tenant2_user1_info["user_id"]], actions_list=actions_list, resources=[t1_u1_bucket1.name], ) bucket_policy2 = json.dumps(bucket_policy2_generated) put_policy = s3lib.resource_op({ "obj": bucket_policy_obj, "resource": "put", "kwargs": dict(ConfirmRemoveSelfBucketAccess=True, Policy=bucket_policy2), }) log.info("put policy response:%s\n" % put_policy) if put_policy is False: raise TestExecError( "Resource execution failed: bucket creation faield") if put_policy is not None: response = HttpResponseParser(put_policy) if response.status_code == 200 or response.status_code == 204: log.info("bucket policy created") else: raise TestExecError("bucket policy creation failed") else: raise TestExecError("bucket policy creation failed") get_modified_policy = rgw_tenant1_user1_c.get_bucket_policy( Bucket=t1_u1_bucket1.name) modified_policy = json.loads(get_modified_policy["Policy"]) log.info("got bucket policy:%s\n" % modified_policy) actions_list_from_modified_policy = modified_policy["Statement"][0][ "Action"] cleaned_actions_list_from_modified_policy = list( map(str, actions_list_from_modified_policy)) log.info("cleaned_actions_list_from_modified_policy: %s" % cleaned_actions_list_from_modified_policy) log.info("actions list to be modified: %s" % actions) cmp_val = utils.cmp(actions, cleaned_actions_list_from_modified_policy) log.info("cmp_val: %s" % cmp_val) if cmp_val != 0: raise TestExecError("modification of bucket policy failed ") if config.bucket_policy_op == "replace": log.info("replacing new bucket policy") new_policy_generated = s3_bucket_policy.gen_bucket_policy( tenants_list=[tenant1], userids_list=[tenant2_user1_info["user_id"]], actions_list=["ListBucket"], resources=[t1_u1_bucket2.name], ) new_policy = json.dumps(new_policy_generated) put_policy = s3lib.resource_op({ "obj": bucket_policy_obj, "resource": "put", "kwargs": dict(ConfirmRemoveSelfBucketAccess=True, Policy=new_policy), }) log.info("put policy response:%s\n" % put_policy) if put_policy is False: raise TestExecError( "Resource execution failed: bucket creation faield") if put_policy is not None: response = HttpResponseParser(put_policy) if response.status_code == 200 or response.status_code == 204: log.info("new bucket policy created") else: raise TestExecError("bucket policy creation failed") else: raise TestExecError("bucket policy creation failed") if config.bucket_policy_op == "delete": log.info("in delete bucket policy") delete_policy = s3lib.resource_op({ "obj": bucket_policy_obj, "resource": "delete", "args": None }) if delete_policy is False: raise TestExecError( "Resource execution failed: bucket creation faield") if delete_policy is not None: response = HttpResponseParser(delete_policy) if response.status_code == 200 or response.status_code == 204: log.info("bucket policy deleted") else: raise TestExecError("bucket policy deletion failed") else: raise TestExecError("bucket policy deletion failed") # confirming once again by calling get_bucket_policy try: rgw_tenant1_user1_c.get_bucket_policy(Bucket=t1_u1_bucket1.name) raise TestExecError("bucket policy did not get deleted") except boto3exception.ClientError as e: log.info(e.response) response = HttpResponseParser(e.response) if response.error["Code"] == "NoSuchBucketPolicy": log.info("bucket policy deleted") else: raise TestExecError("bucket policy did not get deleted") # log.info('get_policy after deletion: %s' % get_policy) # check sync status if a multisite cluster reusable.check_sync_status() # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) # create user all_users_info = s3lib.create_users(config.user_count) for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) rgw_conn = auth.do_auth() rgw_conn2 = auth.do_auth_using_client() # create buckets if config.test_ops['create_bucket'] is True: log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name = utils.gen_bucket_name_from_userid( each_user['user_id'], rand_no=1) bucket = reusable.create_bucket(bucket_name, rgw_conn, each_user) if config.test_ops['enable_versioning'] is True: log.info('bucket versionig test on bucket: %s' % bucket.name) # bucket_versioning = s3_ops.resource_op(rgw_conn, 'BucketVersioning', bucket.name) bucket_versioning = s3lib.resource_op({ 'obj': rgw_conn, 'resource': 'BucketVersioning', 'args': [bucket.name] }) version_status = s3lib.resource_op({ 'obj': bucket_versioning, 'resource': 'status', 'args': None }) if version_status is None: log.info('bucket versioning still not enabled') # enabling bucket versioning version_enable_status = s3lib.resource_op({ 'obj': bucket_versioning, 'resource': 'enable', 'args': None }) response = HttpResponseParser(version_enable_status) if response.status_code == 200: log.info('version enabled') else: raise TestExecError("version enable failed") if config.test_ops['create_object'] is True: # upload data for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket.name, oc) if config.test_ops['version_count'] > 0: for vc in range(config.test_ops['version_count']): log.info('version count for %s is %s' % (s3_object_name, str(vc))) log.info('modifying data: %s' % s3_object_name) reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user, append_data=True, append_msg='hello object for version: %s\n' % str(vc)) else: log.info('s3 objects to create: %s' % config.objects_count) reusable.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user) bucket_life_cycle = s3lib.resource_op({ 'obj': rgw_conn, 'resource': 'BucketLifecycleConfiguration', 'args': [bucket.name] }) life_cycle = basic_lifecycle_config(prefix="key", days=20, id="rul1") put_bucket_life_cycle = s3lib.resource_op({ "obj": bucket_life_cycle, "resource": "put", "kwargs": dict(LifecycleConfiguration=life_cycle) }) log.info('put bucket life cycle:\n%s' % put_bucket_life_cycle) if put_bucket_life_cycle is False: raise TestExecError( "Resource execution failed: bucket creation faield") if put_bucket_life_cycle is not None: response = HttpResponseParser(put_bucket_life_cycle) if response.status_code == 200: log.info('bucket life cycle added') else: raise TestExecError("bucket lifecycle addition failed") else: raise TestExecError("bucket lifecycle addition failed") log.info('trying to retrieve bucket lifecycle config') get_bucket_life_cycle_config = s3lib.resource_op({ "obj": rgw_conn2, "resource": 'get_bucket_lifecycle_configuration', "kwargs": dict(Bucket=bucket.name) }) if get_bucket_life_cycle_config is False: raise TestExecError( "bucket lifecycle config retrieval failed") if get_bucket_life_cycle_config is not None: response = HttpResponseParser(get_bucket_life_cycle_config) if response.status_code == 200: log.info('bucket life cycle retrieved') else: raise TestExecError( "bucket lifecycle config retrieval failed") else: raise TestExecError("bucket life cycle retrieved") if config.test_ops['create_object'] is True: for oc in range(config.objects_count): s3_object_name = utils.gen_s3_object_name( bucket.name, oc) if config.test_ops['version_count'] > 0: if config.test_ops.get('delete_versioned_object', None) is True: log.info( 'list all the versions of the object and delete the ' 'current version of the object') log.info('all versions for the object: %s\n' % s3_object_name) versions = bucket.object_versions.filter( Prefix=s3_object_name) t1 = [] for version in versions: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) t1.append(version.version_id) s3_object = s3lib.resource_op({ 'obj': rgw_conn, 'resource': 'Object', 'args': [bucket.name, s3_object_name] }) # log.info('object version to delete: %s -> %s' % (versions[0].object_key, # versions[0].version_id)) delete_response = s3_object.delete() log.info('delete response: %s' % delete_response) if delete_response['DeleteMarker'] is True: log.info( 'object delete marker is set to true') else: raise TestExecError( "'object delete marker is set to false" ) log.info( 'available versions for the object after delete marker is set' ) t2 = [] versions_after_delete_marker_is_set = bucket.object_versions.filter( Prefix=s3_object_name) for version in versions_after_delete_marker_is_set: log.info( 'key_name: %s --> version_id: %s' % (version.object_key, version.version_id)) t2.append(version.version_id) t2.pop() if t1 == t2: log.info('versions remained intact') else: raise TestExecError( 'versions are not intact after delete marker is set' ) # modify bucket lifecycle configuration, modify expiration days here for the test case. if config.test_ops.get('modify_lifecycle', False) is True: log.info('modifying lifecycle configuration') life_cycle_modifed = basic_lifecycle_config( prefix="key", days=15, id="rul1", status="Disabled") put_bucket_life_cycle = s3lib.resource_op({ "obj": bucket_life_cycle, "resource": "put", "kwargs": dict(LifecycleConfiguration=life_cycle_modifed) }) log.info('put bucket life cycle:\n%s' % put_bucket_life_cycle) if put_bucket_life_cycle is False: raise TestExecError( "Resource execution failed: bucket creation faield" ) if put_bucket_life_cycle is not None: response = HttpResponseParser(put_bucket_life_cycle) if response.status_code == 200: log.info('bucket life cycle added') else: raise TestExecError( "bucket lifecycle addition failed") else: raise TestExecError("bucket lifecycle addition failed") log.info('trying to retrieve bucket lifecycle config') get_bucket_life_cycle_config = s3lib.resource_op({ "obj": rgw_conn2, "resource": 'get_bucket_lifecycle_configuration', "kwargs": dict(Bucket=bucket.name) }) if get_bucket_life_cycle_config is False: raise TestExecError( "bucket lifecycle config retrieval failed") if get_bucket_life_cycle_config is not None: response = HttpResponseParser( get_bucket_life_cycle_config) modified_expiration_days = get_bucket_life_cycle_config[ 'Rules'][0]['Expiration']['Days'] log.info('modified expiration days: %s' % modified_expiration_days) if response.status_code == 200 and modified_expiration_days == 15: log.info( 'bucket life cycle retrieved after modifying') else: raise TestExecError( "bucket lifecycle config retrieval failed after modifying" ) else: raise TestExecError( "bucket lifecycle config retrieval failed after modifying" ) # disable bucket lifecycle configuration if config.test_ops.get('disable_lifecycle', False) is True: log.info('disabling lifecycle configuration') life_cycle_disabled_config = basic_lifecycle_config( prefix="key", days=20, id="rul1", status="Disabled") put_bucket_life_cycle = s3lib.resource_op({ "obj": bucket_life_cycle, "resource": "put", "kwargs": dict(LifecycleConfiguration=life_cycle_disabled_config) }) log.info('put bucket life cycle:\n%s' % put_bucket_life_cycle) if put_bucket_life_cycle is False: raise TestExecError( "Resource execution failed: bucket creation faield" ) if put_bucket_life_cycle is not None: response = HttpResponseParser(put_bucket_life_cycle) if response.status_code == 200: log.info('bucket life cycle added') else: raise TestExecError( "bucket lifecycle addition failed") else: raise TestExecError("bucket lifecycle addition failed") log.info('trying to retrieve bucket lifecycle config') get_bucket_life_cycle_config = s3lib.resource_op({ "obj": rgw_conn2, "resource": 'get_bucket_lifecycle_configuration', "kwargs": dict(Bucket=bucket.name) }) if get_bucket_life_cycle_config is False: raise TestExecError( "bucket lifecycle config retrieval failed") if get_bucket_life_cycle_config is not None: response = HttpResponseParser( get_bucket_life_cycle_config) if response.status_code == 200 and get_bucket_life_cycle_config[ 'Rules'][0]['Status'] == 'Disabled': log.info('disabled_status: %s' % get_bucket_life_cycle_config['Rules'][0] ['Status']) log.info( 'bucket life cycle retrieved after disabled') else: raise TestExecError( "bucket lifecycle config retrieval failed after disabled" ) else: raise TestExecError( "bucket lifecycle config retrieval failed after disabled" ) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() write_bucket_io_info = BucketIoInfo() io_info_initialize.initialize(basic_io_structure.initial()) ceph_conf = CephConfOp() rgw_service = RGWService() # create user all_users_info = s3lib.create_users(config.user_count) if config.test_ops.get('encryption_algorithm', None) is not None: log.info('encryption enabled, making ceph config changes') ceph_conf.set_to_ceph_conf('global', ConfigOpts.rgw_crypt_require_ssl, "false") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info('RGW service restarted') for each_user in all_users_info: # authenticate auth = Auth(each_user, ssl=config.ssl) if config.use_aws4 is True: rgw_conn = auth.do_auth(**{'signature_version': 's3v4'}) else: rgw_conn = auth.do_auth() objects_created_list = [] if config.test_ops['create_bucket'] is True: log.info('no of buckets to create: %s' % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( each_user['user_id'], rand_no=bc) log.info('creating bucket with name: %s' % bucket_name_to_create) bucket = reusable.create_bucket(bucket_name_to_create, rgw_conn, each_user) if config.test_ops.get('enable_version', False): log.info('enable bucket version') reusable.enable_versioning(bucket, rgw_conn, each_user, write_bucket_io_info) if config.test_ops['create_object'] is True: if config.test_ops['object_structure'] == 'flat': # uploading data log.info('top level s3 objects to create: %s' % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join( TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) if config.test_ops.get( 'upload_type') == 'multipart': log.info('upload type: multipart') reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user) else: log.info('upload type: normal') reusable.upload_object(s3_object_name, bucket, TEST_DATA_PATH, config, each_user) objects_created_list.append( (s3_object_name, s3_object_path)) #deleting the local file created after upload if config.local_file_delete is True: log.info( 'deleting local file created after the upload' ) utils.exec_shell_cmd('rm -rf %s' % s3_object_path) #this covers listing of a bucket with pseudo directories and objects in it ; Unable to list contents of large buckets https://bugzilla.redhat.com/show_bug.cgi?id=1874645#c72 if config.test_ops['object_structure'] == 'pseudo': log.info( f'pseudo directories to create {config.pseudo_dir_count} with {config.objects_count} objects in each' ) for count in range(config.pseudo_dir_count): s3_pseudo_dir_name = utils.gen_s3_object_name( bucket_name_to_create, count) s3_object_path = os.path.join( TEST_DATA_PATH, s3_pseudo_dir_name) manage_data.pseudo_dir_generator(s3_object_path) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_pseudo_object_name( s3_pseudo_dir_name, oc) log.info('s3 object name: %s' % s3_object_name) s3_object_path = os.path.join( TEST_DATA_PATH, s3_object_name) log.info('s3 object path: %s' % s3_object_path) if config.test_ops.get( 'upload_type') == 'multipart': log.info('upload type: multipart') reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user) else: log.info('upload type: normal') reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, each_user) #deleting the local file created after upload if config.local_file_delete is True: log.info( 'deleting local file created after the upload' ) utils.exec_shell_cmd('rm -rf %s' % s3_object_path) # listing bucket with only pseudo directories ; Bug allows ordered bucket listing to get stuck -- 4.1 https://bugzilla.redhat.com/show_bug.cgi?id=1853052#c0 if config.test_ops['create_object'] is False: if config.test_ops[ 'object_structure'] == 'pseudo-dir-only': log.info( f'pseudo directories to create {config.pseudo_dir_count}' ) for count in range(config.pseudo_dir_count): s3_pseudo_dir_name = utils.gen_s3_object_name( bucket_name_to_create, count) utils.create_psuedo_dir(s3_pseudo_dir_name, bucket) # radoslist listing of the bucket if config.test_ops['radoslist'] is True: log.info( 'executing the command radosgw-admin bucket radoslist ' ) radoslist = utils.exec_shell_cmd( "radosgw-admin bucket radoslist --bucket %s" % bucket_name_to_create) if radoslist is False: raise TestExecError( "Radoslist command execution failed") # get the configuration parameter cmd = 'ceph daemon `ls -t /var/run/ceph/ceph-client.rgw.*.asok|head -1` config show |grep rgw_bucket_index_max_aio' max_aio_output = utils.exec_shell_cmd(cmd) max_aio = max_aio_output.split()[1] # bucket stats to get the num_objects of the bucket bucket_stats = utils.exec_shell_cmd( "radosgw-admin bucket stats --bucket %s" % bucket_name_to_create) bucket_stats_json = json.loads(bucket_stats) bkt_num_objects = bucket_stats_json['usage']['rgw.main'][ 'num_objects'] # ordered listing via radosgw-admin command and noting time taken log.info( 'measure the execution time taken to list via radosgw-admin command' ) if config.test_ops['radosgw_listing_ordered'] is True: log.info('ordered listing via radosgw-admin command') rgw_cmd_time = reusable.time_to_list_via_radosgw( bucket_name_to_create, 'ordered') if rgw_cmd_time > 0: rgw_cmd_time_secs = "{:.4f}".format(rgw_cmd_time) rgw_cmd_time_mins = "{:.4f}".format(rgw_cmd_time / 60) log.info( f'with rgw_bucket_index_max_aio = {max_aio} time taken for ordered listing of {bkt_num_objects} objects is : {rgw_cmd_time_secs} secs ; {rgw_cmd_time_mins} mins' ) else: raise TestExecError( "object listing via radosgw-admin command failed") # unordered listing via radosgw-admin command and noting time taken if config.test_ops['radosgw_listing_ordered'] is False: log.info('unordered listing via radosgw-admin command') rgw_time = reusable.time_to_list_via_radosgw( bucket_name_to_create, 'unordered') if rgw_time > 0: rgw_time_secs = "{:.4f}".format(rgw_time) rgw_time_mins = "{:.4f}".format(rgw_time / 60) log.info( f'with rgw_bucket_index_max_aio = {max_aio} time taken for unordered listing of {bkt_num_objects} objects is : {rgw_time_secs} secs ; {rgw_time_mins} mins' ) else: raise TestExecError( "object listing via radosgw-admin command failed") # listing via boto and noting the time taken log.info('measure the execution time taken to list via boto') boto_time = reusable.time_to_list_via_boto( bucket_name_to_create, rgw_conn) if boto_time > 0: boto_time_secs = "{:.4f}".format(boto_time) boto_time_mins = "{:.4f}".format(boto_time / 60) log.info( f'with rgw_bucket_index_max_aio = {max_aio} time taken to list {bkt_num_objects} objects via boto : {boto_time_secs} secs ; {boto_time_mins} mins' ) else: raise TestExecError("object listing via boto failed") if config.test_ops.get('delete_bucket_object', False): if config.test_ops.get('enable_version', False): for name, path in objects_created_list: reusable.delete_version_object(bucket, name, path, rgw_conn, each_user) else: reusable.delete_objects(bucket) time.sleep(30) reusable.delete_bucket(bucket) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!") if config.user_remove is True: reusable.remove_user(each_user)