def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_config_set = CephConfOp() rgw_service = RGWService() if config.sts is None: raise TestExecError("sts policies are missing in yaml config") # create users config.user_count = 2 users_info = s3lib.create_users(config.user_count) user1, user2 = users_info[0], users_info[1] log.info("adding sts config to ceph.conf") sesison_encryption_token = "abcdefghijklmnoq" ceph_config_set.set_to_ceph_conf( "global", ConfigOpts.rgw_sts_key, sesison_encryption_token ) ceph_config_set.set_to_ceph_conf("global", ConfigOpts.rgw_s3_auth_use_sts, "True") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") # Adding caps for user1 add_caps_cmd = ( 'sudo radosgw-admin caps add --uid="{user_id}" --caps="roles=*"'.format( user_id=user1["user_id"] ) ) utils.exec_shell_cmd(add_caps_cmd) # user1 auth with iam_client auth = Auth(user1, ssl=config.ssl) iam_client = auth.do_auth_iam_client() # policy document policy_document = json.dumps(config.sts["policy_document"]).replace(" ", "") policy_document = policy_document.replace("<user_name>", user2["user_id"]) print(policy_document) # role policy role_policy = json.dumps(config.sts["role_policy"]).replace(" ", "") print(role_policy) role_name = f"S3RoleOf.{user1['user_id']}" log.info(f"role_name: {role_name}") # role creation happens here log.info("creating role") create_role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, Path="/", RoleName=role_name, ) log.info("create_role_response") log.info(create_role_response) # Put role policy happening here policy_name = f"policy.{user1['user_id']}" log.info(f"policy_name: {policy_name}") log.info("putting role policy") put_policy_response = iam_client.put_role_policy( RoleName=role_name, PolicyName=policy_name, PolicyDocument=role_policy ) log.info("put_policy_response") log.info(put_policy_response) # bucket creation operations now bucket_name = "testbucket" + user1["user_id"] # authenticating user1 for bucket creation operation auth = Auth(user1, ssl=config.ssl) user1_info = { "access_key": user1["access_key"], "secret_key": user1["secret_key"], "user_id": user1["user_id"], } s3_client_u1 = auth.do_auth() # bucket creation operation bucket = reusable.create_bucket(bucket_name, s3_client_u1, user1_info) # uploading objects to the bucket if config.test_ops["create_object"]: # uploading data log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name(bucket_name, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, user1_info, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, user1_info, ) auth = Auth(user2, ssl=config.ssl) sts_client = auth.do_auth_sts_client() log.info("assuming role") assume_role_response = sts_client.assume_role( RoleArn=create_role_response["Role"]["Arn"], RoleSessionName=user1["user_id"], DurationSeconds=3600, ) log.info(assume_role_response) assumed_role_user_info = { "access_key": assume_role_response["Credentials"]["AccessKeyId"], "secret_key": assume_role_response["Credentials"]["SecretAccessKey"], "session_token": assume_role_response["Credentials"]["SessionToken"], "user_id": user2["user_id"], } log.info("got the credentials after assume role") s3client = Auth(assumed_role_user_info, ssl=config.ssl) s3_client = s3client.do_auth_using_client() io_info_initialize.initialize(basic_io_structure.initial()) write_user_info = AddUserInfo() basic_io_structure = BasicIOInfoStructure() user_info = basic_io_structure.user( **{ "user_id": assumed_role_user_info["user_id"], "access_key": assumed_role_user_info["access_key"], "secret_key": assumed_role_user_info["secret_key"], } ) write_user_info.add_user_info(user_info) unexisting_object = bucket_name + "_unexisting_object" try: response = s3_client.head_object(Bucket=bucket_name, Key=unexisting_object) except botocore.exceptions.ClientError as e: response_code = e.response["Error"]["Code"] log.info(response_code) if e.response["Error"]["Code"] == "404": log.info("404 Unexisting Object Not Found") elif e.response["Error"]["Code"] == "403": raise TestExecError("Error code : 403 - HeadObject operation: Forbidden")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_config_set = CephConfOp() rgw_service = RGWService() if config.sts is None: raise TestExecError("sts policies are missing in yaml config") # create users config.user_count = 2 users_info = s3lib.create_users(config.user_count) # user1 is the owner user1, user2 = users_info[0], users_info[1] log.info("adding sts config to ceph.conf") sesison_encryption_token = "abcdefghijklmnoq" ceph_config_set.set_to_ceph_conf( "global", ConfigOpts.rgw_sts_key, sesison_encryption_token ) ceph_config_set.set_to_ceph_conf("global", ConfigOpts.rgw_s3_auth_use_sts, "True") srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") auth = Auth(user1, ssl=config.ssl) iam_client = auth.do_auth_iam_client() policy_document = json.dumps(config.sts["policy_document"]).replace(" ", "") policy_document = policy_document.replace("<user_name>", user2["user_id"]) role_policy = json.dumps(config.sts["role_policy"]).replace(" ", "") add_caps_cmd = ( 'sudo radosgw-admin caps add --uid="{user_id}" --caps="roles=*"'.format( user_id=user1["user_id"] ) ) utils.exec_shell_cmd(add_caps_cmd) role_name = f"S3RoleOf.{user1['user_id']}" log.info(f"role_name: {role_name}") log.info("creating role") create_role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, Path="/", RoleName=role_name, ) log.info("create_role_response") log.info(create_role_response) policy_name = f"policy.{user1['user_id']}" log.info(f"policy_name: {policy_name}") log.info("putting role policy") put_policy_response = iam_client.put_role_policy( RoleName=role_name, PolicyName=policy_name, PolicyDocument=role_policy ) log.info("put_policy_response") log.info(put_policy_response) auth = Auth(user2, ssl=config.ssl) sts_client = auth.do_auth_sts_client() log.info("assuming role") assume_role_response = sts_client.assume_role( RoleArn=create_role_response["Role"]["Arn"], RoleSessionName=user1["user_id"], DurationSeconds=3600, ) log.info(assume_role_response) assumed_role_user_info = { "access_key": assume_role_response["Credentials"]["AccessKeyId"], "secret_key": assume_role_response["Credentials"]["SecretAccessKey"], "session_token": assume_role_response["Credentials"]["SessionToken"], "user_id": user2["user_id"], } log.info("got the credentials after assume role") s3client = Auth(assumed_role_user_info, ssl=config.ssl) s3_client_rgw = s3client.do_auth() io_info_initialize.initialize(basic_io_structure.initial()) write_user_info = AddUserInfo() basic_io_structure = BasicIOInfoStructure() user_info = basic_io_structure.user( **{ "user_id": assumed_role_user_info["user_id"], "access_key": assumed_role_user_info["access_key"], "secret_key": assumed_role_user_info["secret_key"], } ) write_user_info.add_user_info(user_info) buckets_created = [] if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name = utils.gen_bucket_name_from_userid( assumed_role_user_info["user_id"], rand_no=bc ) log.info("creating bucket with name: %s" % bucket_name) bucket = reusable.create_bucket( bucket_name, s3_client_rgw, assumed_role_user_info ) buckets_created.append(bucket) if config.test_ops["create_object"] is True: for bucket in buckets_created: # uploading data log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name(bucket.name, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, assumed_role_user_info, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, assumed_role_user_info, ) if config.test_ops["server_side_copy"] is True: bucket1, bucket2 = buckets_created # copy object1 from bucket1 to bucket2 with the same name as in bucket1 log.info("copying first object from bucket1 to bucket2") all_keys_in_buck1 = [] for obj in bucket1.objects.all(): all_keys_in_buck1.append(obj.key) copy_source = {"Bucket": bucket1.name, "Key": all_keys_in_buck1[0]} copy_object_name = all_keys_in_buck1[0] + "_copied_obj" log.info(f"copy object name: {copy_object_name}") bucket2.copy(copy_source, copy_object_name) # list the objects in bucket2 log.info("listing all objects im bucket2 after copy") all_bucket2_objs = [] for obj in bucket2.objects.all(): log.info(obj.key) all_bucket2_objs.append(obj.key) # check for object existence in bucket2 if copy_object_name in all_bucket2_objs: log.info("server side copy successful") else: raise TestExecError("server side copy operation was not successful") # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")
def test_exec(config): io_info_initialize = IOInfoInitialize() basic_io_structure = BasicIOInfoStructure() io_info_initialize.initialize(basic_io_structure.initial()) ceph_config_set = CephConfOp() rgw_service = RGWService() # create users config.user_count = 2 users_info = s3lib.create_users(config.user_count) # user1 is the owner user1, user2 = users_info[0], users_info[1] log.info("adding sts config to ceph.conf") sesison_encryption_token = "abcdefghijklmnoq" ceph_config_set.set_to_ceph_conf("global", ConfigOpts.rgw_sts_key, sesison_encryption_token) ceph_config_set.set_to_ceph_conf("global", ConfigOpts.rgw_s3_auth_use_sts, True) srv_restarted = rgw_service.restart() time.sleep(30) if srv_restarted is False: raise TestExecError("RGW service restart failed") else: log.info("RGW service restarted") auth = Auth(user1) iam_client = auth.do_auth_iam_client() """ TODO: policy_document and role_policy can be used valid dict types. need to explore on this. """ policy_document = ( '{"Version":"2012-10-17",' '"Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/%s"]},' '"Action":["sts:AssumeRole"]}]}' % (user2["user_id"])) role_policy = ('{"Version":"2012-10-17",' '"Statement":{"Effect":"Allow",' '"Action":"s3:*",' '"Resource":"arn:aws:s3:::*"}}') add_caps_cmd = ( 'sudo radosgw-admin caps add --uid="{user_id}" --caps="roles=*"'. format(user_id=user1["user_id"])) utils.exec_shell_cmd(add_caps_cmd) # log.info(policy_document) role_name = f"S3RoleOf.{user1['user_id']}" log.info(f"role_name: {role_name}") log.info("creating role") create_role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, Path="/", RoleName=role_name, ) log.info("create_role_response") log.info(create_role_response) policy_name = f"policy.{user1['user_id']}" log.info(f"policy_name: {policy_name}") log.info("putting role policy") put_policy_response = iam_client.put_role_policy( RoleName=role_name, PolicyName=policy_name, PolicyDocument=role_policy) log.info("put_policy_response") log.info(put_policy_response) auth = Auth(user2) sts_client = auth.do_auth_sts_client() log.info("assuming role") assume_role_response = sts_client.assume_role( RoleArn=create_role_response["Role"]["Arn"], RoleSessionName=user1["user_id"], DurationSeconds=3600, ) log.info(assume_role_response) assumed_role_user_info = { "access_key": assume_role_response["Credentials"]["AccessKeyId"], "secret_key": assume_role_response["Credentials"]["SecretAccessKey"], "session_token": assume_role_response["Credentials"]["SessionToken"], "user_id": user2["user_id"], } log.info("got the credentials after assume role") s3client = Auth(assumed_role_user_info) s3_client_rgw = s3client.do_auth() io_info_initialize.initialize(basic_io_structure.initial()) write_user_info = AddUserInfo() basic_io_structure = BasicIOInfoStructure() user_info = basic_io_structure.user( **{ "user_id": assumed_role_user_info["user_id"], "access_key": assumed_role_user_info["access_key"], "secret_key": assumed_role_user_info["secret_key"], }) write_user_info.add_user_info(user_info) if config.test_ops["create_bucket"] is True: log.info("no of buckets to create: %s" % config.bucket_count) for bc in range(config.bucket_count): bucket_name_to_create = utils.gen_bucket_name_from_userid( assumed_role_user_info["user_id"], rand_no=bc) log.info("creating bucket with name: %s" % bucket_name_to_create) bucket = reusable.create_bucket(bucket_name_to_create, s3_client_rgw, assumed_role_user_info) if config.test_ops["create_object"] is True: # uploading data log.info("s3 objects to create: %s" % config.objects_count) for oc, size in list(config.mapped_sizes.items()): config.obj_size = size s3_object_name = utils.gen_s3_object_name( bucket_name_to_create, oc) log.info("s3 object name: %s" % s3_object_name) s3_object_path = os.path.join(TEST_DATA_PATH, s3_object_name) log.info("s3 object path: %s" % s3_object_path) if config.test_ops.get("upload_type") == "multipart": log.info("upload type: multipart") reusable.upload_mutipart_object( s3_object_name, bucket, TEST_DATA_PATH, config, assumed_role_user_info, ) else: log.info("upload type: normal") reusable.upload_object( s3_object_name, bucket, TEST_DATA_PATH, config, assumed_role_user_info, ) # check for any crashes during the execution crash_info = reusable.check_for_crash() if crash_info: raise TestExecError("ceph daemon crash found!")