def store_uploaded_file(f, name): img_dir = '/tmp/image/' if not os.path.exists(img_dir): os.mkdir(img_dir) file_name = id_generator() path = '%s%s' % (img_dir, file_name) with open(path, 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) calling_format = boto.s3.connection.OrdinaryCallingFormat() connection = boto.s3.connection.S3Connection( aws_access_key_id=settings.IMAGECRUD['access_key'], aws_secret_access_key=settings.IMAGECRUD['secret_key'], is_secure=False, host=settings.IMAGECRUD['s3_host'], port=settings.IMAGECRUD['s3_port'], calling_format=calling_format, path=settings.IMAGECRUD['s3_path']) try: bucket = connection.get_bucket(settings.IMAGECRUD['img_bucket']) except: bucket = connection.create_bucket(settings.IMAGECRUD['img_bucket']) key_name = '%s.jpg' % id_generator() key = bucket.new_key(key_name) key.set_contents_from_filename(path) key.set_canned_acl('public-read') key.close() return 'http://%s:%s%s/%s/%s' % ( settings.IMAGECRUD['s3_host'], settings.IMAGECRUD['s3_port'], settings.IMAGECRUD['s3_path'], settings.IMAGECRUD['img_bucket'], key_name)
def store_uploaded_file(f, name): img_dir='/tmp/image/' if not os.path.exists(img_dir): os.mkdir(img_dir) file_name = id_generator() path = '%s%s' % (img_dir,file_name) with open(path , 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) calling_format=boto.s3.connection.OrdinaryCallingFormat() connection = boto.s3.connection.S3Connection(aws_access_key_id=settings.IMAGECRUD['access_key'], aws_secret_access_key=settings.IMAGECRUD['secret_key'], is_secure=False, host=settings.IMAGECRUD['s3_host'], port=settings.IMAGECRUD['s3_port'], calling_format=calling_format, path=settings.IMAGECRUD['s3_path']) try: bucket = connection.get_bucket(settings.IMAGECRUD['img_bucket']) except: bucket = connection.create_bucket(settings.IMAGECRUD['img_bucket']) key_name= '%s.jpg' % id_generator() key = bucket.new_key(key_name) key.set_contents_from_filename(path) key.set_canned_acl('public-read') key.close() return 'http://%s:%s%s/%s/%s' % (settings.IMAGECRUD['s3_host'], settings.IMAGECRUD['s3_port'],settings.IMAGECRUD['s3_path'],settings.IMAGECRUD['img_bucket'],key_name)
def get_new_bucket(connection=None): """ Get a bucket that exists and is empty. Always recreates a bucket from scratch. This is useful to also reset ACLs and such. """ if connection is None: connection = s3.main name = get_new_bucket_name() # the only way for this to fail with a pre-existing bucket is if # someone raced us between setup nuke_prefixed_buckets and here; # ignore that as astronomically unlikely bucket = connection.create_bucket(name) return bucket
def get_new_bucket(target=None, name=None, headers=None): """ Get a bucket that exists and is empty. Always recreates a bucket from scratch. This is useful to also reset ACLs and such. """ if target is None: target = targets.main.default connection = target.connection if name is None: name = get_new_bucket_name() # the only way for this to fail with a pre-existing bucket is if # someone raced us between setup nuke_prefixed_buckets and here; # ignore that as astronomically unlikely bucket = connection.create_bucket(name, location=target.conf.api_name, headers=headers) return bucket
def test_bucket(): s3_access_key=os.environ['S3_ACCESS_KEY_ID'] s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY'] connection = boto.s3.connection.S3Connection( aws_access_key_id= s3_access_key, aws_secret_access_key=s3_secret_key, is_secure=False, port=8000, host='localhost', calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) print 'create a bucket' bucket = connection.create_bucket('bucket1') keys = ['mykey1', 'mykey2', 'mykey3', 'mykey4', 'obj1'] keys_to_delete = ['obj1'] for key in keys: obj = boto.s3.key.Key(bucket) obj.key = key obj.set_contents_from_string('This is ' + key) print 'bucket content:' for k in bucket.list(): print k for o in bucket.list(prefix='obj'): print(o.key) result = bucket.delete_keys(keys_to_delete) print result for o in bucket.list(prefix='obj'): print(o.key) print 'after delete_keys' for k in bucket.list(): print k for k in bucket.list(): bucket.delete_key(k) connection.delete_bucket(bucket)
def test_bucket_create_special_key_names(): s3_access_key=os.environ['S3_ACCESS_KEY_ID'] s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY'] connection = boto.s3.connection.S3Connection( aws_access_key_id= s3_access_key, aws_secret_access_key=s3_secret_key, is_secure=False, port=8000, host='localhost', calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) bucket = connection.create_bucket('version1') bucket.configure_versioning(True) objname = 'goodobj' i = 0 c = [] k = [] c.append('content-{i}'.format(i=i)) key = bucket.new_key(objname) key.set_contents_from_string(c[i]) key = bucket.get_key(objname) s = key.get_contents_as_string() print key, ' contents ' , s objname = '_testobj' key = bucket.new_key(objname) key.set_contents_from_string(c[i]) key = bucket.get_key(objname) s = key.get_contents_as_string() print key, ' contents ' , s
def test_bucket(): s3_access_key=os.environ['S3_ACCESS_KEY_ID'] s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY'] connection = boto.s3.connection.S3Connection( aws_access_key_id= s3_access_key, aws_secret_access_key=s3_secret_key, is_secure=False, port=8000, host='localhost', calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket_name = 'myfoo' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') # should be private already but guarantee it key.set_acl('private') print 'key=' + key.key acl = get_acl(key) print acl (err, out) = rgwadmin(['policy', '--bucket', bucket.name, '--object', key.key], check_status=True, format='xml') print "rgwadmin: ", err print out assert acl == out.strip('\n')
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ assert config is None or isinstance(config, list) \ or isinstance(config, dict), \ "task s3tests only supports a list or dictionary for configuration" all_clients = ['client.{id}'.format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] if config is None: config = all_clients if isinstance(config, list): config = dict.fromkeys(config) clients = config.keys() multi_region_run = rgw_utils.multi_region_enabled(ctx) client = clients[0]; # default choice, multi-region code may overwrite this if multi_region_run: client = rgw_utils.get_master_client(ctx, clients) # once the client is chosen, pull the host name and assigned port out of # the role_endpoints that were assigned by the rgw task (remote_host, remote_port) = ctx.rgw.role_endpoints[client] ## user1='foo' user2='fud' subuser1='foo:foo1' subuser2='foo:foo2' display_name1='Foo' display_name2='Fud' email='*****@*****.**' email2='*****@*****.**' access_key='9te6NH5mcdcq0Tc5i8i1' secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2='p5YnriCv1nAtykxBrupQ' secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name='myfoo' bucket_name2='mybar' # connect to rgw connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) connection2 = boto.s3.connection.S3Connection( aws_access_key_id=access_key2, aws_secret_access_key=secret_key2, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user1, '--display-name', display_name1, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4' ], check_status=True) # TESTCASE 'duplicate email','user','create','existing user email','fails' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email, ]) assert err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # this whole block should only be run if regions have been configured if multi_region_run: rgw_utils.radosgw_agent_sync_all(ctx) # post-sync, validate that user1 exists on the sync destination host for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): dest_client = c_config['dest'] (err, out) = rgwadmin(ctx, dest_client, ['metadata', 'list', 'user']) (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # compare the metadata between different regions, make sure it matches for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True) (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True) assert out1 == out2 # suspend a user on the master, then check the status on the destination for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err, out) = rgwadmin(ctx, source_client, ['user', 'suspend', '--uid', user1]) rgw_utils.radosgw_agent_sync_all(ctx) (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True) assert out['suspended'] # delete a user on the master, then check that it's gone on the destination for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err, out) = rgwadmin(ctx, source_client, ['user', 'rm', '--uid', user1], check_status=True) rgw_utils.radosgw_agent_sync_all(ctx) (err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user1]) assert out is None (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1]) assert out is None # then recreate it so later tests pass (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user1, '--display-name', display_name1, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4' ], check_status=True) # now do the multi-region bucket tests # Create a second user for the following tests (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '4' ], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user2], check_status=True) assert out is not None # create a bucket and do a sync bucket = connection.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) assert out1 == out2 # get the bucket.instance info and compare that src_bucket_id = out1['data']['bucket']['bucket_id'] dest_bucket_id = out2['data']['bucket']['bucket_id'] (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket.instance:{bucket_name}:{bucket_instance}'.format( bucket_name=bucket_name2,bucket_instance=src_bucket_id)], check_status=True) (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get', 'bucket.instance:{bucket_name}:{bucket_instance}'.format( bucket_name=bucket_name2,bucket_instance=dest_bucket_id)], check_status=True) assert out1 == out2 for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] # Attempt to create a new connection with user1 to the destination RGW # and use that to attempt a delete (that should fail) exception_encountered = False try: (dest_remote_host, dest_remote_port) = ctx.rgw.role_endpoints[dest_client] connection_dest = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=dest_remote_port, host=dest_remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # this should fail connection_dest.delete_bucket(bucket_name2) except boto.exception.S3ResponseError as e: assert e.status == 301 exception_encountered = True # confirm that the expected exception was seen assert exception_encountered # now delete the bucket on the source RGW and do another sync bucket.delete() rgw_utils.radosgw_agent_sync_all(ctx) # make sure that the bucket no longer exists in either region for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)]) (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)]) # Both of the previous calls should have errors due to requesting # metadata for non-existent buckets assert err1 assert err2 # create a bucket and then sync it bucket = connection.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) assert out1 == out2 # Now delete the bucket and recreate it with a different user # within the same window of time and then sync. bucket.delete() bucket = connection2.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches # user2 should own the bucket in both regions for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) assert out1 == out2 assert out1['data']['owner'] == user2 assert out1['data']['owner'] != user1 # now we're going to use this bucket to test meta-data update propagation for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] # get the metadata so we can tweak it (err, orig_data) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) # manually edit mtime for this bucket to be 300 seconds in the past new_data = copy.deepcopy(orig_data) new_data['mtime'] = orig_data['mtime'] - 300 assert new_data != orig_data (err, out) = rgwadmin(ctx, source_client, ['metadata', 'put', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], stdin=StringIO(json.dumps(new_data)), check_status=True) # get the metadata and make sure that the 'put' worked (err, out) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) assert out == new_data # sync to propagate the new metadata rgw_utils.radosgw_agent_sync_all(ctx) # get the metadata from the dest and compare it to what we just set # and what the source region has. (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) # yeah for the transitive property assert out1 == out2 assert out1 == new_data # now we delete the bucket bucket.delete() # Delete user2 as later tests do not expect it to exist. # Verify that it is gone on both regions for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err, out) = rgwadmin(ctx, source_client, ['user', 'rm', '--uid', user2], check_status=True) rgw_utils.radosgw_agent_sync_all(ctx) # The two 'user info' calls should fail and not return any data # since we just deleted this user. (err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user2]) assert out is None (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user2]) assert out is None # end of 'if multi_region_run:' # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--uid', user1, '--access-key', access_key2, '--secret', secret_key2, ], check_status=True) # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--uid', user1, '--access-key', access_key2, ], check_status=True) assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' subuser_access = 'full' subuser_perm = 'full-control' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser1, '--access', subuser_access ], check_status=True) # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'modify', '--subuser', subuser1, '--secret', swift_secret1, '--key-type', 'swift', ], check_status=True) # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert out['subusers'][0]['permissions'] == subuser_perm # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser2, '--secret', swift_secret2, '--key-type', 'swift', ], check_status=True) # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser1, '--key-type', 'swift', ], check_status=True) assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser1, ], check_status=True) assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser2, '--key-type', 'swift', '--purge-keys', ], check_status=True) assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 0 if multi_region_run: rgw_utils.radosgw_agent_sync_all(ctx) # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True) assert len(out) >= 1 assert bucket_name in out; # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4' bucket2 = connection.create_bucket(bucket_name + '2') bucket3 = connection.create_bucket(bucket_name + '3') bucket4 = connection.create_bucket(bucket_name + '4') # the 5th should fail. failed = False try: connection.create_bucket(bucket_name + '5') except Exception: failed = True assert failed # delete the buckets bucket2.delete() bucket3.delete() bucket4.delete() # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['owner'] == user1 bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0]['id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # reclaim it key.delete() # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name], check_status=True) # create a second user to link the bucket to (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '1', ], check_status=True) # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('two') except boto.exception.S3ResponseError: denied = True assert not denied # delete the object key.delete() # link the bucket to another user (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name], check_status=True) # try to remove user, should fail (has a linked bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2]) assert err # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked' (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name], check_status=True) # relink the bucket to the first user and delete the second user (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user1, '--bucket', bucket_name], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2], check_status=True) # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = 'four' key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) # now delete it (err, out) = rgwadmin(ctx, client, ['object', 'rm', '--bucket', bucket_name, '--object', object_name], check_status=True) # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True) assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' if obj[:4] == 'meta' or obj[:4] == 'data': continue (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj], check_status=True) assert len(log) > 0 # exempt bucket_name2 from checking as it was only used for multi-region tests assert log['bucket'].find(bucket_name) == 0 or log['bucket'].find(bucket_name2) == 0 assert log['bucket'] != bucket_name or log['bucket_id'] == bucket_id assert log['bucket_owner'] == user1 or log['bucket'] == bucket_name + '5' or log['bucket'] == bucket_name2 for entry in log['log_entries']: assert entry['bucket'] == log['bucket'] assert entry['user'] == user1 or log['bucket'] == bucket_name + '5' or log['bucket'] == bucket_name2 # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj], check_status=True) # TODO: show log by bucket+date # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj']) # last operation we did is delete obj, wait for it to flush if successful_ops(out) > 0: break; time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] total = user_summary['total'] assert total['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] for entry in user_summary['categories']: assert entry['successful_ops'] > 0 assert user_summary['user'] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'] for cat in test_categories: (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat], check_status=True) assert len(out['summary']) > 0 user_summary = out['summary'][0] assert user_summary['user'] == user1 assert len(user_summary['categories']) == 1 entry = user_summary['categories'][0] assert entry['category'] == cat assert entry['successful_ops'] > 0 # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1], check_status=True) (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) == 0 assert len(out['summary']) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('five') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('six') # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = 'foo'*10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) # now delete the head big_key.delete() # wait a bit to give the garbage collector time to cycle time.sleep(15) (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) > 0 # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage' (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True) #confirm (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) == 0 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # delete should fail because ``key`` still exists try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 key.delete() bucket.delete() # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') # should be private already but guarantee it key.set_acl('private') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key], check_status=True) acl = key.get_xml_acl() assert acl == out.strip('\n') # add another grantee by making the object public read key.set_acl('public-read') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key], check_status=True) acl = key.get_xml_acl() assert acl == out.strip('\n') # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) key_name = ['eight', 'nine', 'ten', 'eleven'] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) (err, out) = rgwadmin(ctx, client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'], check_status=True) # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps='user=read' (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps]) assert out['caps'][0]['perm'] == 'read' # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps]) assert not out['caps'] # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) key.set_contents_from_string('twelve') (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1, '--purge-data' ], check_status=True) # TESTCASE 'rm-user3','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule' # (err, out) = rgwadmin(ctx, client, ['zone', 'get']) orig_placement_pools = len(out['placement_pools']) # removed this test, it is not correct to assume that zone has default placement, it really # depends on how we set it up before # # assert len(out) > 0 # assert len(out['placement_pools']) == 1 # default_rule = out['placement_pools'][0] # assert default_rule['key'] == 'default-placement' rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}} out['placement_pools'].append(rule) (err, out) = rgwadmin(ctx, client, ['zone', 'set'], stdin=StringIO(json.dumps(out)), check_status=True) (err, out) = rgwadmin(ctx, client, ['zone', 'get']) assert len(out) > 0 assert len(out['placement_pools']) == orig_placement_pools + 1
finally: count_ins += 1 f.close() print("Successfully created %s instances" % (count_success_ins)) connection = boto.s3.connection.S3Connection( aws_access_key_id='adf938e8df4841129a5ca7089ab6b0ee', aws_secret_access_key='66f187d931f04af5805e899421bbfbef', port=8888, host='swift.rc.nectar.org.au', is_secure=True, validate_certs=False, calling_format=boto.s3.connection.OrdinaryCallingFormat() ) # buckets = connection.get_all_buckets() container_name = "twitter_container" try : b = connection.create_bucket(container_name) except boto.exception.S3CreateError as e: if e.status == 409: print ("already exists the container: %s" % (container_name)) pass except: print "Unexpected error:", sys.exc_info()[0] b = connection.get_bucket('twitter_container', validate=False) print ("Bucket: %s is ready" %(b.name))
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ global log assert config is None or isinstance(config, list) \ or isinstance(config, dict), \ "task s3tests only supports a list or dictionary for configuration" all_clients = [ 'client.{id}'.format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client') ] if config is None: config = all_clients if isinstance(config, list): config = dict.fromkeys(config) clients = config.keys() multi_region_run = rgw_utils.multi_region_enabled(ctx) client = clients[0] # default choice, multi-region code may overwrite this if multi_region_run: client = rgw_utils.get_master_client(ctx, clients) # once the client is chosen, pull the host name and assigned port out of # the role_endpoints that were assigned by the rgw task (remote_host, remote_port) = ctx.rgw.role_endpoints[client] ## user1 = 'foo' user2 = 'fud' subuser1 = 'foo:foo1' subuser2 = 'foo:foo2' display_name1 = 'Foo' display_name2 = 'Fud' email = '*****@*****.**' email2 = '*****@*****.**' access_key = '9te6NH5mcdcq0Tc5i8i1' secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2 = 'p5YnriCv1nAtykxBrupQ' secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name = 'myfoo' bucket_name2 = 'mybar' # connect to rgw connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) connection2 = boto.s3.connection.S3Connection( aws_access_key_id=access_key2, aws_secret_access_key=secret_key2, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin( ctx, client, [ 'user', 'create', '--uid', user1, '--display-name', display_name1, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4' ], check_status=True) # TESTCASE 'duplicate email','user','create','existing user email','fails' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email, ]) assert err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # this whole block should only be run if regions have been configured if multi_region_run: rgw_utils.radosgw_agent_sync_all(ctx) # post-sync, validate that user1 exists on the sync destination host for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): dest_client = c_config['dest'] (err, out) = rgwadmin(ctx, dest_client, ['metadata', 'list', 'user']) (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # compare the metadata between different regions, make sure it matches log.debug( 'compare the metadata between different regions, make sure it matches' ) for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin( ctx, source_client, ['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True) (err2, out2) = rgwadmin( ctx, dest_client, ['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True) assert out1 == out2 # suspend a user on the master, then check the status on the destination log.debug( 'suspend a user on the master, then check the status on the destination' ) for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err, out) = rgwadmin(ctx, source_client, ['user', 'suspend', '--uid', user1]) rgw_utils.radosgw_agent_sync_all(ctx) (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True) assert out['suspended'] # delete a user on the master, then check that it's gone on the destination log.debug( 'delete a user on the master, then check that it\'s gone on the destination' ) for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err, out) = rgwadmin(ctx, source_client, ['user', 'rm', '--uid', user1], check_status=True) rgw_utils.radosgw_agent_sync_all(ctx) (err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user1]) assert out is None (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1]) assert out is None # then recreate it so later tests pass (err, out) = rgwadmin( ctx, client, [ 'user', 'create', '--uid', user1, '--display-name', display_name1, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4' ], check_status=True) # now do the multi-region bucket tests log.debug('now do the multi-region bucket tests') # Create a second user for the following tests log.debug('Create a second user for the following tests') (err, out) = rgwadmin( ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '4' ], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user2], check_status=True) assert out is not None # create a bucket and do a sync log.debug('create a bucket and do a sync') bucket = connection.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches log.debug( 'compare the metadata for the bucket between different regions, make sure it matches' ) for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin( ctx, source_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) (err2, out2) = rgwadmin( ctx, dest_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) assert out1 == out2 # get the bucket.instance info and compare that src_bucket_id = out1['data']['bucket']['bucket_id'] dest_bucket_id = out2['data']['bucket']['bucket_id'] (err1, out1) = rgwadmin( ctx, source_client, [ 'metadata', 'get', 'bucket.instance:{bucket_name}:{bucket_instance}'.format( bucket_name=bucket_name2, bucket_instance=src_bucket_id) ], check_status=True) (err2, out2) = rgwadmin( ctx, dest_client, [ 'metadata', 'get', 'bucket.instance:{bucket_name}:{bucket_instance}'.format( bucket_name=bucket_name2, bucket_instance=dest_bucket_id) ], check_status=True) del out1['data']['bucket_info']['bucket']['pool'] del out1['data']['bucket_info']['bucket']['index_pool'] del out2['data']['bucket_info']['bucket']['pool'] del out2['data']['bucket_info']['bucket']['index_pool'] assert out1 == out2 same_region = 0 for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] source_region = rgw_utils.region_for_client(ctx, source_client) dest_region = rgw_utils.region_for_client(ctx, dest_client) # 301 is only returned for requests to something in a different region if source_region == dest_region: log.debug( '301 is only returned for requests to something in a different region' ) same_region += 1 continue # Attempt to create a new connection with user1 to the destination RGW log.debug( 'Attempt to create a new connection with user1 to the destination RGW' ) # and use that to attempt a delete (that should fail) exception_encountered = False try: (dest_remote_host, dest_remote_port) = ctx.rgw.role_endpoints[dest_client] connection_dest = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=dest_remote_port, host=dest_remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # this should fail connection_dest.delete_bucket(bucket_name2) except boto.exception.S3ResponseError as e: assert e.status == 301 exception_encountered = True # confirm that the expected exception was seen assert exception_encountered # now delete the bucket on the source RGW and do another sync log.debug( 'now delete the bucket on the source RGW and do another sync') bucket.delete() rgw_utils.radosgw_agent_sync_all(ctx) if same_region == len(ctx.radosgw_agent.config): bucket.delete() rgw_utils.radosgw_agent_sync_all(ctx) # make sure that the bucket no longer exists in either region log.debug( 'make sure that the bucket no longer exists in either region') for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin(ctx, source_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ]) (err2, out2) = rgwadmin(ctx, dest_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ]) # Both of the previous calls should have errors due to requesting # metadata for non-existent buckets assert err1 assert err2 # create a bucket and then sync it log.debug('create a bucket and then sync it') bucket = connection.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches log.debug( 'compare the metadata for the bucket between different regions, make sure it matches' ) for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin( ctx, source_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) (err2, out2) = rgwadmin( ctx, dest_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) assert out1 == out2 # Now delete the bucket and recreate it with a different user log.debug( 'Now delete the bucket and recreate it with a different user') # within the same window of time and then sync. bucket.delete() bucket = connection2.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches log.debug( 'compare the metadata for the bucket between different regions, make sure it matches' ) # user2 should own the bucket in both regions for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin( ctx, source_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) (err2, out2) = rgwadmin( ctx, dest_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) assert out1 == out2 assert out1['data']['owner'] == user2 assert out1['data']['owner'] != user1 # now we're going to use this bucket to test meta-data update propagation log.debug( 'now we\'re going to use this bucket to test meta-data update propagation' ) for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] # get the metadata so we can tweak it log.debug('get the metadata so we can tweak it') (err, orig_data) = rgwadmin( ctx, source_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) # manually edit mtime for this bucket to be 300 seconds in the past log.debug( 'manually edit mtime for this bucket to be 300 seconds in the past' ) new_data = copy.deepcopy(orig_data) new_data['mtime'] = orig_data['mtime'] - 300 assert new_data != orig_data (err, out) = rgwadmin( ctx, source_client, [ 'metadata', 'put', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], stdin=StringIO(json.dumps(new_data)), check_status=True) # get the metadata and make sure that the 'put' worked log.debug('get the metadata and make sure that the \'put\' worked') (err, out) = rgwadmin( ctx, source_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) assert out == new_data # sync to propagate the new metadata log.debug('sync to propagate the new metadata') rgw_utils.radosgw_agent_sync_all(ctx) # get the metadata from the dest and compare it to what we just set log.debug( 'get the metadata from the dest and compare it to what we just set' ) # and what the source region has. (err1, out1) = rgwadmin( ctx, source_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) (err2, out2) = rgwadmin( ctx, dest_client, [ 'metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2) ], check_status=True) # yeah for the transitive property assert out1 == out2 assert out1 == new_data # now we delete the bucket log.debug('now we delete the bucket') bucket.delete() log.debug('sync to propagate the deleted bucket') rgw_utils.radosgw_agent_sync_all(ctx) # Delete user2 as later tests do not expect it to exist. # Verify that it is gone on both regions for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err, out) = rgwadmin(ctx, source_client, ['user', 'rm', '--uid', user2], check_status=True) rgw_utils.radosgw_agent_sync_all(ctx) # The two 'user info' calls should fail and not return any data # since we just deleted this user. (err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user2]) assert out is None (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user2]) assert out is None # Test data sync # First create a bucket for data sync test purpose bucket = connection.create_bucket(bucket_name + 'data') # Create a tiny file and check if in sync for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): if c_config.get('metadata-only'): continue source_client = c_config['src'] dest_client = c_config['dest'] k = boto.s3.key.Key(bucket) k.key = 'tiny_file' k.set_contents_from_string("123456789") time.sleep(rgw_utils.radosgw_data_log_window(ctx, source_client)) rgw_utils.radosgw_agent_sync_all(ctx, data=True) (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client] dest_connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=dest_port, host=dest_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) dest_k = dest_connection.get_bucket(bucket_name + 'data').get_key('tiny_file') assert k.get_contents_as_string() == dest_k.get_contents_as_string( ) # check that deleting it removes it from the dest zone k.delete() time.sleep(rgw_utils.radosgw_data_log_window(ctx, source_client)) rgw_utils.radosgw_agent_sync_all(ctx, data=True) dest_bucket = dest_connection.get_bucket(bucket_name + 'data') dest_k = dest_bucket.get_key('tiny_file') assert dest_k == None, 'object not deleted from destination zone' # finally we delete the bucket bucket.delete() bucket = connection.create_bucket(bucket_name + 'data2') for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): if c_config.get('metadata-only'): continue source_client = c_config['src'] dest_client = c_config['dest'] (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client] dest_connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=dest_port, host=dest_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) for i in range(20): k = boto.s3.key.Key(bucket) k.key = 'tiny_file_' + str(i) k.set_contents_from_string(str(i) * 100) time.sleep(rgw_utils.radosgw_data_log_window(ctx, source_client)) rgw_utils.radosgw_agent_sync_all(ctx, data=True) for i in range(20): dest_k = dest_connection.get_bucket( bucket_name + 'data2').get_key('tiny_file_' + str(i)) assert (str(i) * 100) == dest_k.get_contents_as_string() k = boto.s3.key.Key(bucket) k.key = 'tiny_file_' + str(i) k.delete() # check that deleting removes the objects from the dest zone time.sleep(rgw_utils.radosgw_data_log_window(ctx, source_client)) rgw_utils.radosgw_agent_sync_all(ctx, data=True) for i in range(20): dest_bucket = dest_connection.get_bucket(bucket_name + 'data2') dest_k = dest_bucket.get_key('tiny_file_' + str(i)) assert dest_k == None, 'object %d not deleted from destination zone' % i bucket.delete() # end of 'if multi_region_run:' # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--uid', user1, '--access-key', access_key2, '--secret', secret_key2, ], check_status=True) # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][ 'access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][ 'secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--uid', user1, '--access-key', access_key2, ], check_status=True) assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' subuser_access = 'full' subuser_perm = 'full-control' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser1, '--access', subuser_access ], check_status=True) # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'modify', '--subuser', subuser1, '--secret', swift_secret1, '--key-type', 'swift', ], check_status=True) # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert out['subusers'][0]['permissions'] == subuser_perm # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser2, '--secret', swift_secret2, '--key-type', 'swift', ], check_status=True) # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][ 'user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[ 'swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser1, '--key-type', 'swift', ], check_status=True) assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser1, ], check_status=True) assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser2, '--key-type', 'swift', '--purge-keys', ], check_status=True) assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 0 if multi_region_run: rgw_utils.radosgw_agent_sync_all(ctx) # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True) assert len(out) >= 1 assert bucket_name in out # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4' bucket2 = connection.create_bucket(bucket_name + '2') bucket3 = connection.create_bucket(bucket_name + '3') bucket4 = connection.create_bucket(bucket_name + '4') # the 5th should fail. failed = False try: connection.create_bucket(bucket_name + '5') except Exception: failed = True assert failed # delete the buckets bucket2.delete() bucket3.delete() bucket4.delete() # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['owner'] == user1 bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0][ 'id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # reclaim it key.delete() # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (err, out) = rgwadmin( ctx, client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name], check_status=True) # create a second user to link the bucket to (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '1', ], check_status=True) # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('two') except boto.exception.S3ResponseError: denied = True assert not denied # delete the object key.delete() # link the bucket to another user (err, out) = rgwadmin( ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name], check_status=True) # try to remove user, should fail (has a linked bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2]) assert err # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked' (err, out) = rgwadmin( ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name], check_status=True) # relink the bucket to the first user and delete the second user (err, out) = rgwadmin( ctx, client, ['bucket', 'link', '--uid', user1, '--bucket', bucket_name], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2], check_status=True) # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = 'four' key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) # now delete it (err, out) = rgwadmin( ctx, client, ['object', 'rm', '--bucket', bucket_name, '--object', object_name], check_status=True) # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True) assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' if obj[:4] == 'meta' or obj[:4] == 'data': continue (err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj], check_status=True) assert len(rgwlog) > 0 # exempt bucket_name2 from checking as it was only used for multi-region tests assert rgwlog['bucket'].find( bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0 assert rgwlog['bucket'] != bucket_name or rgwlog[ 'bucket_id'] == bucket_id assert rgwlog['bucket_owner'] == user1 or rgwlog[ 'bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2 for entry in rgwlog['log_entries']: log.debug('checking log entry: ', entry) assert entry['bucket'] == rgwlog['bucket'] possible_buckets = [bucket_name + '5', bucket_name2] user = entry['user'] assert user == user1 or user.endswith('system-user') or \ rgwlog['bucket'] in possible_buckets # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj], check_status=True) # TODO: show log by bucket+date # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes (err, out) = rgwadmin( ctx, client, ['usage', 'show', '--categories', 'delete_obj' ]) # last operation we did is delete obj, wait for it to flush if successful_ops(out) > 0: break time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] total = user_summary['total'] assert total['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] for entry in user_summary['categories']: assert entry['successful_ops'] > 0 assert user_summary['user'] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = [ 'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket' ] for cat in test_categories: (err, out) = rgwadmin( ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat], check_status=True) assert len(out['summary']) > 0 user_summary = out['summary'][0] assert user_summary['user'] == user1 assert len(user_summary['categories']) == 1 entry = user_summary['categories'][0] assert entry['category'] == cat assert entry['successful_ops'] > 0 # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1], check_status=True) (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) == 0 assert len(out['summary']) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('five') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('six') # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = 'foo' * 10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) # now delete the head big_key.delete() # wait a bit to give the garbage collector time to cycle time.sleep(15) (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) > 0 # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage' (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True) #confirm (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) == 0 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # delete should fail because ``key`` still exists try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 key.delete() bucket.delete() # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') # should be private already but guarantee it key.set_acl('private') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key], check_status=True) acl = key.get_xml_acl() assert acl == out.strip('\n') # add another grantee by making the object public read key.set_acl('public-read') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key], check_status=True) acl = key.get_xml_acl() assert acl == out.strip('\n') # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) key_name = ['eight', 'nine', 'ten', 'eleven'] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) (err, out) = rgwadmin( ctx, client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'], check_status=True) # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps = 'user=read' (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps]) assert out['caps'][0]['perm'] == 'read' # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps]) assert not out['caps'] # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) key.set_contents_from_string('twelve') (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1, '--purge-data'], check_status=True) # TESTCASE 'rm-user3','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule' # (err, out) = rgwadmin(ctx, client, ['zone', 'get']) orig_placement_pools = len(out['placement_pools']) # removed this test, it is not correct to assume that zone has default placement, it really # depends on how we set it up before # # assert len(out) > 0 # assert len(out['placement_pools']) == 1 # default_rule = out['placement_pools'][0] # assert default_rule['key'] == 'default-placement' rule = { 'key': 'new-placement', 'val': { 'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2' } } out['placement_pools'].append(rule) (err, out) = rgwadmin(ctx, client, ['zone', 'set'], stdin=StringIO(json.dumps(out)), check_status=True) (err, out) = rgwadmin(ctx, client, ['zone', 'get']) assert len(out) > 0 assert len(out['placement_pools']) == orig_placement_pools + 1
(err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '4' ], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user2], check_status=True) assert out is not None # create a bucket and do a sync log.debug('create a bucket and do a sync') bucket = connection.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches log.debug('compare the metadata for the bucket between different regions, make sure it matches') for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config['src'] dest_client = c_config['dest'] (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)], check_status=True) log.debug('metadata 1 %r', out1) log.debug('metadata 2 %r', out2)
import argparse import os from boto.connection import AWSAuthConnection s3_access_key=os.environ['S3_ACCESS_KEY_ID'] s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY'] parser = argparse.ArgumentParser(description='s3_tenant_test') parser.add_argument('--port', type=int, action='store', default=8000 ) parser.add_argument('--key',type=str, action='store', default=s3_access_key) parser.add_argument('--secret',type=str, action='store', default=s3_secret_key) parser.add_argument('--tenant',type=str, action='store', default='testx') parser.add_argument('--bucket',type=str, action='store', default='bucket1') args = parser.parse_args() print "connecting" connection = boto.s3.connection.S3Connection( aws_access_key_id= args.key, aws_secret_access_key=args.secret, is_secure=False, port=args.port, host="localhost", calling_format = boto.s3.connection.OrdinaryCallingFormat()) print "bucket " + args.bucket + " tenant " + args.tenant bucket = connection.create_bucket(args.tenant + ":" + args.bucket) print bucket bucket = connection.get_bucket(args.tenant + ":" + args.bucket)
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ global log assert ( config is None or isinstance(config, list) or isinstance(config, dict) ), "task s3tests only supports a list or dictionary for configuration" all_clients = ["client.{id}".format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, "client")] if config is None: config = all_clients if isinstance(config, list): config = dict.fromkeys(config) clients = config.keys() multi_region_run = rgw_utils.multi_region_enabled(ctx) client = clients[0] # default choice, multi-region code may overwrite this if multi_region_run: client = rgw_utils.get_master_client(ctx, clients) # once the client is chosen, pull the host name and assigned port out of # the role_endpoints that were assigned by the rgw task (remote_host, remote_port) = ctx.rgw.role_endpoints[client] ## user1 = "foo" user2 = "fud" subuser1 = "foo:foo1" subuser2 = "foo:foo2" display_name1 = "Foo" display_name2 = "Fud" email = "*****@*****.**" email2 = "*****@*****.**" access_key = "9te6NH5mcdcq0Tc5i8i1" secret_key = "Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu" access_key2 = "p5YnriCv1nAtykxBrupQ" secret_key2 = "Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh" swift_secret1 = "gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL" swift_secret2 = "ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy" bucket_name = "myfoo" bucket_name2 = "mybar" # connect to rgw connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) connection2 = boto.s3.connection.S3Connection( aws_access_key_id=access_key2, aws_secret_access_key=secret_key2, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin( ctx, client, [ "user", "create", "--uid", user1, "--display-name", display_name1, "--email", email, "--access-key", access_key, "--secret", secret_key, "--max-buckets", "4", ], check_status=True, ) # TESTCASE 'duplicate email','user','create','existing user email','fails' (err, out) = rgwadmin( ctx, client, ["user", "create", "--uid", user2, "--display-name", display_name2, "--email", email] ) assert err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True) assert out["user_id"] == user1 assert out["email"] == email assert out["display_name"] == display_name1 assert len(out["keys"]) == 1 assert out["keys"][0]["access_key"] == access_key assert out["keys"][0]["secret_key"] == secret_key assert not out["suspended"] # this whole block should only be run if regions have been configured if multi_region_run: rgw_utils.radosgw_agent_sync_all(ctx) # post-sync, validate that user1 exists on the sync destination host for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): dest_client = c_config["dest"] (err, out) = rgwadmin(ctx, dest_client, ["metadata", "list", "user"]) (err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1], check_status=True) assert out["user_id"] == user1 assert out["email"] == email assert out["display_name"] == display_name1 assert len(out["keys"]) == 1 assert out["keys"][0]["access_key"] == access_key assert out["keys"][0]["secret_key"] == secret_key assert not out["suspended"] # compare the metadata between different regions, make sure it matches log.debug("compare the metadata between different regions, make sure it matches") for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] (err1, out1) = rgwadmin( ctx, source_client, ["metadata", "get", "user:{uid}".format(uid=user1)], check_status=True ) (err2, out2) = rgwadmin( ctx, dest_client, ["metadata", "get", "user:{uid}".format(uid=user1)], check_status=True ) assert out1 == out2 # suspend a user on the master, then check the status on the destination log.debug("suspend a user on the master, then check the status on the destination") for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] (err, out) = rgwadmin(ctx, source_client, ["user", "suspend", "--uid", user1]) rgw_utils.radosgw_agent_sync_all(ctx) (err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1], check_status=True) assert out["suspended"] # delete a user on the master, then check that it's gone on the destination log.debug("delete a user on the master, then check that it's gone on the destination") for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] (err, out) = rgwadmin(ctx, source_client, ["user", "rm", "--uid", user1], check_status=True) rgw_utils.radosgw_agent_sync_all(ctx) (err, out) = rgwadmin(ctx, source_client, ["user", "info", "--uid", user1]) assert out is None (err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1]) assert out is None # then recreate it so later tests pass (err, out) = rgwadmin( ctx, client, [ "user", "create", "--uid", user1, "--display-name", display_name1, "--email", email, "--access-key", access_key, "--secret", secret_key, "--max-buckets", "4", ], check_status=True, ) # now do the multi-region bucket tests log.debug("now do the multi-region bucket tests") # Create a second user for the following tests log.debug("Create a second user for the following tests") (err, out) = rgwadmin( ctx, client, [ "user", "create", "--uid", user2, "--display-name", display_name2, "--email", email2, "--access-key", access_key2, "--secret", secret_key2, "--max-buckets", "4", ], check_status=True, ) (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user2], check_status=True) assert out is not None # create a bucket and do a sync log.debug("create a bucket and do a sync") bucket = connection.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches log.debug("compare the metadata for the bucket between different regions, make sure it matches") for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] (err1, out1) = rgwadmin( ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) (err2, out2) = rgwadmin( ctx, dest_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) assert out1 == out2 # get the bucket.instance info and compare that src_bucket_id = out1["data"]["bucket"]["bucket_id"] dest_bucket_id = out2["data"]["bucket"]["bucket_id"] (err1, out1) = rgwadmin( ctx, source_client, [ "metadata", "get", "bucket.instance:{bucket_name}:{bucket_instance}".format( bucket_name=bucket_name2, bucket_instance=src_bucket_id ), ], check_status=True, ) (err2, out2) = rgwadmin( ctx, dest_client, [ "metadata", "get", "bucket.instance:{bucket_name}:{bucket_instance}".format( bucket_name=bucket_name2, bucket_instance=dest_bucket_id ), ], check_status=True, ) del out1["data"]["bucket_info"]["bucket"]["pool"] del out1["data"]["bucket_info"]["bucket"]["index_pool"] del out2["data"]["bucket_info"]["bucket"]["pool"] del out2["data"]["bucket_info"]["bucket"]["index_pool"] assert out1 == out2 same_region = 0 for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] source_region = rgw_utils.region_for_client(ctx, source_client) dest_region = rgw_utils.region_for_client(ctx, dest_client) # 301 is only returned for requests to something in a different region if source_region == dest_region: log.debug("301 is only returned for requests to something in a different region") same_region += 1 continue # Attempt to create a new connection with user1 to the destination RGW log.debug("Attempt to create a new connection with user1 to the destination RGW") # and use that to attempt a delete (that should fail) exception_encountered = False try: (dest_remote_host, dest_remote_port) = ctx.rgw.role_endpoints[dest_client] connection_dest = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=dest_remote_port, host=dest_remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # this should fail connection_dest.delete_bucket(bucket_name2) except boto.exception.S3ResponseError as e: assert e.status == 301 exception_encountered = True # confirm that the expected exception was seen assert exception_encountered # now delete the bucket on the source RGW and do another sync log.debug("now delete the bucket on the source RGW and do another sync") bucket.delete() rgw_utils.radosgw_agent_sync_all(ctx) if same_region == len(ctx.radosgw_agent.config): bucket.delete() rgw_utils.radosgw_agent_sync_all(ctx) # make sure that the bucket no longer exists in either region log.debug("make sure that the bucket no longer exists in either region") for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] (err1, out1) = rgwadmin( ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)] ) (err2, out2) = rgwadmin( ctx, dest_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)] ) # Both of the previous calls should have errors due to requesting # metadata for non-existent buckets assert err1 assert err2 # create a bucket and then sync it log.debug("create a bucket and then sync it") bucket = connection.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches log.debug("compare the metadata for the bucket between different regions, make sure it matches") for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] (err1, out1) = rgwadmin( ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) (err2, out2) = rgwadmin( ctx, dest_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) assert out1 == out2 # Now delete the bucket and recreate it with a different user log.debug("Now delete the bucket and recreate it with a different user") # within the same window of time and then sync. bucket.delete() bucket = connection2.create_bucket(bucket_name2) rgw_utils.radosgw_agent_sync_all(ctx) # compare the metadata for the bucket between different regions, make sure it matches log.debug("compare the metadata for the bucket between different regions, make sure it matches") # user2 should own the bucket in both regions for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] (err1, out1) = rgwadmin( ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) (err2, out2) = rgwadmin( ctx, dest_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) assert out1 == out2 assert out1["data"]["owner"] == user2 assert out1["data"]["owner"] != user1 # now we're going to use this bucket to test meta-data update propagation log.debug("now we're going to use this bucket to test meta-data update propagation") for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] # get the metadata so we can tweak it log.debug("get the metadata so we can tweak it") (err, orig_data) = rgwadmin( ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) # manually edit mtime for this bucket to be 300 seconds in the past log.debug("manually edit mtime for this bucket to be 300 seconds in the past") new_data = copy.deepcopy(orig_data) new_data["mtime"] = orig_data["mtime"] - 300 assert new_data != orig_data (err, out) = rgwadmin( ctx, source_client, ["metadata", "put", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], stdin=StringIO(json.dumps(new_data)), check_status=True, ) # get the metadata and make sure that the 'put' worked log.debug("get the metadata and make sure that the 'put' worked") (err, out) = rgwadmin( ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) assert out == new_data # sync to propagate the new metadata log.debug("sync to propagate the new metadata") rgw_utils.radosgw_agent_sync_all(ctx) # get the metadata from the dest and compare it to what we just set log.debug("get the metadata from the dest and compare it to what we just set") # and what the source region has. (err1, out1) = rgwadmin( ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) (err2, out2) = rgwadmin( ctx, dest_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)], check_status=True, ) # yeah for the transitive property assert out1 == out2 assert out1 == new_data # now we delete the bucket log.debug("now we delete the bucket") bucket.delete() log.debug("sync to propagate the deleted bucket") rgw_utils.radosgw_agent_sync_all(ctx) # Delete user2 as later tests do not expect it to exist. # Verify that it is gone on both regions for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): source_client = c_config["src"] dest_client = c_config["dest"] (err, out) = rgwadmin(ctx, source_client, ["user", "rm", "--uid", user2], check_status=True) rgw_utils.radosgw_agent_sync_all(ctx) # The two 'user info' calls should fail and not return any data # since we just deleted this user. (err, out) = rgwadmin(ctx, source_client, ["user", "info", "--uid", user2]) assert out is None (err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user2]) assert out is None # Test data sync # First create a bucket for data sync test purpose bucket = connection.create_bucket(bucket_name + "data") # Create a tiny file and check if in sync for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): if c_config.get("metadata-only"): continue for full in (True, False): source_client = c_config["src"] dest_client = c_config["dest"] k = boto.s3.key.Key(bucket) k.key = "tiny_file" k.set_contents_from_string("123456789") safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client) time.sleep(safety_window) rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full) (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client] dest_connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=dest_port, host=dest_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) dest_k = dest_connection.get_bucket(bucket_name + "data").get_key("tiny_file") assert k.get_contents_as_string() == dest_k.get_contents_as_string() # check that deleting it removes it from the dest zone k.delete() time.sleep(safety_window) # full sync doesn't handle deleted objects yet rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False) dest_bucket = dest_connection.get_bucket(bucket_name + "data") dest_k = dest_bucket.get_key("tiny_file") assert dest_k == None, "object not deleted from destination zone" # finally we delete the bucket bucket.delete() bucket = connection.create_bucket(bucket_name + "data2") for agent_client, c_config in ctx.radosgw_agent.config.iteritems(): if c_config.get("metadata-only"): continue for full in (True, False): source_client = c_config["src"] dest_client = c_config["dest"] (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client] dest_connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=dest_port, host=dest_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) for i in range(20): k = boto.s3.key.Key(bucket) k.key = "tiny_file_" + str(i) k.set_contents_from_string(str(i) * 100) safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client) time.sleep(safety_window) rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full) for i in range(20): dest_k = dest_connection.get_bucket(bucket_name + "data2").get_key("tiny_file_" + str(i)) assert (str(i) * 100) == dest_k.get_contents_as_string() k = boto.s3.key.Key(bucket) k.key = "tiny_file_" + str(i) k.delete() # check that deleting removes the objects from the dest zone time.sleep(safety_window) # full sync doesn't delete deleted objects yet rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False) for i in range(20): dest_bucket = dest_connection.get_bucket(bucket_name + "data2") dest_k = dest_bucket.get_key("tiny_file_" + str(i)) assert dest_k == None, "object %d not deleted from destination zone" % i bucket.delete() # end of 'if multi_region_run:' # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ["user", "suspend", "--uid", user1], check_status=True) # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True) assert out["suspended"] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ["user", "enable", "--uid", user1], check_status=True) # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True) assert not out["suspended"] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin( ctx, client, ["key", "create", "--uid", user1, "--access-key", access_key2, "--secret", secret_key2], check_status=True, ) # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True) assert len(out["keys"]) == 2 assert out["keys"][0]["access_key"] == access_key2 or out["keys"][1]["access_key"] == access_key2 assert out["keys"][0]["secret_key"] == secret_key2 or out["keys"][1]["secret_key"] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, ["key", "rm", "--uid", user1, "--access-key", access_key2], check_status=True) assert len(out["keys"]) == 1 assert out["keys"][0]["access_key"] == access_key assert out["keys"][0]["secret_key"] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' subuser_access = "full" subuser_perm = "full-control" (err, out) = rgwadmin( ctx, client, ["subuser", "create", "--subuser", subuser1, "--access", subuser_access], check_status=True ) # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin( ctx, client, ["subuser", "modify", "--subuser", subuser1, "--secret", swift_secret1, "--key-type", "swift"], check_status=True, ) # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1]) assert out["subusers"][0]["permissions"] == subuser_perm # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True) assert len(out["swift_keys"]) == 1 assert out["swift_keys"][0]["user"] == subuser1 assert out["swift_keys"][0]["secret_key"] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin( ctx, client, ["subuser", "create", "--subuser", subuser2, "--secret", swift_secret2, "--key-type", "swift"], check_status=True, ) # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True) assert len(out["swift_keys"]) == 2 assert out["swift_keys"][0]["user"] == subuser2 or out["swift_keys"][1]["user"] == subuser2 assert out["swift_keys"][0]["secret_key"] == swift_secret2 or out["swift_keys"][1]["secret_key"] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, ["key", "rm", "--subuser", subuser1, "--key-type", "swift"], check_status=True) assert len(out["swift_keys"]) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (err, out) = rgwadmin(ctx, client, ["subuser", "rm", "--subuser", subuser1], check_status=True) assert len(out["subusers"]) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (err, out) = rgwadmin( ctx, client, ["subuser", "rm", "--subuser", subuser2, "--key-type", "swift", "--purge-keys"], check_status=True ) assert len(out["swift_keys"]) == 0 assert len(out["subusers"]) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--uid", user1], check_status=True) assert len(out) == 0 if multi_region_run: rgw_utils.radosgw_agent_sync_all(ctx) # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ["bucket", "list", "--uid", user1], check_status=True) assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ["bucket", "list", "--uid", user1], check_status=True) assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ["bucket", "list"], check_status=True) assert len(out) >= 1 assert bucket_name in out # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4' bucket2 = connection.create_bucket(bucket_name + "2") bucket3 = connection.create_bucket(bucket_name + "3") bucket4 = connection.create_bucket(bucket_name + "4") # the 5th should fail. failed = False try: connection.create_bucket(bucket_name + "5") except Exception: failed = True assert failed # delete the buckets bucket2.delete() bucket3.delete() bucket4.delete() # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True) assert out["owner"] == user1 bucket_id = out["id"] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--uid", user1], check_status=True) assert len(out) == 1 assert out[0]["id"] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string("one") # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True) assert out["id"] == bucket_id assert out["usage"]["rgw.main"]["num_objects"] == 1 assert out["usage"]["rgw.main"]["size_kb"] > 0 # reclaim it key.delete() # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (err, out) = rgwadmin(ctx, client, ["bucket", "unlink", "--uid", user1, "--bucket", bucket_name], check_status=True) # create a second user to link the bucket to (err, out) = rgwadmin( ctx, client, [ "user", "create", "--uid", user2, "--display-name", display_name2, "--access-key", access_key2, "--secret", secret_key2, "--max-buckets", "1", ], check_status=True, ) # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string("two") except boto.exception.S3ResponseError: denied = True assert not denied # delete the object key.delete() # link the bucket to another user (err, out) = rgwadmin(ctx, client, ["metadata", "get", "bucket:{n}".format(n=bucket_name)], check_status=True) bucket_data = out["data"] assert bucket_data["bucket"]["name"] == bucket_name bucket_id = bucket_data["bucket"]["bucket_id"] # link the bucket to another user (err, out) = rgwadmin( ctx, client, ["bucket", "link", "--uid", user2, "--bucket", bucket_name, "--bucket-id", bucket_id], check_status=True, ) # try to remove user, should fail (has a linked bucket) (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user2]) assert err # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked' (err, out) = rgwadmin(ctx, client, ["bucket", "unlink", "--uid", user2, "--bucket", bucket_name], check_status=True) # relink the bucket to the first user and delete the second user (err, out) = rgwadmin( ctx, client, ["bucket", "link", "--uid", user1, "--bucket", bucket_name, "--bucket-id", bucket_id], check_status=True, ) (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user2], check_status=True) # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = "four" key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) # now delete it (err, out) = rgwadmin( ctx, client, ["object", "rm", "--bucket", bucket_name, "--object", object_name], check_status=True ) # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True) assert out["id"] == bucket_id assert out["usage"]["rgw.main"]["num_objects"] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ["log", "list"], check_status=True) assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' if obj[:4] == "meta" or obj[:4] == "data": continue (err, rgwlog) = rgwadmin(ctx, client, ["log", "show", "--object", obj], check_status=True) assert len(rgwlog) > 0 # exempt bucket_name2 from checking as it was only used for multi-region tests assert rgwlog["bucket"].find(bucket_name) == 0 or rgwlog["bucket"].find(bucket_name2) == 0 assert rgwlog["bucket"] != bucket_name or rgwlog["bucket_id"] == bucket_id assert ( rgwlog["bucket_owner"] == user1 or rgwlog["bucket"] == bucket_name + "5" or rgwlog["bucket"] == bucket_name2 ) for entry in rgwlog["log_entries"]: log.debug("checking log entry: ", entry) assert entry["bucket"] == rgwlog["bucket"] possible_buckets = [bucket_name + "5", bucket_name2] user = entry["user"] assert user == user1 or user.endswith("system-user") or rgwlog["bucket"] in possible_buckets # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ["log", "rm", "--object", obj], check_status=True) # TODO: show log by bucket+date # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes (err, out) = rgwadmin( ctx, client, ["usage", "show", "--categories", "delete_obj"] ) # last operation we did is delete obj, wait for it to flush if successful_ops(out) > 0: break time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ["usage", "show"], check_status=True) assert len(out["entries"]) > 0 assert len(out["summary"]) > 0 user_summary = out["summary"][0] total = user_summary["total"] assert total["successful_ops"] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1], check_status=True) assert len(out["entries"]) > 0 assert len(out["summary"]) > 0 user_summary = out["summary"][0] for entry in user_summary["categories"]: assert entry["successful_ops"] > 0 assert user_summary["user"] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = ["create_bucket", "put_obj", "delete_obj", "delete_bucket"] for cat in test_categories: (err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1, "--categories", cat], check_status=True) assert len(out["summary"]) > 0 user_summary = out["summary"][0] assert user_summary["user"] == user1 assert len(user_summary["categories"]) == 1 entry = user_summary["categories"][0] assert entry["category"] == cat assert entry["successful_ops"] > 0 # the usage flush interval is 30 seconds, wait that much an then some # to make sure everything has been flushed time.sleep(35) # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (err, out) = rgwadmin(ctx, client, ["usage", "trim", "--uid", user1], check_status=True) (err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1], check_status=True) assert len(out["entries"]) == 0 assert len(out["summary"]) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ["user", "suspend", "--uid", user1], check_status=True) # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string("five") except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ["user", "enable", "--uid", user1], check_status=True) # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string("six") # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = "foo" * 10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) # now delete the head big_key.delete() # wait a bit to give the garbage collector time to cycle time.sleep(15) (err, out) = rgwadmin(ctx, client, ["gc", "list"]) assert len(out) > 0 # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage' (err, out) = rgwadmin(ctx, client, ["gc", "process"], check_status=True) # confirm (err, out) = rgwadmin(ctx, client, ["gc", "list"]) assert len(out) == 0 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1]) assert err # delete should fail because ``key`` still exists try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 key.delete() bucket.delete() # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string("seven") # should be private already but guarantee it key.set_acl("private") (err, out) = rgwadmin(ctx, client, ["policy", "--bucket", bucket.name, "--object", key.key], check_status=True) acl = key.get_xml_acl() assert acl == out.strip("\n") # add another grantee by making the object public read key.set_acl("public-read") (err, out) = rgwadmin(ctx, client, ["policy", "--bucket", bucket.name, "--object", key.key], check_status=True) acl = key.get_xml_acl() assert acl == out.strip("\n") # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) key_name = ["eight", "nine", "ten", "eleven"] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) (err, out) = rgwadmin(ctx, client, ["bucket", "rm", "--bucket", bucket_name, "--purge-objects"], check_status=True) # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps = "user=read" (err, out) = rgwadmin(ctx, client, ["caps", "add", "--uid", user1, "--caps", caps]) assert out["caps"][0]["perm"] == "read" # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (err, out) = rgwadmin(ctx, client, ["caps", "rm", "--uid", user1, "--caps", caps]) assert not out["caps"] # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1]) assert err # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) key.set_contents_from_string("twelve") (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1, "--purge-data"], check_status=True) # TESTCASE 'rm-user3','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1]) assert err # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule' # (err, out) = rgwadmin(ctx, client, ["zone", "get"]) orig_placement_pools = len(out["placement_pools"]) # removed this test, it is not correct to assume that zone has default placement, it really # depends on how we set it up before # # assert len(out) > 0 # assert len(out['placement_pools']) == 1 # default_rule = out['placement_pools'][0] # assert default_rule['key'] == 'default-placement' rule = {"key": "new-placement", "val": {"data_pool": ".rgw.buckets.2", "index_pool": ".rgw.buckets.index.2"}} out["placement_pools"].append(rule) (err, out) = rgwadmin(ctx, client, ["zone", "set"], stdin=StringIO(json.dumps(out)), check_status=True) (err, out) = rgwadmin(ctx, client, ["zone", "get"]) assert len(out) > 0 assert len(out["placement_pools"]) == orig_placement_pools + 1
def task(ctx, config): """ Test radosgw-admin functionality through the RESTful interface """ assert config is None or isinstance(config, list) \ or isinstance(config, dict), \ "task s3tests only supports a list or dictionary for configuration" all_clients = ['client.{id}'.format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] if config is None: config = all_clients if isinstance(config, list): config = dict.fromkeys(config) clients = config.keys() # just use the first client... client = clients[0] ## admin_user = '******' admin_display_name = 'Ms. Admin User' admin_access_key = 'MH1WC2XQ1S8UISFDZC8W' admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG' admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write' user1 = 'foo' user2 = 'fud' subuser1 = 'foo:foo1' subuser2 = 'foo:foo2' display_name1 = 'Foo' display_name2 = 'Fud' email = '*****@*****.**' access_key = '9te6NH5mcdcq0Tc5i8i1' secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2 = 'p5YnriCv1nAtykxBrupQ' secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name = 'myfoo' # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'create-admin-user','user','create','administrative user','succeeds' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', admin_user, '--display-name', admin_display_name, '--access-key', admin_access_key, '--secret', admin_secret_key, '--max-buckets', '0', '--caps', admin_caps ]) logging.error(out) logging.error(err) assert not err (remote,) = ctx.cluster.only(client).remotes.iterkeys() remote_host = remote.name.split('@')[1] admin_conn = boto.s3.connection.S3Connection( aws_access_key_id=admin_access_key, aws_secret_access_key=admin_secret_key, is_secure=False, port=7280, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1}) assert ret == 404 # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'create'], {'uid' : user1, 'display-name' : display_name1, 'email' : email, 'access-key' : access_key, 'secret-key' : secret_key, 'max-buckets' : '4' }) assert ret == 200 # TESTCASE 'info-existing','user','info','existing user','returns correct info' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True}) assert ret == 200 # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert ret == 200 assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'}) assert not err # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert ret == 200 assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['key', 'create'], {'uid' : user1, 'access-key' : access_key2, 'secret-key' : secret_key2 }) assert ret == 200 # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert ret == 200 assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (ret, out) = rgwadmin_rest(admin_conn, ['key', 'rm'], {'uid' : user1, 'access-key' : access_key2 }) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'create'], {'subuser' : subuser1, 'secret-key' : swift_secret1, 'key-type' : 'swift' }) assert ret == 200 # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert ret == 200 assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'create'], {'subuser' : subuser2, 'secret-key' : swift_secret2, 'key-type' : 'swift' }) assert ret == 200 # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert ret == 200 assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (ret, out) = rgwadmin_rest(admin_conn, ['key', 'rm'], {'subuser' : subuser1, 'key-type' :'swift' }) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'rm'], {'subuser' : subuser1 }) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'rm'], {'subuser' : subuser2, 'key-type' : 'swift', '{purge-keys' :True }) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1}) assert ret == 200 assert len(out) == 0 # connect to rgw connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=7280, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True}) assert ret == 200 assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1}) assert ret == 200 assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True}) assert ret == 200 assert out['owner'] == user1 bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True}) assert ret == 200 assert len(out) == 1 assert out[0]['id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True}) assert ret == 200 assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # reclaim it key.delete() # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], {'uid' : user1, 'bucket' : bucket_name}) assert ret == 200 # create a second user to link the bucket to (ret, out) = rgwadmin_rest(admin_conn, ['user', 'create'], {'uid' : user2, 'display-name' : display_name2, 'access-key' : access_key2, 'secret-key' : secret_key2, 'max-buckets' : '1', }) assert ret == 200 # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('two') except boto.exception.S3ResponseError: denied = True assert not denied # delete the object key.delete() # link the bucket to another user (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {'uid' : user2, 'bucket' : bucket_name}) assert ret == 200 # try creating an object with the first user which should cause an error key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('three') except boto.exception.S3ResponseError: denied = True assert denied # relink the bucket to the first user and delete the second user (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {'uid' : user1, 'bucket' : bucket_name}) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user2}) assert ret == 200 # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = 'four' key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) # now delete it (ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {'bucket' : bucket_name, 'object' : object_name}) assert ret == 200 # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True}) assert ret == 200 assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # create a bucket for deletion stats useless_bucket = connection.create_bucket('useless_bucket') useless_key = useless_bucket.new_key('useless_key') useless_key.set_contents_from_string('useless string') # delete it useless_key.delete() useless_bucket.delete() # wait for the statistics to flush time.sleep(60) # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'categories' : 'delete_obj'}) # last operation we did is delete obj, wait for it to flush if successful_ops(out) > 0: break time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show']) assert ret == 200 assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] total = user_summary['total'] assert total['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1}) assert ret == 200 assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] for entry in user_summary['categories']: assert entry['successful_ops'] > 0 assert user_summary['user'] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'] for cat in test_categories: (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1, 'categories' : cat}) assert ret == 200 assert len(out['summary']) > 0 user_summary = out['summary'][0] assert user_summary['user'] == user1 assert len(user_summary['categories']) == 1 entry = user_summary['categories'][0] assert entry['category'] == cat assert entry['successful_ops'] > 0 # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid' : user1}) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1}) assert ret == 200 assert len(out['entries']) == 0 assert len(out['summary']) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True}) assert ret == 200 # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('five') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'}) assert ret == 200 # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('six') # TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = 'foo'*10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) # now delete the head big_key.delete() # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1}) assert ret == 409 # delete should fail because ``key`` still exists try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 key.delete() bucket.delete() # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') # should be private already but guarantee it key.set_acl('private') (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key}) assert ret == 200 acl = key.get_xml_acl() assert acl == out.strip('\n') # add another grantee by making the object public read key.set_acl('public-read') (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key}) assert ret == 200 acl = key.get_xml_acl() assert acl == out.strip('\n') # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) key_name = ['eight', 'nine', 'ten', 'eleven'] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], {'bucket' : bucket_name, 'purge-objects' : True}) assert ret == 200 # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps = 'usage=read' (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {'uid' : user1, 'user-caps' : caps}) assert ret == 200 assert out[0]['perm'] == 'read' # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {'uid' : user1, 'user-caps' : caps}) assert ret == 200 assert not out # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1}) assert ret == 409 # TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) key.set_contents_from_string('twelve') (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1, 'purge-data' : True}) assert ret == 200 # TESTCASE 'rm-user3','user','info','deleted user','fails' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1}) assert ret == 404
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ global log assert ctx.rgw.config, \ "radosgw_admin task needs a config passed from the rgw task" config = ctx.rgw.config log.debug('config is: %r', config) clients_from_config = config.keys() # choose first client as default client = clients_from_config[0] # once the client is chosen, pull the host name and assigned port out of # the role_endpoints that were assigned by the rgw task (remote_host, remote_port) = ctx.rgw.role_endpoints[client] ## user1='foo' user2='fud' subuser1='foo:foo1' subuser2='foo:foo2' display_name1='Foo' display_name2='Fud' email='*****@*****.**' email2='*****@*****.**' access_key='9te6NH5mcdcq0Tc5i8i1' secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2='p5YnriCv1nAtykxBrupQ' secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name='myfoo' bucket_name2='mybar' # connect to rgw connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) connection2 = boto.s3.connection.S3Connection( aws_access_key_id=access_key2, aws_secret_access_key=secret_key2, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user1, '--display-name', display_name1, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4' ], check_status=True) # TESTCASE 'duplicate email','user','create','existing user email','fails' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email, ]) assert err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--uid', user1, '--access-key', access_key2, '--secret', secret_key2, ], check_status=True) # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--uid', user1, '--access-key', access_key2, ], check_status=True) assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' subuser_access = 'full' subuser_perm = 'full-control' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser1, '--access', subuser_access ], check_status=True) # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'modify', '--subuser', subuser1, '--secret', swift_secret1, '--key-type', 'swift', ], check_status=True) # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert out['subusers'][0]['permissions'] == subuser_perm # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser2, '--secret', swift_secret2, '--key-type', 'swift', ], check_status=True) # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser1, '--key-type', 'swift', ], check_status=True) assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser1, ], check_status=True) assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser2, '--key-type', 'swift', '--purge-keys', ], check_status=True) assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 0 # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True) assert len(out) >= 1 assert bucket_name in out; # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4' bucket2 = connection.create_bucket(bucket_name + '2') bucket3 = connection.create_bucket(bucket_name + '3') bucket4 = connection.create_bucket(bucket_name + '4') # the 5th should fail. failed = False try: connection.create_bucket(bucket_name + '5') except Exception: failed = True assert failed # delete the buckets bucket2.delete() bucket3.delete() bucket4.delete() # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['owner'] == user1 bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0]['id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # reclaim it key.delete() # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name], check_status=True) # create a second user to link the bucket to (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '1', ], check_status=True) # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('two') except boto.exception.S3ResponseError: denied = True assert not denied # delete the object key.delete() # link the bucket to another user (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)], check_status=True) bucket_data = out['data'] assert bucket_data['bucket']['name'] == bucket_name bucket_id = bucket_data['bucket']['bucket_id'] # link the bucket to another user (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id], check_status=True) # try to remove user, should fail (has a linked bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2]) assert err # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked' (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name], check_status=True) # relink the bucket to the first user and delete the second user (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2], check_status=True) # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = 'four' key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) # now delete it (err, out) = rgwadmin(ctx, client, ['object', 'rm', '--bucket', bucket_name, '--object', object_name], check_status=True) # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True) assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' if obj[:4] == 'meta' or obj[:4] == 'data' or obj[:18] == 'obj_delete_at_hint': continue (err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj], check_status=True) assert len(rgwlog) > 0 # exempt bucket_name2 from checking as it was only used for multi-region tests assert rgwlog['bucket'].find(bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0 assert rgwlog['bucket'] != bucket_name or rgwlog['bucket_id'] == bucket_id assert rgwlog['bucket_owner'] == user1 or rgwlog['bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2 for entry in rgwlog['log_entries']: log.debug('checking log entry: ', entry) assert entry['bucket'] == rgwlog['bucket'] possible_buckets = [bucket_name + '5', bucket_name2] user = entry['user'] assert user == user1 or user.endswith('system-user') or \ rgwlog['bucket'] in possible_buckets # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj], check_status=True) # TODO: show log by bucket+date # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj']) # last operation we did is delete obj, wait for it to flush if get_user_successful_ops(out, user1) > 0: break time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = get_user_summary(out, user1) total = user_summary['total'] assert total['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] for entry in user_summary['categories']: assert entry['successful_ops'] > 0 assert user_summary['user'] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'] for cat in test_categories: (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat], check_status=True) assert len(out['summary']) > 0 user_summary = out['summary'][0] assert user_summary['user'] == user1 assert len(user_summary['categories']) == 1 entry = user_summary['categories'][0] assert entry['category'] == cat assert entry['successful_ops'] > 0 # the usage flush interval is 30 seconds, wait that much an then some # to make sure everything has been flushed time.sleep(35) # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1], check_status=True) (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) == 0 assert len(out['summary']) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('five') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('six') # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = 'foo'*10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) # now delete the head big_key.delete() # wait a bit to give the garbage collector time to cycle time.sleep(15) (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) > 0 # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage' (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True) #confirm (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) == 0 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # delete should fail because ``key`` still exists try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 key.delete() bucket.delete() # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') # should be private already but guarantee it key.set_acl('private') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key], check_status=True, format='xml') acl = get_acl(key) assert acl == out.strip('\n') # add another grantee by making the object public read key.set_acl('public-read') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key], check_status=True, format='xml') acl = get_acl(key) assert acl == out.strip('\n') # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) key_name = ['eight', 'nine', 'ten', 'eleven'] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) (err, out) = rgwadmin(ctx, client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'], check_status=True) # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps='user=read' (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps]) assert out['caps'][0]['perm'] == 'read' # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps]) assert not out['caps'] # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) key.set_contents_from_string('twelve') (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1, '--purge-data' ], check_status=True) # TESTCASE 'rm-user3','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule' # (err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default']) orig_placement_pools = len(out['placement_pools']) # removed this test, it is not correct to assume that zone has default placement, it really # depends on how we set it up before # # assert len(out) > 0 # assert len(out['placement_pools']) == 1 # default_rule = out['placement_pools'][0] # assert default_rule['key'] == 'default-placement' rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}} out['placement_pools'].append(rule) (err, out) = rgwadmin(ctx, client, ['zone', 'set'], stdin=StringIO(json.dumps(out)), check_status=True) (err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default']) assert len(out) > 0 assert len(out['placement_pools']) == orig_placement_pools + 1
count_success_ins += 1 finally: count_ins += 1 f.close() print("Successfully created %s instances" % (count_success_ins)) connection = boto.s3.connection.S3Connection( aws_access_key_id='adf938e8df4841129a5ca7089ab6b0ee', aws_secret_access_key='66f187d931f04af5805e899421bbfbef', port=8888, host='swift.rc.nectar.org.au', is_secure=True, validate_certs=False, calling_format=boto.s3.connection.OrdinaryCallingFormat()) # buckets = connection.get_all_buckets() container_name = "twitter_container" try: b = connection.create_bucket(container_name) except boto.exception.S3CreateError as e: if e.status == 409: print("already exists the container: %s" % (container_name)) pass except: print "Unexpected error:", sys.exc_info()[0] b = connection.get_bucket('twitter_container', validate=False) print("Bucket: %s is ready" % (b.name))
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ assert config is None or isinstance(config, list) \ or isinstance(config, dict), \ "task s3tests only supports a list or dictionary for configuration" all_clients = [ 'client.{id}'.format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client') ] if config is None: config = all_clients if isinstance(config, list): config = dict.fromkeys(config) clients = config.keys() # just use the first client... client = clients[0] ## user = '******' subuser1 = 'foo:foo1' subuser2 = 'foo:foo2' display_name = 'Foo' email = '*****@*****.**' access_key = '9te6NH5mcdcq0Tc5i8i1' secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2 = 'p5YnriCv1nAtykxBrupQ' secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name = 'myfoo' # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user, '--display-name', display_name, '--email', email, '--access-key', access_key, '--secret', secret_key ]) assert not err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert out['user_id'] == user assert out['email'] == email assert out['display_name'] == display_name assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user]) assert not err # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user]) assert not err # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--uid', user, '--access-key', access_key2, '--secret', secret_key2, ]) assert not err # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][ 'access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][ 'secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--uid', user, '--access-key', access_key2, ]) assert not err assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--subuser', subuser1, '--secret', swift_secret1, '--key-type', 'swift', ]) assert not err # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--subuser', subuser2, '--secret', swift_secret2, '--key-type', 'swift', ]) assert not err # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][ 'user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[ 'swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser1, '--key-type', 'swift', ]) assert not err assert len(out['swift_keys']) == 1 # TESTCASE 'rm-swift-key2','key','rm','subuser','succeeds, second key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser2, '--key-type', 'swift', ]) assert not err assert len(out['swift_keys']) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user]) assert not err assert len(out) == 0 # connect to rgw (remote, ) = ctx.cluster.only(client).remotes.iterkeys() (remote_user, remote_host) = remote.name.split('@') connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=7280, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user]) assert not err assert len(out) == 0 # creat a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user]) assert not err assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name]) assert not err assert out['owner'] == user bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user]) assert not err assert len(out) == 1 assert out[0][ 'id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket-id', '%d' % bucket_id]) assert not err assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # reclaim it key.delete() # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket-id', '%d' % bucket_id]) assert not err assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['log', 'list']) assert not err assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj]) assert not err assert len(log) > 0 assert log['bucket'] == bucket_name assert log['bucket_id'] == bucket_id assert log['bucket_owner'] == user for entry in log['log_entries']: assert entry['bucket'] == bucket_name assert entry['user'] == user # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj]) assert not err # TODO: show log by bucket+date # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user]) assert not err # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('two') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user]) assert not err # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('three') # TESTCASE 'rm-user','user','rm','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user]) assert not err # TESTCASE 'rm-user2','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert err
def setUpClass(cls): connection = boto.connect_s3(aws_access_key_id=key_id, aws_secret_access_key=access_key) connection.create_bucket(TestIntegrationS3.mart)
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ assert config is None or isinstance(config, list) \ or isinstance(config, dict), \ "task s3tests only supports a list or dictionary for configuration" all_clients = ['client.{id}'.format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] if config is None: config = all_clients if isinstance(config, list): config = dict.fromkeys(config) clients = config.keys() # just use the first client... client = clients[0]; ## user='******' subuser1='foo:foo1' subuser2='foo:foo2' display_name='Foo' email='*****@*****.**' access_key='9te6NH5mcdcq0Tc5i8i1' secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2='p5YnriCv1nAtykxBrupQ' secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name='myfoo' # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user, '--display-name', display_name, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4', ]) assert not err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert out['user_id'] == user assert out['email'] == email assert out['display_name'] == display_name assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user]) assert not err # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user]) assert not err # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--uid', user, '--access-key', access_key2, '--secret', secret_key2, ]) assert not err # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--uid', user, '--access-key', access_key2, ]) assert not err assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser1, '--secret', swift_secret1, '--key-type', 'swift', ]) assert not err # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser2, '--secret', swift_secret2, '--key-type', 'swift', ]) assert not err # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert not err assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser1, '--key-type', 'swift', ]) assert not err assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser1, ]) assert not err assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser2, '--key-type', 'swift', '--purge-keys', ]) assert not err assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user]) assert not err assert len(out) == 0 # connect to rgw (remote,) = ctx.cluster.only(client).remotes.iterkeys() (remote_user, remote_host) = remote.name.split('@') connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=7280, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user]) assert not err assert len(out) == 0 # creat a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user]) assert not err assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4' bucket2 = connection.create_bucket(bucket_name + '2') bucket3 = connection.create_bucket(bucket_name + '3') bucket4 = connection.create_bucket(bucket_name + '4') # the 5th should fail. failed = False try: bucket5 = connection.create_bucket(bucket_name + '5') except: failed = True assert failed # delete the buckets bucket2.delete() bucket3.delete() bucket4.delete() # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name]) assert not err assert out['owner'] == user bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user]) assert not err assert len(out) == 1 assert out[0]['id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket-id', bucket_id]) assert not err assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # reclaim it key.delete() # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket-id', bucket_id]) assert not err assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['log', 'list']) assert not err assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj]) assert not err assert len(log) > 0 assert log['bucket'].find(bucket_name) == 0 assert log['bucket'] != bucket_name or log['bucket_id'] == bucket_id assert log['bucket_owner'] == user or log['bucket'] == bucket_name + '5' for entry in log['log_entries']: assert entry['bucket'] == log['bucket'] assert entry['user'] == user or log['bucket'] == bucket_name + '5' # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj]) assert not err # TODO: show log by bucket+date # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= 45: (err, out) = rgwadmin(ctx, client, ['usage', 'show']) if len(out['entries']) > 0: break; time.sleep(1) assert time.time() - timestamp <= 45 # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show']) assert not err assert len(out['entries']) > 0 assert len(out['summary']) > 0 for entry in out['summary']: assert entry['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user]) assert not err assert len(out['entries']) > 0 assert len(out['summary']) > 0 for entry in out['summary']: assert entry['successful_ops'] > 0 assert entry['user'] == user # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user]) assert not err (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user]) assert not err assert len(out['entries']) == 0 assert len(out['summary']) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user]) assert not err # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('two') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user]) assert not err # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('three') # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user]) assert err # delete should fail fails = False try: bucket.delete() except: fails = True assert fails key.delete() bucket.delete() # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user]) assert not err # TESTCASE 'rm-user2','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user]) assert err
args = parser.parse_args() s3_access_key=os.environ['S3_ACCESS_KEY_ID'] s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY'] connection = boto.s3.connection.S3Connection( aws_access_key_id= s3_access_key, aws_secret_access_key=s3_secret_key, is_secure=False, port=8000, host='localhost', calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) print 'create a bucket' bucket = connection.create_bucket(args.name) obj1 = boto.s3.key.Key(bucket) obj1.key = 'foo' obj1.set_contents_from_string('This is a test of S3') obj2 = boto.s3.key.Key(bucket) obj2.key='abc' obj2.delete() b = connection.get_bucket(args.name) print 'bucket list' for k in b.list(): print k print "second time"
s3_access_key=os.environ['S3_ACCESS_KEY_ID'] s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY'] parser = argparse.ArgumentParser(description='create buckets') parser.add_argument('--num', type=int, action='store', default=100 ) parser.add_argument('--port', type=int, action='store', default=8000 ) parser.add_argument('--key',type=str, action='store', default=s3_access_key) parser.add_argument('--secret',type=str, action='store', default=s3_secret_key) args = parser.parse_args() #client_access_key=os.environ['SYSTEM_ACCESS_KEY_ID'] #client_secret_key=os.environ['SYSTEM_SECRET_ACCESS_KEY'] connection = boto.s3.connection.S3Connection( aws_access_key_id= args.key, aws_secret_access_key=args.secret, is_secure=False, port=args.port, host='localhost', calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) base_name='bucket' for i in range(args.num): print 'creating '+ `i` bucket = connection.create_bucket(base_name + `i`) print bucket
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ global log assert ctx.rgw.config, \ "radosgw_admin task needs a config passed from the rgw task" config = ctx.rgw.config log.debug('config is: %r', config) clients_from_config = config.keys() # choose first client as default client = clients_from_config[0] # once the client is chosen, pull the host name and assigned port out of # the role_endpoints that were assigned by the rgw task (remote_host, remote_port) = ctx.rgw.role_endpoints[client] ## user1 = 'foo' user2 = 'fud' subuser1 = 'foo:foo1' subuser2 = 'foo:foo2' display_name1 = 'Foo' display_name2 = 'Fud' email = '*****@*****.**' email2 = '*****@*****.**' access_key = '9te6NH5mcdcq0Tc5i8i1' secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2 = 'p5YnriCv1nAtykxBrupQ' secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name = 'myfoo' bucket_name2 = 'mybar' # connect to rgw connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) connection2 = boto.s3.connection.S3Connection( aws_access_key_id=access_key2, aws_secret_access_key=secret_key2, is_secure=False, port=remote_port, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin( ctx, client, [ 'user', 'create', '--uid', user1, '--display-name', display_name1, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4' ], check_status=True) # TESTCASE 'duplicate email','user','create','existing user email','fails' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email, ]) assert err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--uid', user1, '--access-key', access_key2, '--secret', secret_key2, ], check_status=True) # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][ 'access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][ 'secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--uid', user1, '--access-key', access_key2, ], check_status=True) assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' subuser_access = 'full' subuser_perm = 'full-control' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser1, '--access', subuser_access ], check_status=True) # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'modify', '--subuser', subuser1, '--secret', swift_secret1, '--key-type', 'swift', ], check_status=True) # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert out['subusers'][0]['permissions'] == subuser_perm # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser2, '--secret', swift_secret2, '--key-type', 'swift', ], check_status=True) # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][ 'user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[ 'swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser1, '--key-type', 'swift', ], check_status=True) assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser1, ], check_status=True) assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser2, '--key-type', 'swift', '--purge-keys', ], check_status=True) assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 0 # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True) assert len(out) >= 1 assert bucket_name in out # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4' bucket2 = connection.create_bucket(bucket_name + '2') bucket3 = connection.create_bucket(bucket_name + '3') bucket4 = connection.create_bucket(bucket_name + '4') # the 5th should fail. failed = False try: connection.create_bucket(bucket_name + '5') except Exception: failed = True assert failed # delete the buckets bucket2.delete() bucket3.delete() bucket4.delete() # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['owner'] == user1 bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0][ 'id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # reclaim it key.delete() # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (err, out) = rgwadmin( ctx, client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name], check_status=True) # create a second user to link the bucket to (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '1', ], check_status=True) # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('two') except boto.exception.S3ResponseError: denied = True assert not denied # delete the object key.delete() # link the bucket to another user (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)], check_status=True) bucket_data = out['data'] assert bucket_data['bucket']['name'] == bucket_name bucket_id = bucket_data['bucket']['bucket_id'] # link the bucket to another user (err, out) = rgwadmin(ctx, client, [ 'bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id ], check_status=True) # try to remove user, should fail (has a linked bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2]) assert err # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked' (err, out) = rgwadmin( ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name], check_status=True) # relink the bucket to the first user and delete the second user (err, out) = rgwadmin(ctx, client, [ 'bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id ], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2], check_status=True) # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = 'four' key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) # now delete it (err, out) = rgwadmin( ctx, client, ['object', 'rm', '--bucket', bucket_name, '--object', object_name], check_status=True) # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True) assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' if obj[: 4] == 'meta' or obj[: 4] == 'data' or obj[: 18] == 'obj_delete_at_hint': continue (err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj], check_status=True) assert len(rgwlog) > 0 # exempt bucket_name2 from checking as it was only used for multi-region tests assert rgwlog['bucket'].find( bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0 assert rgwlog['bucket'] != bucket_name or rgwlog[ 'bucket_id'] == bucket_id assert rgwlog['bucket_owner'] == user1 or rgwlog[ 'bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2 for entry in rgwlog['log_entries']: log.debug('checking log entry: ', entry) assert entry['bucket'] == rgwlog['bucket'] possible_buckets = [bucket_name + '5', bucket_name2] user = entry['user'] assert user == user1 or user.endswith('system-user') or \ rgwlog['bucket'] in possible_buckets # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj], check_status=True) # TODO: show log by bucket+date # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes (err, out) = rgwadmin( ctx, client, ['usage', 'show', '--categories', 'delete_obj' ]) # last operation we did is delete obj, wait for it to flush if get_user_successful_ops(out, user1) > 0: break time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = get_user_summary(out, user1) total = user_summary['total'] assert total['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] for entry in user_summary['categories']: assert entry['successful_ops'] > 0 assert user_summary['user'] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = [ 'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket' ] for cat in test_categories: (err, out) = rgwadmin( ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat], check_status=True) assert len(out['summary']) > 0 user_summary = out['summary'][0] assert user_summary['user'] == user1 assert len(user_summary['categories']) == 1 entry = user_summary['categories'][0] assert entry['category'] == cat assert entry['successful_ops'] > 0 # the usage flush interval is 30 seconds, wait that much an then some # to make sure everything has been flushed time.sleep(35) # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1], check_status=True) (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) == 0 assert len(out['summary']) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('five') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('six') # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = 'foo' * 10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) # now delete the head big_key.delete() # wait a bit to give the garbage collector time to cycle time.sleep(15) (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) > 0 # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage' (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True) #confirm (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) == 0 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # delete should fail because ``key`` still exists try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 key.delete() bucket.delete() # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') # should be private already but guarantee it key.set_acl('private') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key], check_status=True, format='xml') acl = get_acl(key) assert acl == out.strip('\n') # add another grantee by making the object public read key.set_acl('public-read') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key], check_status=True, format='xml') acl = get_acl(key) assert acl == out.strip('\n') # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) key_name = ['eight', 'nine', 'ten', 'eleven'] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) (err, out) = rgwadmin( ctx, client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'], check_status=True) # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps = 'user=read' (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps]) assert out['caps'][0]['perm'] == 'read' # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps]) assert not out['caps'] # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) key.set_contents_from_string('twelve') (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1, '--purge-data'], check_status=True) # TESTCASE 'rm-user3','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule' # (err, out) = rgwadmin(ctx, client, ['zone', 'get', '--rgw-zone', 'default']) orig_placement_pools = len(out['placement_pools']) # removed this test, it is not correct to assume that zone has default placement, it really # depends on how we set it up before # # assert len(out) > 0 # assert len(out['placement_pools']) == 1 # default_rule = out['placement_pools'][0] # assert default_rule['key'] == 'default-placement' rule = { 'key': 'new-placement', 'val': { 'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2' } } out['placement_pools'].append(rule) (err, out) = rgwadmin(ctx, client, ['zone', 'set'], stdin=StringIO(json.dumps(out)), check_status=True) (err, out) = rgwadmin(ctx, client, ['zone', 'get', '--rgw-zone', 'default']) assert len(out) > 0 assert len(out['placement_pools']) == orig_placement_pools + 1
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ assert config is None or isinstance(config, list) \ or isinstance(config, dict), \ "task s3tests only supports a list or dictionary for configuration" all_clients = ['client.{id}'.format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] if config is None: config = all_clients if isinstance(config, list): config = dict.fromkeys(config) clients = config.keys() # just use the first client... client = clients[0]; ## user1='foo' user2='fud' subuser1='foo:foo1' subuser2='foo:foo2' display_name1='Foo' display_name2='Fud' email='*****@*****.**' access_key='9te6NH5mcdcq0Tc5i8i1' secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2='p5YnriCv1nAtykxBrupQ' secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name='myfoo' # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user1, '--display-name', display_name1, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4' ]) assert not err # TESTCASE 'duplicate email','user','create','existing user email','fails' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email, ]) assert err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert not err assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1]) assert not err # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert not err assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1]) assert not err # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert not err assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--uid', user1, '--access-key', access_key2, '--secret', secret_key2, ]) assert not err # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert not err assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--uid', user1, '--access-key', access_key2, ]) assert not err assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' subuser_access = 'full' subuser_perm = 'full-control' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser1, '--access', subuser_access ]) assert not err # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'modify', '--subuser', subuser1, '--secret', swift_secret1, '--key-type', 'swift', ]) assert not err # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert out['subusers'][0]['permissions'] == subuser_perm # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert not err assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser2, '--secret', swift_secret2, '--key-type', 'swift', ]) assert not err # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert not err assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser1, '--key-type', 'swift', ]) assert not err assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser1, ]) assert not err assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser2, '--key-type', 'swift', '--purge-keys', ]) assert not err assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1]) assert not err assert len(out) == 0 # connect to rgw (remote,) = ctx.cluster.only(client).remotes.iterkeys() (remote_user, remote_host) = remote.name.split('@') connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=7280, host=remote_host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1]) assert not err assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1]) assert not err assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list']) assert not err assert len(out) >= 1 assert bucket_name in out; # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4' bucket2 = connection.create_bucket(bucket_name + '2') bucket3 = connection.create_bucket(bucket_name + '3') bucket4 = connection.create_bucket(bucket_name + '4') # the 5th should fail. failed = False try: connection.create_bucket(bucket_name + '5') except: failed = True assert failed # delete the buckets bucket2.delete() bucket3.delete() bucket4.delete() # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name]) assert not err assert out['owner'] == user1 bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1]) assert not err assert len(out) == 1 assert out[0]['id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name]) assert not err assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # reclaim it key.delete() # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name]) assert not err # create a second user to link the bucket to (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '1', ]) assert not err # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('two') except boto.exception.S3ResponseError: denied = True assert not denied # delete the object key.delete() # link the bucket to another user (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name]) assert not err # try to remove user, should fail (has a linked bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2]) assert err # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked' (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name]) assert not err # relink the bucket to the first user and delete the second user (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user1, '--bucket', bucket_name]) assert not err (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2]) assert not err # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = 'four' key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) # now delete it (err, out) = rgwadmin(ctx, client, ['object', 'rm', '--bucket', bucket_name, '--object', object_name]) assert not err # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, [ 'bucket', 'stats', '--bucket', bucket_name]) assert not err assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['log', 'list']) assert not err assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj]) assert not err assert len(log) > 0 assert log['bucket'].find(bucket_name) == 0 assert log['bucket'] != bucket_name or log['bucket_id'] == bucket_id assert log['bucket_owner'] == user1 or log['bucket'] == bucket_name + '5' for entry in log['log_entries']: assert entry['bucket'] == log['bucket'] assert entry['user'] == user1 or log['bucket'] == bucket_name + '5' # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj]) assert not err # TODO: show log by bucket+date # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj']) # last operation we did is delete obj, wait for it to flush if successful_ops(out) > 0: break; time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show']) assert not err assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] total = user_summary['total'] assert total['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1]) assert not err assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] for entry in user_summary['categories']: assert entry['successful_ops'] > 0 assert user_summary['user'] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'] for cat in test_categories: (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat]) assert not err assert len(out['summary']) > 0 user_summary = out['summary'][0] assert user_summary['user'] == user1 assert len(user_summary['categories']) == 1 entry = user_summary['categories'][0] assert entry['category'] == cat assert entry['successful_ops'] > 0 # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1]) assert not err (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1]) assert not err assert len(out['entries']) == 0 assert len(out['summary']) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1]) assert not err # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('five') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1]) assert not err # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('six') # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = 'foo'*10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) # now delete the head big_key.delete() # wait a bit to give the garbage collector time to cycle time.sleep(15) (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) > 0 # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage' (err, out) = rgwadmin(ctx, client, ['gc', 'process']) assert not err #confirm (err, out) = rgwadmin(ctx, client, ['gc', 'list']) assert len(out) == 0 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # delete should fail because ``key`` still exists fails = False try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 key.delete() bucket.delete() # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') # should be private already but guarantee it key.set_acl('private') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key]) assert not err acl = key.get_xml_acl() assert acl == out.strip('\n') # add another grantee by making the object public read key.set_acl('public-read') (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key]) assert not err acl = key.get_xml_acl() assert acl == out.strip('\n') # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) key_name = ['eight', 'nine', 'ten', 'eleven'] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) (err, out) = rgwadmin(ctx, client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects']) assert not err # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps='user=read' (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps]) assert out['caps'][0]['perm'] == 'read' # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps]) assert not out['caps'] # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) key.set_contents_from_string('twelve') (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1, '--purge-data' ]) assert not err # TESTCASE 'rm-user3','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule' (err, out) = rgwadmin(ctx, client, ['zone', 'get']) assert len(out) > 0 assert len(out['placement_pools']) == 1 default_rule = out['placement_pools'][0] assert default_rule['key'] == 'default-placement' rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}} out['placement_pools'].append(rule) (err, out) = rgwadmin(ctx, client, ['zone', 'set'], stdin=StringIO(json.dumps(out))) assert not err (err, out) = rgwadmin(ctx, client, ['zone', 'get']) assert len(out) > 0 assert len(out['placement_pools']) == 2
def task(ctx, config): """ Test radosgw-admin functionality through the RESTful interface """ assert config is None or isinstance(config, list) \ or isinstance(config, dict), \ "task s3tests only supports a list or dictionary for configuration" all_clients = [ 'client.{id}'.format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client') ] if config is None: config = all_clients if isinstance(config, list): config = dict.fromkeys(config) clients = config.keys() # just use the first client... client = next(iter(clients)) ## admin_user = '******' admin_display_name = 'Ms. Admin User' admin_access_key = 'MH1WC2XQ1S8UISFDZC8W' admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG' admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write; info=read' user1 = 'foo' user2 = 'fud' subuser1 = 'foo:foo1' subuser2 = 'foo:foo2' display_name1 = 'Foo' display_name2 = 'Fud' email = '*****@*****.**' access_key = '9te6NH5mcdcq0Tc5i8i1' secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2 = 'p5YnriCv1nAtykxBrupQ' secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name = 'myfoo' # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'create-admin-user','user','create','administrative user','succeeds' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', admin_user, '--display-name', admin_display_name, '--access-key', admin_access_key, '--secret', admin_secret_key, '--max-buckets', '0', '--caps', admin_caps ]) logging.error(out) logging.error(err) assert not err assert hasattr(ctx, 'rgw'), 'radosgw-admin-rest must run after the rgw task' endpoint = ctx.rgw.role_endpoints.get(client) assert endpoint, 'no rgw endpoint for {}'.format(client) admin_conn = boto.s3.connection.S3Connection( aws_access_key_id=admin_access_key, aws_secret_access_key=admin_secret_key, is_secure=True if endpoint.cert else False, port=endpoint.port, host=endpoint.hostname, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1}) assert ret == 404 # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (ret, out) = rgwadmin_rest( admin_conn, ['user', 'create'], { 'uid': user1, 'display-name': display_name1, 'email': email, 'access-key': access_key, 'secret-key': secret_key, 'max-buckets': '4' }) assert ret == 200 # TESTCASE 'list-no-user','user','list','list user keys','user list object' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], { 'list': '', 'max-entries': 0 }) assert ret == 200 assert out['count'] == 0 assert out['truncated'] == True assert len(out['keys']) == 0 assert len(out['marker']) > 0 # TESTCASE 'list-user-without-marker','user','list','list user keys','user list object' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], { 'list': '', 'max-entries': 1 }) assert ret == 200 assert out['count'] == 1 assert out['truncated'] == True assert len(out['keys']) == 1 assert len(out['marker']) > 0 marker = out['marker'] # TESTCASE 'list-user-with-marker','user','list','list user keys','user list object' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], { 'list': '', 'max-entries': 1, 'marker': marker }) assert ret == 200 assert out['count'] == 1 assert out['truncated'] == False assert len(out['keys']) == 1 # TESTCASE 'info-existing','user','info','existing user','returns correct info' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] assert out['tenant'] == '' assert out['max_buckets'] == 4 assert out['caps'] == [] assert out['op_mask'] == 'read, write, delete' assert out['default_placement'] == '' assert out['default_storage_class'] == '' assert out['placement_tags'] == [] assert not out['bucket_quota']['enabled'] assert not out['bucket_quota']['check_on_raw'] assert out['bucket_quota']['max_size'] == -1 assert out['bucket_quota']['max_size_kb'] == 0 assert out['bucket_quota']['max_objects'] == -1 assert not out['user_quota']['enabled'] assert not out['user_quota']['check_on_raw'] assert out['user_quota']['max_size'] == -1 assert out['user_quota']['max_size_kb'] == 0 assert out['user_quota']['max_objects'] == -1 assert out['temp_url_keys'] == [] assert out['type'] == 'rgw' assert out['mfa_ids'] == [] # TESTCASE 'info-existing','user','info','existing user query with wrong uid but correct access key','returns correct info' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], { 'access-key': access_key, 'uid': 'uid_not_exist' }) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] assert out['tenant'] == '' assert out['max_buckets'] == 4 assert out['caps'] == [] assert out['op_mask'] == "read, write, delete" assert out['default_placement'] == '' assert out['default_storage_class'] == '' assert out['placement_tags'] == [] assert not out['bucket_quota']['enabled'] assert not out['bucket_quota']['check_on_raw'] assert out['bucket_quota']['max_size'] == -1 assert out['bucket_quota']['max_size_kb'] == 0 assert out['bucket_quota']['max_objects'] == -1 assert not out['user_quota']['enabled'] assert not out['user_quota']['check_on_raw'] assert out['user_quota']['max_size'] == -1 assert out['user_quota']['max_size_kb'] == 0 assert out['user_quota']['max_objects'] == -1 assert out['temp_url_keys'] == [] assert out['type'] == 'rgw' assert out['mfa_ids'] == [] # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], { 'uid': user1, 'suspended': True }) assert ret == 200 # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert ret == 200 assert out['suspended'] assert out['email'] == email # TESTCASE 're-enable','user','enable','suspended user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], { 'uid': user1, 'suspended': 'false' }) assert not err # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert ret == 200 assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['key', 'create'], { 'uid': user1, 'access-key': access_key2, 'secret-key': secret_key2 }) assert ret == 200 # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert ret == 200 assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][ 'access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][ 'secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (ret, out) = rgwadmin_rest(admin_conn, ['key', 'rm'], { 'uid': user1, 'access-key': access_key2 }) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'create'], { 'subuser': subuser1, 'secret-key': swift_secret1, 'key-type': 'swift' }) assert ret == 200 # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert ret == 200 assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'create'], { 'subuser': subuser2, 'secret-key': swift_secret2, 'key-type': 'swift' }) assert ret == 200 # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert ret == 200 assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][ 'user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[ 'swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (ret, out) = rgwadmin_rest(admin_conn, ['key', 'rm'], { 'subuser': subuser1, 'key-type': 'swift' }) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'rm'], {'subuser': subuser1}) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'rm'], { 'subuser': subuser2, 'key-type': 'swift', '{purge-keys': True }) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid': user1}) assert ret == 200 assert len(out) == 0 # connect to rgw connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=True if endpoint.cert else False, port=endpoint.port, host=endpoint.hostname, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], { 'uid': user1, 'stats': True }) assert ret == 200 assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid': user1}) assert ret == 200 assert len(out) == 1 assert out[0] == bucket_name # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], { 'bucket': bucket_name, 'stats': True }) assert ret == 200 assert out['owner'] == user1 assert out['tenant'] == '' bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], { 'uid': user1, 'stats': True }) assert ret == 200 assert len(out) == 1 assert out[0][ 'id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], { 'bucket': bucket_name, 'stats': True }) assert ret == 200 assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 # TESTCASE 'bucket-stats6', 'bucket', 'stats', 'non-existent bucket', 'fails, 'bucket not found error' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket': 'doesnotexist'}) assert ret == 404 assert out['Code'] == 'NoSuchBucket' # reclaim it key.delete() # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], { 'uid': user1, 'bucket': bucket_name }) assert ret == 200 # create a second user to link the bucket to (ret, out) = rgwadmin_rest( admin_conn, ['user', 'create'], { 'uid': user2, 'display-name': display_name2, 'access-key': access_key2, 'secret-key': secret_key2, 'max-buckets': '1', }) assert ret == 200 # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('two') except boto.exception.S3ResponseError: denied = True assert not denied # delete the object key.delete() # link the bucket to another user (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], { 'uid': user2, 'bucket': bucket_name, 'bucket-id': bucket_id, }) assert ret == 200 # try creating an object with the first user which should cause an error key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('three') except boto.exception.S3ResponseError: denied = True assert denied # relink the bucket to the first user and delete the second user (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], { 'uid': user1, 'bucket': bucket_name, 'bucket-id': bucket_id, }) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user2}) assert ret == 200 # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = 'four' key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) # now delete it (ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], { 'bucket': bucket_name, 'object': object_name }) assert ret == 200 # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], { 'bucket': bucket_name, 'stats': True }) assert ret == 200 assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # create a bucket for deletion stats useless_bucket = connection.create_bucket('useless-bucket') useless_key = useless_bucket.new_key('useless_key') useless_key.set_contents_from_string('useless string') # delete it useless_key.delete() useless_bucket.delete() # wait for the statistics to flush time.sleep(60) # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes (ret, out) = rgwadmin_rest( admin_conn, ['usage', 'show'], {'categories': 'delete_obj' }) # last operation we did is delete obj, wait for it to flush if get_user_successful_ops(out, user1) > 0: break time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show']) assert ret == 200 assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = get_user_summary(out, user1) total = user_summary['total'] assert total['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid': user1}) assert ret == 200 assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] for entry in user_summary['categories']: assert entry['successful_ops'] > 0 assert user_summary['user'] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = [ 'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket' ] for cat in test_categories: (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], { 'uid': user1, 'categories': cat }) assert ret == 200 assert len(out['summary']) > 0 user_summary = out['summary'][0] assert user_summary['user'] == user1 assert len(user_summary['categories']) == 1 entry = user_summary['categories'][0] assert entry['category'] == cat assert entry['successful_ops'] > 0 # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid': user1}) assert ret == 200 (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid': user1}) assert ret == 200 assert len(out['entries']) == 0 assert len(out['summary']) == 0 # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], { 'uid': user1, 'suspended': True }) assert ret == 200 # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('five') except boto.exception.S3ResponseError as e: assert e.status == 403 # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], { 'uid': user1, 'suspended': 'false' }) assert ret == 200 # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('six') # TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = 'foo' * 10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) # now delete the head big_key.delete() # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user1}) assert ret == 409 # delete should fail because ``key`` still exists try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 key.delete() bucket.delete() # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') # should be private already but guarantee it key.set_acl('private') (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], { 'bucket': bucket.name, 'object': key.key }) assert ret == 200 assert len(out['acl']['grant_map']) == 1 # add another grantee by making the object public read key.set_acl('public-read') (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], { 'bucket': bucket.name, 'object': key.key }) assert ret == 200 assert len(out['acl']['grant_map']) == 2 # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) key_name = ['eight', 'nine', 'ten', 'eleven'] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], { 'bucket': bucket_name, 'purge-objects': True }) assert ret == 200 # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps = 'usage=read' (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], { 'uid': user1, 'user-caps': caps }) assert ret == 200 assert out[0]['perm'] == 'read' # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], { 'uid': user1, 'user-caps': caps }) assert ret == 200 assert not out # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user1}) assert ret == 409 # TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) key = boto.s3.key.Key(bucket) key.set_contents_from_string('twelve') (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], { 'uid': user1, 'purge-data': True }) assert ret == 200 # TESTCASE 'rm-user3','user','info','deleted user','fails' (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1}) assert ret == 404 # TESTCASE 'info' 'display info' 'succeeds' (ret, out) = rgwadmin_rest(admin_conn, ['info', '']) assert ret == 200 info = out['info'] backends = info['storage_backends'] name = backends[0]['name'] fsid = backends[0]['cluster_id'] # name is always "rados" at time of writing, but zipper would allow # other backends, at some point assert len(name) > 0 # fsid is a uuid, but I'm not going to try to parse it assert len(fsid) > 0
args = parser.parse_args() s3_access_key=os.environ['S3_ACCESS_KEY_ID'] s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY'] connection = boto.s3.connection.S3Connection( aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_key, is_secure=False, port=args.port, host=args.host, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) print 'create a bucket' bucket = connection.create_bucket(args.bucket) start = 0 while start < args.num: threads = [] num_threads = 0 while num_threads < args.num_threads and start < args.num: end = min(args.num, start + args.slice) t = threading.Thread(target=create_objs, args=(bucket, start, end,)) threads.append(t) t. start() start += args.slice num_threads += 1 for t in threads:
args = parser.parse_args() s3_access_key=os.environ['S3_ACCESS_KEY_ID'] s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY'] connection = boto.s3.connection.S3Connection( aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_key, is_secure=False, port=8000, host='127.0.0.1', calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) print 'create a bucket' bucket = connection.create_bucket('bucket10') for i in range(args.num): print 'creating obj'+ `i` obj = boto.s3.key.Key(bucket) obj.key = 'obj' + `i` obj.set_contents_from_string('This is a test of S3') print 'list objects' i = 0 for k in bucket.list(): print 'obj ' + `i` print k i = i + 1
def task(ctx, config): """ Test radosgw-admin functionality against a running rgw instance. """ global log assert ctx.rgw.config, \ "radosgw_admin task needs a config passed from the rgw task" config = ctx.rgw.config log.debug('config is: %r', config) clients_from_config = config.keys() # choose first client as default client = next(iter(clients_from_config)) # once the client is chosen, pull the host name and assigned port out of # the role_endpoints that were assigned by the rgw task endpoint = ctx.rgw.role_endpoints[client] cleanup(ctx, client) ## user1 = 'foo' user2 = 'fud' user3 = 'bar' user4 = 'bud' subuser1 = 'foo:foo1' subuser2 = 'foo:foo2' display_name1 = 'Foo' display_name2 = 'Fud' display_name3 = 'Bar' email = '*****@*****.**' access_key = '9te6NH5mcdcq0Tc5i8i1' secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu' access_key2 = 'p5YnriCv1nAtykxBrupQ' secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh' access_key3 = 'NX5QOQKC6BH2IDN8HC7A' secret_key3 = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn' swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL' swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy' bucket_name = 'myfoo' bucket_name2 = 'mybar' # connect to rgw connection = boto.s3.connection.S3Connection( aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=endpoint.port, host=endpoint.hostname, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) connection.auth_region_name = 'us-east-1' connection2 = boto.s3.connection.S3Connection( aws_access_key_id=access_key2, aws_secret_access_key=secret_key2, is_secure=False, port=endpoint.port, host=endpoint.hostname, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) connection2.auth_region_name = 'us-east-1' connection3 = boto.s3.connection.S3Connection( aws_access_key_id=access_key3, aws_secret_access_key=secret_key3, is_secure=False, port=endpoint.port, host=endpoint.hostname, calling_format=boto.s3.connection.OrdinaryCallingFormat(), ) connection3.auth_region_name = 'us-east-1' acc = usage_acc() rl = requestlog_queue(acc.generate_make_entry()) connection.set_request_hook(rl) connection2.set_request_hook(rl) connection3.set_request_hook(rl) # legend (test cases can be easily grep-ed out) # TESTCASE 'testname','object','method','operation','assertion' # TESTCASE 'usage-show0' 'usage' 'show' 'all usage' 'succeeds' (err, summary0) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True) # TESTCASE 'info-nosuch','user','info','non-existent user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'create-ok','user','create','w/all valid info','succeeds' (err, out) = rgwadmin( ctx, client, [ 'user', 'create', '--uid', user1, '--display-name', display_name1, '--email', email, '--access-key', access_key, '--secret', secret_key, '--max-buckets', '4' ], check_status=True) # TESTCASE 'duplicate email','user','create','existing user email','fails' (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--email', email, ]) assert err # TESTCASE 'info-existing','user','info','existing user','returns correct info' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['user_id'] == user1 assert out['email'] == email assert out['display_name'] == display_name1 assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key assert not out['suspended'] # TESTCASE 'suspend-ok','user','suspend','active user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert out['suspended'] # TESTCASE 're-enable','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert not out['suspended'] # TESTCASE 'add-keys','key','create','w/valid info','succeeds' (err, out) = rgwadmin(ctx, client, [ 'key', 'create', '--uid', user1, '--access-key', access_key2, '--secret', secret_key2, ], check_status=True) # TESTCASE 'info-new-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['keys']) == 2 assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][ 'access_key'] == access_key2 assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][ 'secret_key'] == secret_key2 # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--uid', user1, '--access-key', access_key2, ], check_status=True) assert len(out['keys']) == 1 assert out['keys'][0]['access_key'] == access_key assert out['keys'][0]['secret_key'] == secret_key # TESTCASE 'add-swift-key','key','create','swift key','succeeds' subuser_access = 'full' subuser_perm = 'full-control' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser1, '--access', subuser_access ], check_status=True) # TESTCASE 'add-swift-key','key','create','swift key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'modify', '--subuser', subuser1, '--secret', swift_secret1, '--key-type', 'swift', ], check_status=True) # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert out['subusers'][0]['permissions'] == subuser_perm # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 1 assert out['swift_keys'][0]['user'] == subuser1 assert out['swift_keys'][0]['secret_key'] == swift_secret1 # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'create', '--subuser', subuser2, '--secret', swift_secret2, '--key-type', 'swift', ], check_status=True) # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True) assert len(out['swift_keys']) == 2 assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][ 'user'] == subuser2 assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[ 'swift_keys'][1]['secret_key'] == swift_secret2 # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed' (err, out) = rgwadmin(ctx, client, [ 'key', 'rm', '--subuser', subuser1, '--key-type', 'swift', ], check_status=True) assert len(out['swift_keys']) == 1 # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser1, ], check_status=True) assert len(out['subusers']) == 1 # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed' (err, out) = rgwadmin(ctx, client, [ 'subuser', 'rm', '--subuser', subuser2, '--key-type', 'swift', '--purge-keys', ], check_status=True) assert len(out['swift_keys']) == 0 assert len(out['subusers']) == 0 # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 0 # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 0 # create a first bucket bucket = connection.create_bucket(bucket_name) rl.log_and_clear("create_bucket", bucket_name, user1) # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0] == bucket_name bucket_list = connection.get_all_buckets() assert len(bucket_list) == 1 assert bucket_list[0].name == bucket_name rl.log_and_clear("list_buckets", '', user1) # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list' (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True) assert len(out) >= 1 assert bucket_name in out # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4' bucket2 = connection.create_bucket(bucket_name + '2') rl.log_and_clear("create_bucket", bucket_name + '2', user1) bucket3 = connection.create_bucket(bucket_name + '3') rl.log_and_clear("create_bucket", bucket_name + '3', user1) bucket4 = connection.create_bucket(bucket_name + '4') rl.log_and_clear("create_bucket", bucket_name + '4', user1) # the 5th should fail. failed = False try: connection.create_bucket(bucket_name + '5') except Exception: failed = True assert failed rl.log_and_clear("create_bucket", bucket_name + '5', user1) # delete the buckets bucket2.delete() rl.log_and_clear("delete_bucket", bucket_name + '2', user1) bucket3.delete() rl.log_and_clear("delete_bucket", bucket_name + '3', user1) bucket4.delete() rl.log_and_clear("delete_bucket", bucket_name + '4', user1) # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['owner'] == user1 bucket_id = out['id'] # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True) assert len(out) == 1 assert out[0][ 'id'] == bucket_id # does it return the same ID twice in a row? # use some space key = boto.s3.key.Key(bucket) key.set_contents_from_string('one') rl.log_and_clear("put_obj", bucket_name, user1) # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 1 assert out['usage']['rgw.main']['size_kb'] > 0 #validate we have a positive user stats now (err, out) = rgwadmin(ctx, client, ['user', 'stats', '--uid', user1, '--sync-stats'], check_status=True) assert out['stats']['size'] > 0 # reclaim it key.delete() rl.log_and_clear("delete_obj", bucket_name, user1) # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error' (err, out) = rgwadmin( ctx, client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name], check_status=True) # create a second user to link the bucket to (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '1', ], check_status=True) # try creating an object with the first user before the bucket is relinked denied = False key = boto.s3.key.Key(bucket) try: key.set_contents_from_string('two') except boto.exception.S3ResponseError: denied = True assert not denied rl.log_and_clear("put_obj", bucket_name, user1) # delete the object key.delete() rl.log_and_clear("delete_obj", bucket_name, user1) # link the bucket to another user (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)], check_status=True) bucket_data = out['data'] assert bucket_data['bucket']['name'] == bucket_name bucket_id = bucket_data['bucket']['bucket_id'] # link the bucket to another user (err, out) = rgwadmin(ctx, client, [ 'bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id ], check_status=True) # try to remove user, should fail (has a linked bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2]) assert err # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked' (err, out) = rgwadmin( ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name], check_status=True) # relink the bucket to the first user and delete the second user (err, out) = rgwadmin(ctx, client, [ 'bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id ], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2], check_status=True) #TESTCASE 'bucket link', 'bucket', 'tenanted user', 'succeeds' tenant_name = "testx" # create a tenanted user to link the bucket to (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--tenant', tenant_name, '--uid', 'tenanteduser', '--display-name', 'tenanted-user', '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '1', ], check_status=True) # link the bucket to a tenanted user (err, out) = rgwadmin(ctx, client, [ 'bucket', 'link', '--bucket', '/' + bucket_name, '--tenant', tenant_name, '--uid', 'tenanteduser' ], check_status=True) # check if the bucket name has tenant/ prefix (err, out) = rgwadmin(ctx, client, [ 'metadata', 'get', 'bucket:{n}'.format(n=tenant_name + '/' + bucket_name) ], check_status=True) bucket_data = out['data'] assert bucket_data['bucket']['name'] == bucket_name assert bucket_data['bucket']['tenant'] == tenant_name # relink the bucket to the first user and delete the tenanted user (err, out) = rgwadmin(ctx, client, [ 'bucket', 'link', '--bucket', tenant_name + '/' + bucket_name, '--uid', user1 ], check_status=True) (err, out) = rgwadmin( ctx, client, ['user', 'rm', '--tenant', tenant_name, '--uid', 'tenanteduser'], check_status=True) # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed' # upload an object object_name = 'four' key = boto.s3.key.Key(bucket, object_name) key.set_contents_from_string(object_name) rl.log_and_clear("put_obj", bucket_name, user1) # fetch it too (for usage stats presently) s = key.get_contents_as_string(encoding='ascii') rl.log_and_clear("get_obj", bucket_name, user1) assert s == object_name # list bucket too (for usage stats presently) keys = list(bucket.list()) rl.log_and_clear("list_bucket", bucket_name, user1) assert len(keys) == 1 assert keys[0].name == object_name # now delete it (err, out) = rgwadmin( ctx, client, ['object', 'rm', '--bucket', bucket_name, '--object', object_name], check_status=True) # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--bucket', bucket_name], check_status=True) assert out['id'] == bucket_id assert out['usage']['rgw.main']['num_objects'] == 0 # list log objects # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects' (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True) assert len(out) > 0 for obj in out: # TESTCASE 'log-show','log','show','after activity','returns expected info' if obj[: 4] == 'meta' or obj[: 4] == 'data' or obj[: 18] == 'obj_delete_at_hint': continue (err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj], check_status=True) assert len(rgwlog) > 0 # skip any entry for which there is no bucket name--e.g., list_buckets, # since that is valid but cannot pass the following checks entry_bucket_name = rgwlog['bucket'] if entry_bucket_name.strip() != "": # exempt bucket_name2 from checking as it was only used for multi-region tests assert rgwlog['bucket'].find( bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0 assert rgwlog['bucket'] != bucket_name or rgwlog[ 'bucket_id'] == bucket_id assert rgwlog['bucket_owner'] == user1 or rgwlog[ 'bucket'] == bucket_name + '5' or rgwlog[ 'bucket'] == bucket_name2 for entry in rgwlog['log_entries']: log.debug('checking log entry: ', entry) assert entry['bucket'] == rgwlog['bucket'] possible_buckets = [bucket_name + '5', bucket_name2] user = entry['user'] assert user == user1 or user.endswith('system-user') or \ rgwlog['bucket'] in possible_buckets # TESTCASE 'log-rm','log','rm','delete log objects','succeeds' (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj], check_status=True) # TODO: show log by bucket+date # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1], check_status=True) # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects' denied = False try: key = boto.s3.key.Key(bucket) key.set_contents_from_string('five') except boto.exception.S3ResponseError as e: denied = True assert e.status == 403 assert denied rl.log_and_clear("put_obj", bucket_name, user1) # TESTCASE 'user-renable2','user','enable','suspended user','succeeds' (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True) # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects' key = boto.s3.key.Key(bucket) key.set_contents_from_string('six') rl.log_and_clear("put_obj", bucket_name, user1) # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection' # create an object large enough to be split into multiple parts test_string = 'foo' * 10000000 big_key = boto.s3.key.Key(bucket) big_key.set_contents_from_string(test_string) rl.log_and_clear("put_obj", bucket_name, user1) # now delete the head big_key.delete() rl.log_and_clear("delete_obj", bucket_name, user1) # wait a bit to give the garbage collector time to cycle time.sleep(15) (err, out) = rgwadmin(ctx, client, ['gc', 'list', '--include-all']) assert len(out) > 0 # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage' (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True) #confirm (err, out) = rgwadmin(ctx, client, ['gc', 'list', '--include-all']) # don't assume rgw_gc_obj_min_wait has been overridden omit_tdir = hasattr(ctx.rgw, 'omit_tdir') and ctx.rgw.omit_tdir == True if omit_tdir == False: assert len(out) == 0 # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets' (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # delete should fail because ``key`` still exists try: bucket.delete() except boto.exception.S3ResponseError as e: assert e.status == 409 rl.log_and_clear("delete_bucket", bucket_name, user1) key.delete() rl.log_and_clear("delete_obj", bucket_name, user1) bucket.delete() rl.log_and_clear("delete_bucket", bucket_name, user1) # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy' bucket = connection.create_bucket(bucket_name) rl.log_and_clear("create_bucket", bucket_name, user1) # create an object key = boto.s3.key.Key(bucket) key.set_contents_from_string('seven') rl.log_and_clear("put_obj", bucket_name, user1) # should be private already but guarantee it key.set_acl('private') rl.log_and_clear("put_acls", bucket_name, user1) (err, out) = rgwadmin( ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key.decode()], check_status=True, format='xml') acl = get_acl(key) rl.log_and_clear("get_acls", bucket_name, user1) assert acl == out.strip('\n') # add another grantee by making the object public read key.set_acl('public-read') rl.log_and_clear("put_acls", bucket_name, user1) (err, out) = rgwadmin( ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key.decode()], check_status=True, format='xml') acl = get_acl(key) rl.log_and_clear("get_acls", bucket_name, user1) assert acl == out.strip('\n') # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds' bucket = connection.create_bucket(bucket_name) rl.log_and_clear("create_bucket", bucket_name, user1) key_name = ['eight', 'nine', 'ten', 'eleven'] for i in range(4): key = boto.s3.key.Key(bucket) key.set_contents_from_string(key_name[i]) rl.log_and_clear("put_obj", bucket_name, user1) (err, out) = rgwadmin( ctx, client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'], check_status=True) # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds' caps = 'user=read' (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps]) assert out['caps'][0]['perm'] == 'read' # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds' (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps]) assert not out['caps'] # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets' bucket = connection.create_bucket(bucket_name) rl.log_and_clear("create_bucket", bucket_name, user1) key = boto.s3.key.Key(bucket) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1]) assert err # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds' bucket = connection.create_bucket(bucket_name) rl.log_and_clear("create_bucket", bucket_name, user1) key = boto.s3.key.Key(bucket) key.set_contents_from_string('twelve') rl.log_and_clear("put_obj", bucket_name, user1) time.sleep(35) # need to wait for all usage data to get flushed, should take up to 30 seconds timestamp = time.time() while time.time() - timestamp <= (2 * 60): # wait up to 20 minutes (err, out) = rgwadmin(ctx, client, [ 'usage', 'show', '--categories', 'delete_obj' ]) # one of the operations we did is delete_obj, should be present. if get_user_successful_ops(out, user1) > 0: break time.sleep(1) assert time.time() - timestamp <= (20 * 60) # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 r = acc.compare_results(out) if len(r) != 0: sys.stderr.write(("\n".join(r)) + "\n") assert (len(r) == 0) user_summary = get_user_summary(out, user1) total = user_summary['total'] assert total['successful_ops'] > 0 # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds' (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) > 0 assert len(out['summary']) > 0 user_summary = out['summary'][0] for entry in user_summary['categories']: assert entry['successful_ops'] > 0 assert user_summary['user'] == user1 # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds' test_categories = [ 'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket' ] for cat in test_categories: (err, out) = rgwadmin( ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat], check_status=True) assert len(out['summary']) > 0 user_summary = out['summary'][0] assert user_summary['user'] == user1 assert len(user_summary['categories']) == 1 entry = user_summary['categories'][0] assert entry['category'] == cat assert entry['successful_ops'] > 0 # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'new user', 'succeeds' # create a new user user3 (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user3, '--display-name', display_name3, '--access-key', access_key3, '--secret', secret_key3, '--max-buckets', '4' ], check_status=True) # create a bucket bucket = connection3.create_bucket(bucket_name + '6') rl.log_and_clear("create_bucket", bucket_name + '6', user3) # create object object_name1 = 'thirteen' key1 = boto.s3.key.Key(bucket, object_name1) key1.set_contents_from_string(object_name1) rl.log_and_clear("put_obj", bucket_name + '6', user3) # rename user3 (err, out) = rgwadmin(ctx, client, ['user', 'rename', '--uid', user3, '--new-uid', user4], check_status=True) assert out['user_id'] == user4 assert out['keys'][0]['access_key'] == access_key3 assert out['keys'][0]['secret_key'] == secret_key3 time.sleep(5) # get bucket and object to test if user keys are preserved bucket = connection3.get_bucket(bucket_name + '6') s = key1.get_contents_as_string(encoding='ascii') rl.log_and_clear("get_obj", bucket_name + '6', user4) assert s == object_name1 # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'another existing user', 'fails' # create a new user user2 (err, out) = rgwadmin(ctx, client, [ 'user', 'create', '--uid', user2, '--display-name', display_name2, '--access-key', access_key2, '--secret', secret_key2, '--max-buckets', '4' ], check_status=True) # create a bucket bucket = connection2.create_bucket(bucket_name + '7') rl.log_and_clear("create_bucket", bucket_name + '7', user2) # create object object_name2 = 'fourteen' key2 = boto.s3.key.Key(bucket, object_name2) key2.set_contents_from_string(object_name2) rl.log_and_clear("put_obj", bucket_name + '7', user2) (err, out) = rgwadmin(ctx, client, ['user', 'rename', '--uid', user4, '--new-uid', user2]) assert err # test if user 2 and user4 can still access their bucket and objects after rename fails bucket = connection3.get_bucket(bucket_name + '6') s = key1.get_contents_as_string(encoding='ascii') rl.log_and_clear("get_obj", bucket_name + '6', user4) assert s == object_name1 bucket = connection2.get_bucket(bucket_name + '7') s = key2.get_contents_as_string(encoding='ascii') rl.log_and_clear("get_obj", bucket_name + '7', user2) assert s == object_name2 (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user4, '--purge-data'], check_status=True) (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2, '--purge-data'], check_status=True) time.sleep(5) # should be all through with connection. (anything using connection # should be BEFORE the usage stuff above.) rl.log_and_clear("(before-close)", '-', '-', ignore_this_entry) connection.close() connection = None # the usage flush interval is 30 seconds, wait that much an then some # to make sure everything has been flushed time.sleep(35) # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed' (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1], check_status=True) (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1], check_status=True) assert len(out['entries']) == 0 assert len(out['summary']) == 0 (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1, '--purge-data'], check_status=True) # TESTCASE 'rm-user3','user','rm','deleted user','fails' (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1]) assert err # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule' (err, out) = rgwadmin(ctx, client, ['zone', 'get', '--rgw-zone', 'default']) orig_placement_pools = len(out['placement_pools']) # removed this test, it is not correct to assume that zone has default placement, it really # depends on how we set it up before # # assert len(out) > 0 # assert len(out['placement_pools']) == 1 # default_rule = out['placement_pools'][0] # assert default_rule['key'] == 'default-placement' rule = { 'key': 'new-placement', 'val': { 'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2' } } out['placement_pools'].append(rule) (err, out) = rgwadmin(ctx, client, ['zone', 'set'], stdin=StringIO(json.dumps(out)), check_status=True) (err, out) = rgwadmin(ctx, client, ['zone', 'get']) assert len(out) > 0 assert len(out['placement_pools']) == orig_placement_pools + 1 zonecmd = [ 'zone', 'placement', 'rm', '--rgw-zone', 'default', '--placement-id', 'new-placement' ] (err, out) = rgwadmin(ctx, client, zonecmd, check_status=True) # TESTCASE 'zonegroup-info', 'zonegroup', 'get', 'get zonegroup info', 'succeeds' (err, out) = rgwadmin(ctx, client, ['zonegroup', 'get'], check_status=True)