def configure_compression_in_default_zone(ctx, config):
ceph_config = ctx.ceph['ceph'].conf.get('global', {})
ceph_config.update(ctx.ceph['ceph'].conf.get('client', {}))
for client, c_config in config.iteritems():
ceph_config.update(ctx.ceph['ceph'].conf.get(client, {}))
key = 'rgw compression type'
if not key in ceph_config:
log.debug('No compression setting to enable')
break
compression = ceph_config[key]
log.debug('Configuring compression type = %s', compression)
# XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
# issue a 'radosgw-admin user list' command to trigger this
rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
rgwadmin(ctx,
client,
cmd=[
'zone', 'placement', 'modify', '--rgw-zone', 'default',
'--placement-id', 'default-placement', '--compression',
compression
],
check_status=True)
break # only the first client
def configure_compression(ctx, clients, compression):
""" set a compression type in the default zone placement """
log.info('Configuring compression type = %s', compression)
for client in clients:
# XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
# issue a 'radosgw-admin user list' command to trigger this
rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
rgwadmin(ctx, client,
cmd=['zone', 'placement', 'modify', '--rgw-zone', 'default',
'--placement-id', 'default-placement',
'--compression', compression],
check_status=True)
yield
def configure_compression(ctx, clients, compression):
""" set a compression type in the default zone placement """
log.info('Configuring compression type = %s', compression)
for client in clients:
# XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
# issue a 'radosgw-admin user list' command to trigger this
rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
rgwadmin(ctx, client,
cmd=['zone', 'placement', 'modify', '--rgw-zone', 'default',
'--placement-id', 'default-placement',
'--compression', compression],
check_status=True)
yield
def configure_users_for_client(ctx, config, client, everywhere=False):
"""
Create users by remotely running rgwadmin commands using extracted
user information.
"""
log.info('Configuring users...')
log.info('for client %s', client)
log.info('everywhere %s', everywhere)
# For data sync the master zones and regions must have the
# system users of the secondary zones. To keep this simple,
# just create the system users on every client if regions are
# configured.
clients_to_create_as = [client]
if everywhere:
clients_to_create_as = config.keys()
# extract the user info and append it to the payload tuple for the given
# client
for client, c_config in config.iteritems():
if not c_config:
continue
user_info = extract_user_info(c_config)
if not user_info:
continue
for client_name in clients_to_create_as:
log.debug('Creating user {user} on {client}'.format(
user=user_info['system_key']['user'], client=client_name))
rgwadmin(
ctx,
client_name,
cmd=[
'user',
'create',
'--uid',
user_info['system_key']['user'],
'--access-key',
user_info['system_key']['access_key'],
'--secret',
user_info['system_key']['secret_key'],
'--display-name',
user_info['system_key']['user'],
'--system',
],
check_status=True,
)
yield
def configure_compression_in_default_zone(ctx, config):
compression = ctx.rgw.compression_type
if not compression:
return
log.debug('Configuring compression type = %s', compression)
for client, c_config in config.iteritems():
# XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
# issue a 'radosgw-admin user list' command to trigger this
rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
rgwadmin(ctx, client,
cmd=['zone', 'placement', 'modify', '--rgw-zone', 'default',
'--placement-id', 'default-placement',
'--compression', compression],
check_status=True)
def configure_compression_in_default_zone(ctx, config):
compression = ctx.rgw.compression_type
if not compression:
return
log.debug('Configuring compression type = %s', compression)
for client, c_config in config.iteritems():
# XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
# issue a 'radosgw-admin user list' command to trigger this
rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
rgwadmin(ctx,
client,
cmd=[
'zone', 'placement', 'modify', '--rgw-zone', 'default',
'--placement-id', 'default-placement', '--compression',
compression
],
check_status=True)
def configure_users_for_client(ctx, config, client, everywhere=False):
"""
Create users by remotely running rgwadmin commands using extracted
user information.
"""
log.info('Configuring users...')
log.info('for client %s', client)
log.info('everywhere %s', everywhere)
# For data sync the master zones and regions must have the
# system users of the secondary zones. To keep this simple,
# just create the system users on every client if regions are
# configured.
clients_to_create_as = [client]
if everywhere:
clients_to_create_as = config.keys()
# extract the user info and append it to the payload tuple for the given
# client
for client, c_config in config.iteritems():
if not c_config:
continue
user_info = extract_user_info(c_config)
if not user_info:
continue
for client_name in clients_to_create_as:
log.debug('Creating user {user} on {client}'.format(
user=user_info['system_key']['user'], client=client_name))
rgwadmin(ctx, client_name,
cmd=[
'user', 'create',
'--uid', user_info['system_key']['user'],
'--access-key', user_info['system_key']['access_key'],
'--secret', user_info['system_key']['secret_key'],
'--display-name', user_info['system_key']['user'],
'--system',
],
check_status=True,
)
yield
def configure_storage_classes(ctx, clients, storage_classes):
""" set a compression type in the default zone placement """
sc = [s.strip() for s in storage_classes.split(',')]
for client in clients:
# XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete().
# issue a 'radosgw-admin user list' command to trigger this
rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True)
for storage_class in sc:
log.info('Configuring storage class type = %s', storage_class)
rgwadmin(ctx, client,
cmd=['zonegroup', 'placement', 'add',
'--rgw-zone', 'default',
'--placement-id', 'default-placement',
'--storage-class', storage_class],
check_status=True)
rgwadmin(ctx, client,
cmd=['zone', 'placement', 'add',
'--rgw-zone', 'default',
'--placement-id', 'default-placement',
'--storage-class', storage_class,
'--data-pool', 'default.rgw.buckets.data.' + storage_class.lower()],
check_status=True)
yield
def admin(self, args = None, **kwargs):
""" radosgw-admin command """
args = args or []
args += ['--cluster', self.name]
args += ['--debug-rgw', '0']
if kwargs.pop('read_only', False):
args += ['--rgw-cache-enabled', 'false']
kwargs['decode'] = False
check_retcode = kwargs.pop('check_retcode', True)
r, s = rgwadmin(self.ctx, self.client, args, **kwargs)
if check_retcode:
assert r == 0
return s, r
def admin(self, args=None, **kwargs):
""" radosgw-admin command """
args = args or []
args += ['--cluster', self.name]
args += ['--debug-rgw', '0']
if kwargs.pop('read_only', False):
args += ['--rgw-cache-enabled', 'false']
kwargs['decode'] = False
check_retcode = kwargs.pop('check_retcode', True)
r, s = rgwadmin(self.ctx, self.client, args, **kwargs)
if check_retcode:
assert r == 0
return s, r
def task(ctx, config):
"""
Test radosgw-admin functionality against a running rgw instance.
"""
global log
assert config is None or isinstance(config, list) \
or isinstance(config, dict), \
"task s3tests only supports a list or dictionary for configuration"
all_clients = [
'client.{id}'.format(id=id_)
for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')
]
if config is None:
config = all_clients
if isinstance(config, list):
config = dict.fromkeys(config)
clients = config.keys()
multi_region_run = rgw_utils.multi_region_enabled(ctx)
client = clients[0]
# default choice, multi-region code may overwrite this
if multi_region_run:
client = rgw_utils.get_master_client(ctx, clients)
# once the client is chosen, pull the host name and assigned port out of
# the role_endpoints that were assigned by the rgw task
(remote_host, remote_port) = ctx.rgw.role_endpoints[client]
realm = ctx.rgw.realm
log.debug('radosgw-admin: realm %r', realm)
##
user1 = 'foo'
user2 = 'fud'
subuser1 = 'foo:foo1'
subuser2 = 'foo:foo2'
display_name1 = 'Foo'
display_name2 = 'Fud'
email = '*****@*****.**'
email2 = '*****@*****.**'
access_key = '9te6NH5mcdcq0Tc5i8i1'
secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
access_key2 = 'p5YnriCv1nAtykxBrupQ'
secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
bucket_name = 'myfoo'
bucket_name2 = 'mybar'
# connect to rgw
connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
connection2 = boto.s3.connection.S3Connection(
aws_access_key_id=access_key2,
aws_secret_access_key=secret_key2,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(err, out) = rgwadmin(
ctx,
client, [
'user', 'create', '--uid', user1, '--display-name', display_name1,
'--email', email, '--access-key', access_key, '--secret',
secret_key, '--max-buckets', '4'
],
check_status=True)
# TESTCASE 'duplicate email','user','create','existing user email','fails'
(err, out) = rgwadmin(ctx, client, [
'user',
'create',
'--uid',
user2,
'--display-name',
display_name2,
'--email',
email,
])
assert err
# TESTCASE 'info-existing','user','info','existing user','returns correct info'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# this whole block should only be run if regions have been configured
if multi_region_run:
rgw_utils.radosgw_agent_sync_all(ctx)
# post-sync, validate that user1 exists on the sync destination host
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
dest_client = c_config['dest']
(err, out) = rgwadmin(ctx, dest_client,
['metadata', 'list', 'user'])
(err, out) = rgwadmin(ctx,
dest_client,
['user', 'info', '--uid', user1],
check_status=True)
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# compare the metadata between different regions, make sure it matches
log.debug(
'compare the metadata between different regions, make sure it matches'
)
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(
ctx,
source_client,
['metadata', 'get', 'user:{uid}'.format(uid=user1)],
check_status=True)
(err2, out2) = rgwadmin(
ctx,
dest_client,
['metadata', 'get', 'user:{uid}'.format(uid=user1)],
check_status=True)
assert out1 == out2
# suspend a user on the master, then check the status on the destination
log.debug(
'suspend a user on the master, then check the status on the destination'
)
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err, out) = rgwadmin(ctx, source_client,
['user', 'suspend', '--uid', user1])
rgw_utils.radosgw_agent_sync_all(ctx)
(err, out) = rgwadmin(ctx,
dest_client,
['user', 'info', '--uid', user1],
check_status=True)
assert out['suspended']
# delete a user on the master, then check that it's gone on the destination
log.debug(
'delete a user on the master, then check that it\'s gone on the destination'
)
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err, out) = rgwadmin(ctx,
source_client,
['user', 'rm', '--uid', user1],
check_status=True)
rgw_utils.radosgw_agent_sync_all(ctx)
(err, out) = rgwadmin(ctx, source_client,
['user', 'info', '--uid', user1])
assert out is None
(err, out) = rgwadmin(ctx, dest_client,
['user', 'info', '--uid', user1])
assert out is None
# then recreate it so later tests pass
(err, out) = rgwadmin(
ctx,
client, [
'user', 'create', '--uid', user1, '--display-name',
display_name1, '--email', email, '--access-key',
access_key, '--secret', secret_key, '--max-buckets', '4'
],
check_status=True)
# now do the multi-region bucket tests
log.debug('now do the multi-region bucket tests')
# Create a second user for the following tests
log.debug('Create a second user for the following tests')
(err, out) = rgwadmin(
ctx,
client, [
'user', 'create', '--uid', user2, '--display-name',
display_name2, '--email', email2, '--access-key', access_key2,
'--secret', secret_key2, '--max-buckets', '4'
],
check_status=True)
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user2],
check_status=True)
assert out is not None
# create a bucket and do a sync
log.debug('create a bucket and do a sync')
bucket = connection.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug(
'compare the metadata for the bucket between different regions, make sure it matches'
)
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(
ctx,
source_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
(err2, out2) = rgwadmin(
ctx,
dest_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
assert out1 == out2
# get the bucket.instance info and compare that
src_bucket_id = out1['data']['bucket']['bucket_id']
dest_bucket_id = out2['data']['bucket']['bucket_id']
(err1, out1) = rgwadmin(
ctx,
source_client, [
'metadata', 'get',
'bucket.instance:{bucket_name}:{bucket_instance}'.format(
bucket_name=bucket_name2,
bucket_instance=src_bucket_id)
],
check_status=True)
(err2, out2) = rgwadmin(
ctx,
dest_client, [
'metadata', 'get',
'bucket.instance:{bucket_name}:{bucket_instance}'.format(
bucket_name=bucket_name2,
bucket_instance=dest_bucket_id)
],
check_status=True)
del out1['data']['bucket_info']['bucket']['pool']
del out1['data']['bucket_info']['bucket']['index_pool']
del out2['data']['bucket_info']['bucket']['pool']
del out2['data']['bucket_info']['bucket']['index_pool']
assert out1 == out2
same_region = 0
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
source_region = rgw_utils.region_for_client(ctx, source_client)
dest_region = rgw_utils.region_for_client(ctx, dest_client)
# 301 is only returned for requests to something in a different region
if source_region == dest_region:
log.debug(
'301 is only returned for requests to something in a different region'
)
same_region += 1
continue
# Attempt to create a new connection with user1 to the destination RGW
log.debug(
'Attempt to create a new connection with user1 to the destination RGW'
)
# and use that to attempt a delete (that should fail)
(dest_remote_host,
dest_remote_port) = ctx.rgw.role_endpoints[dest_client]
connection_dest = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_remote_port,
host=dest_remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# this should fail
r, content = send_raw_http_request(connection_dest,
'DELETE',
bucket_name2,
'',
follow_redirects=False)
assert r.status == 301
# now delete the bucket on the source RGW and do another sync
log.debug(
'now delete the bucket on the source RGW and do another sync')
bucket.delete()
rgw_utils.radosgw_agent_sync_all(ctx)
if same_region == len(ctx.radosgw_agent.config):
bucket.delete()
rgw_utils.radosgw_agent_sync_all(ctx)
# make sure that the bucket no longer exists in either region
log.debug(
'make sure that the bucket no longer exists in either region')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(ctx, source_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
])
(err2, out2) = rgwadmin(ctx, dest_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
])
# Both of the previous calls should have errors due to requesting
# metadata for non-existent buckets
assert err1
assert err2
# create a bucket and then sync it
log.debug('create a bucket and then sync it')
bucket = connection.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug(
'compare the metadata for the bucket between different regions, make sure it matches'
)
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(
ctx,
source_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
(err2, out2) = rgwadmin(
ctx,
dest_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
assert out1 == out2
# Now delete the bucket and recreate it with a different user
log.debug(
'Now delete the bucket and recreate it with a different user')
# within the same window of time and then sync.
bucket.delete()
bucket = connection2.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug(
'compare the metadata for the bucket between different regions, make sure it matches'
)
# user2 should own the bucket in both regions
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(
ctx,
source_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
(err2, out2) = rgwadmin(
ctx,
dest_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
assert out1 == out2
assert out1['data']['owner'] == user2
assert out1['data']['owner'] != user1
# now we're going to use this bucket to test meta-data update propagation
log.debug(
'now we\'re going to use this bucket to test meta-data update propagation'
)
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
# get the metadata so we can tweak it
log.debug('get the metadata so we can tweak it')
(err, orig_data) = rgwadmin(
ctx,
source_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
# manually edit mtime for this bucket to be 300 seconds in the past
log.debug(
'manually edit mtime for this bucket to be 300 seconds in the past'
)
new_data = copy.deepcopy(orig_data)
mtime = datetime.datetime.strptime(
orig_data['mtime'],
"%Y-%m-%d %H:%M:%S.%fZ") - datetime.timedelta(300)
new_data['mtime'] = unicode(
mtime.strftime("%Y-%m-%d %H:%M:%S.%fZ"))
log.debug("new mtime ", mtime)
assert new_data != orig_data
(err, out) = rgwadmin(
ctx,
source_client, [
'metadata', 'put',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
stdin=StringIO(json.dumps(new_data)),
check_status=True)
# get the metadata and make sure that the 'put' worked
log.debug('get the metadata and make sure that the \'put\' worked')
(err, out) = rgwadmin(
ctx,
source_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
assert out == new_data
# sync to propagate the new metadata
log.debug('sync to propagate the new metadata')
rgw_utils.radosgw_agent_sync_all(ctx)
# get the metadata from the dest and compare it to what we just set
log.debug(
'get the metadata from the dest and compare it to what we just set'
)
# and what the source region has.
(err1, out1) = rgwadmin(
ctx,
source_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
(err2, out2) = rgwadmin(
ctx,
dest_client, [
'metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
],
check_status=True)
# yeah for the transitive property
assert out1 == out2
assert out1 == new_data
# now we delete the bucket
log.debug('now we delete the bucket')
bucket.delete()
log.debug('sync to propagate the deleted bucket')
rgw_utils.radosgw_agent_sync_all(ctx)
# Delete user2 as later tests do not expect it to exist.
# Verify that it is gone on both regions
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err, out) = rgwadmin(ctx,
source_client,
['user', 'rm', '--uid', user2],
check_status=True)
rgw_utils.radosgw_agent_sync_all(ctx)
# The two 'user info' calls should fail and not return any data
# since we just deleted this user.
(err, out) = rgwadmin(ctx, source_client,
['user', 'info', '--uid', user2])
assert out is None
(err, out) = rgwadmin(ctx, dest_client,
['user', 'info', '--uid', user2])
assert out is None
# Test data sync
# First create a bucket for data sync test purpose
bucket = connection.create_bucket(bucket_name + 'data')
# Create a tiny file and check if in sync
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
if c_config.get('metadata-only'):
continue
for full in (True, False):
source_client = c_config['src']
dest_client = c_config['dest']
k = boto.s3.key.Key(bucket)
k.key = 'tiny_file'
k.set_contents_from_string("123456789")
safety_window = rgw_utils.radosgw_data_log_window(
ctx, source_client)
time.sleep(safety_window)
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
(dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
dest_connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_port,
host=dest_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
dest_k = dest_connection.get_bucket(
bucket_name + 'data').get_key('tiny_file')
assert k.get_contents_as_string(
) == dest_k.get_contents_as_string()
# check that deleting it removes it from the dest zone
k.delete()
time.sleep(safety_window)
# full sync doesn't handle deleted objects yet
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)
dest_bucket = dest_connection.get_bucket(bucket_name + 'data')
dest_k = dest_bucket.get_key('tiny_file')
assert dest_k == None, 'object not deleted from destination zone'
# finally we delete the bucket
bucket.delete()
bucket = connection.create_bucket(bucket_name + 'data2')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
if c_config.get('metadata-only'):
continue
for full in (True, False):
source_client = c_config['src']
dest_client = c_config['dest']
(dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
dest_connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_port,
host=dest_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
for i in range(20):
k = boto.s3.key.Key(bucket)
k.key = 'tiny_file_' + str(i)
k.set_contents_from_string(str(i) * 100)
safety_window = rgw_utils.radosgw_data_log_window(
ctx, source_client)
time.sleep(safety_window)
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
for i in range(20):
dest_k = dest_connection.get_bucket(
bucket_name + 'data2').get_key('tiny_file_' + str(i))
assert (str(i) * 100) == dest_k.get_contents_as_string()
k = boto.s3.key.Key(bucket)
k.key = 'tiny_file_' + str(i)
k.delete()
# check that deleting removes the objects from the dest zone
time.sleep(safety_window)
# full sync doesn't delete deleted objects yet
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)
for i in range(20):
dest_bucket = dest_connection.get_bucket(bucket_name +
'data2')
dest_k = dest_bucket.get_key('tiny_file_' + str(i))
assert dest_k == None, 'object %d not deleted from destination zone' % i
bucket.delete()
# end of 'if multi_region_run:'
# TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
(err, out) = rgwadmin(ctx,
client, ['user', 'suspend', '--uid', user1],
check_status=True)
# TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert out['suspended']
# TESTCASE 're-enable','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx,
client, ['user', 'enable', '--uid', user1],
check_status=True)
# TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert not out['suspended']
# TESTCASE 'add-keys','key','create','w/valid info','succeeds'
(err, out) = rgwadmin(ctx,
client, [
'key',
'create',
'--uid',
user1,
'--access-key',
access_key2,
'--secret',
secret_key2,
],
check_status=True)
# TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert len(out['keys']) == 2
assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][
'access_key'] == access_key2
assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][
'secret_key'] == secret_key2
# TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
(err, out) = rgwadmin(ctx,
client, [
'key',
'rm',
'--uid',
user1,
'--access-key',
access_key2,
],
check_status=True)
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
subuser_access = 'full'
subuser_perm = 'full-control'
(err, out) = rgwadmin(ctx,
client, [
'subuser', 'create', '--subuser', subuser1,
'--access', subuser_access
],
check_status=True)
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
(err, out) = rgwadmin(ctx,
client, [
'subuser',
'modify',
'--subuser',
subuser1,
'--secret',
swift_secret1,
'--key-type',
'swift',
],
check_status=True)
# TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert out['subusers'][0]['permissions'] == subuser_perm
# TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert len(out['swift_keys']) == 1
assert out['swift_keys'][0]['user'] == subuser1
assert out['swift_keys'][0]['secret_key'] == swift_secret1
# TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
(err, out) = rgwadmin(ctx,
client, [
'subuser',
'create',
'--subuser',
subuser2,
'--secret',
swift_secret2,
'--key-type',
'swift',
],
check_status=True)
# TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert len(out['swift_keys']) == 2
assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][
'user'] == subuser2
assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[
'swift_keys'][1]['secret_key'] == swift_secret2
# TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
(err, out) = rgwadmin(ctx,
client, [
'key',
'rm',
'--subuser',
subuser1,
'--key-type',
'swift',
],
check_status=True)
assert len(out['swift_keys']) == 1
# TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
(err, out) = rgwadmin(ctx,
client, [
'subuser',
'rm',
'--subuser',
subuser1,
],
check_status=True)
assert len(out['subusers']) == 1
# TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
(err, out) = rgwadmin(ctx,
client, [
'subuser',
'rm',
'--subuser',
subuser2,
'--key-type',
'swift',
'--purge-keys',
],
check_status=True)
assert len(out['swift_keys']) == 0
assert len(out['subusers']) == 0
# TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--uid', user1],
check_status=True)
assert len(out) == 0
if multi_region_run:
rgw_utils.radosgw_agent_sync_all(ctx)
# TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'list', '--uid', user1],
check_status=True)
assert len(out) == 0
# create a first bucket
bucket = connection.create_bucket(bucket_name)
# TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'list', '--uid', user1],
check_status=True)
assert len(out) == 1
assert out[0] == bucket_name
# TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
assert len(out) >= 1
assert bucket_name in out
# TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
bucket2 = connection.create_bucket(bucket_name + '2')
bucket3 = connection.create_bucket(bucket_name + '3')
bucket4 = connection.create_bucket(bucket_name + '4')
# the 5th should fail.
failed = False
try:
connection.create_bucket(bucket_name + '5')
except Exception:
failed = True
assert failed
# delete the buckets
bucket2.delete()
bucket3.delete()
bucket4.delete()
# TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--bucket', bucket_name],
check_status=True)
assert out['owner'] == user1
bucket_id = out['id']
# TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--uid', user1],
check_status=True)
assert len(out) == 1
assert out[0][
'id'] == bucket_id # does it return the same ID twice in a row?
# use some space
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('one')
# TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--bucket', bucket_name],
check_status=True)
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 1
assert out['usage']['rgw.main']['size_kb'] > 0
# reclaim it
key.delete()
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
(err, out) = rgwadmin(
ctx,
client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
check_status=True)
# create a second user to link the bucket to
(err, out) = rgwadmin(ctx,
client, [
'user',
'create',
'--uid',
user2,
'--display-name',
display_name2,
'--access-key',
access_key2,
'--secret',
secret_key2,
'--max-buckets',
'1',
],
check_status=True)
# try creating an object with the first user before the bucket is relinked
denied = False
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('two')
except boto.exception.S3ResponseError:
denied = True
assert not denied
# delete the object
key.delete()
# link the bucket to another user
(err,
out) = rgwadmin(ctx,
client,
['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
check_status=True)
bucket_data = out['data']
assert bucket_data['bucket']['name'] == bucket_name
bucket_id = bucket_data['bucket']['bucket_id']
# link the bucket to another user
(err, out) = rgwadmin(ctx,
client, [
'bucket', 'link', '--uid', user2, '--bucket',
bucket_name, '--bucket-id', bucket_id
],
check_status=True)
# try to remove user, should fail (has a linked bucket)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
assert err
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
(err, out) = rgwadmin(
ctx,
client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
check_status=True)
# relink the bucket to the first user and delete the second user
(err, out) = rgwadmin(ctx,
client, [
'bucket', 'link', '--uid', user1, '--bucket',
bucket_name, '--bucket-id', bucket_id
],
check_status=True)
(err, out) = rgwadmin(ctx,
client, ['user', 'rm', '--uid', user2],
check_status=True)
# TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
# upload an object
object_name = 'four'
key = boto.s3.key.Key(bucket, object_name)
key.set_contents_from_string(object_name)
# now delete it
(err, out) = rgwadmin(
ctx,
client,
['object', 'rm', '--bucket', bucket_name, '--object', object_name],
check_status=True)
# TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--bucket', bucket_name],
check_status=True)
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 0
# list log objects
# TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
assert len(out) > 0
for obj in out:
# TESTCASE 'log-show','log','show','after activity','returns expected info'
if obj[:
4] == 'meta' or obj[:
4] == 'data' or obj[:
18] == 'obj_delete_at_hint':
continue
(err, rgwlog) = rgwadmin(ctx,
client, ['log', 'show', '--object', obj],
check_status=True)
assert len(rgwlog) > 0
# exempt bucket_name2 from checking as it was only used for multi-region tests
assert rgwlog['bucket'].find(
bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
assert rgwlog['bucket'] != bucket_name or rgwlog[
'bucket_id'] == bucket_id
assert rgwlog['bucket_owner'] == user1 or rgwlog[
'bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
for entry in rgwlog['log_entries']:
log.debug('checking log entry: ', entry)
assert entry['bucket'] == rgwlog['bucket']
possible_buckets = [bucket_name + '5', bucket_name2]
user = entry['user']
assert user == user1 or user.endswith('system-user') or \
rgwlog['bucket'] in possible_buckets
# TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
(err, out) = rgwadmin(ctx,
client, ['log', 'rm', '--object', obj],
check_status=True)
# TODO: show log by bucket+date
# need to wait for all usage data to get flushed, should take up to 30 seconds
timestamp = time.time()
while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes
(err, out) = rgwadmin(
ctx, client,
['usage', 'show', '--categories', 'delete_obj'
]) # last operation we did is delete obj, wait for it to flush
if get_user_successful_ops(out, user1) > 0:
break
time.sleep(1)
assert time.time() - timestamp <= (20 * 60)
# TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
(err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = get_user_summary(out, user1)
total = user_summary['total']
assert total['successful_ops'] > 0
# TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
(err, out) = rgwadmin(ctx,
client, ['usage', 'show', '--uid', user1],
check_status=True)
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = out['summary'][0]
for entry in user_summary['categories']:
assert entry['successful_ops'] > 0
assert user_summary['user'] == user1
# TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
test_categories = [
'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'
]
for cat in test_categories:
(err, out) = rgwadmin(
ctx,
client, ['usage', 'show', '--uid', user1, '--categories', cat],
check_status=True)
assert len(out['summary']) > 0
user_summary = out['summary'][0]
assert user_summary['user'] == user1
assert len(user_summary['categories']) == 1
entry = user_summary['categories'][0]
assert entry['category'] == cat
assert entry['successful_ops'] > 0
# the usage flush interval is 30 seconds, wait that much an then some
# to make sure everything has been flushed
time.sleep(35)
# TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
(err, out) = rgwadmin(ctx,
client, ['usage', 'trim', '--uid', user1],
check_status=True)
(err, out) = rgwadmin(ctx,
client, ['usage', 'show', '--uid', user1],
check_status=True)
assert len(out['entries']) == 0
assert len(out['summary']) == 0
# TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
(err, out) = rgwadmin(ctx,
client, ['user', 'suspend', '--uid', user1],
check_status=True)
# TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
try:
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('five')
except boto.exception.S3ResponseError as e:
assert e.status == 403
# TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx,
client, ['user', 'enable', '--uid', user1],
check_status=True)
# TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('six')
# TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
# create an object large enough to be split into multiple parts
test_string = 'foo' * 10000000
big_key = boto.s3.key.Key(bucket)
big_key.set_contents_from_string(test_string)
# now delete the head
big_key.delete()
# wait a bit to give the garbage collector time to cycle
time.sleep(15)
(err, out) = rgwadmin(ctx, client, ['gc', 'list'])
assert len(out) > 0
# TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
(err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)
#confirm
(err, out) = rgwadmin(ctx, client, ['gc', 'list'])
assert len(out) == 0
# TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
assert err
# delete should fail because ``key`` still exists
try:
bucket.delete()
except boto.exception.S3ResponseError as e:
assert e.status == 409
key.delete()
bucket.delete()
# TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
bucket = connection.create_bucket(bucket_name)
# create an object
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('seven')
# should be private already but guarantee it
key.set_acl('private')
(err,
out) = rgwadmin(ctx,
client,
['policy', '--bucket', bucket.name, '--object', key.key],
check_status=True,
format='xml')
acl = get_acl(key)
assert acl == out.strip('\n')
# add another grantee by making the object public read
key.set_acl('public-read')
(err,
out) = rgwadmin(ctx,
client,
['policy', '--bucket', bucket.name, '--object', key.key],
check_status=True,
format='xml')
acl = get_acl(key)
assert acl == out.strip('\n')
# TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key_name = ['eight', 'nine', 'ten', 'eleven']
for i in range(4):
key = boto.s3.key.Key(bucket)
key.set_contents_from_string(key_name[i])
(err, out) = rgwadmin(
ctx,
client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
check_status=True)
# TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
caps = 'user=read'
(err, out) = rgwadmin(ctx, client,
['caps', 'add', '--uid', user1, '--caps', caps])
assert out['caps'][0]['perm'] == 'read'
# TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
(err, out) = rgwadmin(ctx, client,
['caps', 'rm', '--uid', user1, '--caps', caps])
assert not out['caps']
# TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
assert err
# TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('twelve')
(err, out) = rgwadmin(ctx,
client,
['user', 'rm', '--uid', user1, '--purge-data'],
check_status=True)
# TESTCASE 'rm-user3','user','rm','deleted user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
#
if realm is None:
(err, out) = rgwadmin(ctx, client,
['zone', 'get', '--rgw-zone', 'default'])
else:
(err, out) = rgwadmin(ctx, client, ['zone', 'get'])
orig_placement_pools = len(out['placement_pools'])
# removed this test, it is not correct to assume that zone has default placement, it really
# depends on how we set it up before
#
# assert len(out) > 0
# assert len(out['placement_pools']) == 1
# default_rule = out['placement_pools'][0]
# assert default_rule['key'] == 'default-placement'
rule = {
'key': 'new-placement',
'val': {
'data_pool': '.rgw.buckets.2',
'index_pool': '.rgw.buckets.index.2'
}
}
out['placement_pools'].append(rule)
(err, out) = rgwadmin(ctx,
client, ['zone', 'set'],
stdin=StringIO(json.dumps(out)),
check_status=True)
if realm is None:
(err, out) = rgwadmin(ctx, client,
['zone', 'get', '--rgw-zone', 'default'])
else:
(err, out) = rgwadmin(ctx, client, ['zone', 'get'])
assert len(out) > 0
assert len(out['placement_pools']) == orig_placement_pools + 1
def configure_regions_and_zones(ctx, config, regions, role_endpoints):
"""
Configure regions and zones from rados and rgw.
"""
if not regions:
log.debug(
'In rgw.configure_regions_and_zones() and regions is None. '
'Bailing')
yield
return
log.info('Configuring regions and zones...')
log.debug('config is %r', config)
log.debug('regions are %r', regions)
log.debug('role_endpoints = %r', role_endpoints)
# extract the zone info
role_zones = dict([(client, extract_zone_info(ctx, client, c_config))
for client, c_config in config.iteritems()])
log.debug('roles_zones = %r', role_zones)
# extract the user info and append it to the payload tuple for the given
# client
for client, c_config in config.iteritems():
if not c_config:
user_info = None
else:
user_info = extract_user_info(c_config)
(region, zone, zone_info) = role_zones[client]
role_zones[client] = (region, zone, zone_info, user_info)
region_info = dict([
(region_name, extract_region_info(region_name, r_config))
for region_name, r_config in regions.iteritems()])
fill_in_endpoints(region_info, role_zones, role_endpoints)
# clear out the old defaults
first_mon = teuthology.get_first_mon(ctx, config)
(mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
# removing these objects from .rgw.root and the per-zone root pools
# may or may not matter
rados(ctx, mon,
cmd=['-p', '.rgw.root', 'rm', 'region_info.default'])
rados(ctx, mon,
cmd=['-p', '.rgw.root', 'rm', 'zone_info.default'])
for client in config.iterkeys():
for role, (_, zone, zone_info, user_info) in role_zones.iteritems():
rados(ctx, mon,
cmd=['-p', zone_info['domain_root'],
'rm', 'region_info.default'])
rados(ctx, mon,
cmd=['-p', zone_info['domain_root'],
'rm', 'zone_info.default'])
(remote,) = ctx.cluster.only(role).remotes.keys()
for pool_info in zone_info['placement_pools']:
remote.run(args=['ceph', 'osd', 'pool', 'create',
pool_info['val']['index_pool'], '64', '64'])
if ctx.rgw.ec_data_pool:
create_ec_pool(remote, pool_info['val']['data_pool'],
zone, 64, ctx.rgw.erasure_code_profile)
else:
create_replicated_pool(
remote, pool_info['val']['data_pool'],
64)
rgwadmin(ctx, client,
cmd=['-n', client, 'zone', 'set', '--rgw-zone', zone],
stdin=StringIO(json.dumps(dict(
zone_info.items() + user_info.items()))),
check_status=True)
for region, info in region_info.iteritems():
region_json = json.dumps(info)
log.debug('region info is: %s', region_json)
rgwadmin(ctx, client,
cmd=['-n', client, 'region', 'set'],
stdin=StringIO(region_json),
check_status=True)
if info['is_master']:
rgwadmin(ctx, client,
cmd=['-n', client,
'region', 'default',
'--rgw-region', region],
check_status=True)
rgwadmin(ctx, client, cmd=['-n', client, 'regionmap', 'update'])
yield
def task(ctx, config):
"""
Test radosgw-admin functionality against a running rgw instance.
"""
global log
assert ctx.rgw.config, \
"radosgw_admin task needs a config passed from the rgw task"
config = ctx.rgw.config
log.debug('config is: %r', config)
clients_from_config = config.keys()
# choose first client as default
client = clients_from_config[0]
# once the client is chosen, pull the host name and assigned port out of
# the role_endpoints that were assigned by the rgw task
(remote_host, remote_port) = ctx.rgw.role_endpoints[client]
##
user1='foo'
user2='fud'
subuser1='foo:foo1'
subuser2='foo:foo2'
display_name1='Foo'
display_name2='Fud'
email='*****@*****.**'
email2='*****@*****.**'
access_key='9te6NH5mcdcq0Tc5i8i1'
secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
access_key2='p5YnriCv1nAtykxBrupQ'
secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
bucket_name='myfoo'
bucket_name2='mybar'
# connect to rgw
connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
connection2 = boto.s3.connection.S3Connection(
aws_access_key_id=access_key2,
aws_secret_access_key=secret_key2,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user1,
'--display-name', display_name1,
'--email', email,
'--access-key', access_key,
'--secret', secret_key,
'--max-buckets', '4'
],
check_status=True)
# TESTCASE 'duplicate email','user','create','existing user email','fails'
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user2,
'--display-name', display_name2,
'--email', email,
])
assert err
# TESTCASE 'info-existing','user','info','existing user','returns correct info'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
check_status=True)
# TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert out['suspended']
# TESTCASE 're-enable','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True)
# TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert not out['suspended']
# TESTCASE 'add-keys','key','create','w/valid info','succeeds'
(err, out) = rgwadmin(ctx, client, [
'key', 'create', '--uid', user1,
'--access-key', access_key2, '--secret', secret_key2,
], check_status=True)
# TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1],
check_status=True)
assert len(out['keys']) == 2
assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
# TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
(err, out) = rgwadmin(ctx, client, [
'key', 'rm', '--uid', user1,
'--access-key', access_key2,
], check_status=True)
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
subuser_access = 'full'
subuser_perm = 'full-control'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'create', '--subuser', subuser1,
'--access', subuser_access
], check_status=True)
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'modify', '--subuser', subuser1,
'--secret', swift_secret1,
'--key-type', 'swift',
], check_status=True)
# TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert out['subusers'][0]['permissions'] == subuser_perm
# TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert len(out['swift_keys']) == 1
assert out['swift_keys'][0]['user'] == subuser1
assert out['swift_keys'][0]['secret_key'] == swift_secret1
# TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'create', '--subuser', subuser2,
'--secret', swift_secret2,
'--key-type', 'swift',
], check_status=True)
# TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert len(out['swift_keys']) == 2
assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
# TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
(err, out) = rgwadmin(ctx, client, [
'key', 'rm', '--subuser', subuser1,
'--key-type', 'swift',
], check_status=True)
assert len(out['swift_keys']) == 1
# TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'rm', '--subuser', subuser1,
], check_status=True)
assert len(out['subusers']) == 1
# TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'rm', '--subuser', subuser2,
'--key-type', 'swift', '--purge-keys',
], check_status=True)
assert len(out['swift_keys']) == 0
assert len(out['subusers']) == 0
# TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1],
check_status=True)
assert len(out) == 0
# TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
assert len(out) == 0
# create a first bucket
bucket = connection.create_bucket(bucket_name)
# TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
assert len(out) == 1
assert out[0] == bucket_name
# TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
assert len(out) >= 1
assert bucket_name in out;
# TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
bucket2 = connection.create_bucket(bucket_name + '2')
bucket3 = connection.create_bucket(bucket_name + '3')
bucket4 = connection.create_bucket(bucket_name + '4')
# the 5th should fail.
failed = False
try:
connection.create_bucket(bucket_name + '5')
except Exception:
failed = True
assert failed
# delete the buckets
bucket2.delete()
bucket3.delete()
bucket4.delete()
# TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, [
'bucket', 'stats', '--bucket', bucket_name], check_status=True)
assert out['owner'] == user1
bucket_id = out['id']
# TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
(err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True)
assert len(out) == 1
assert out[0]['id'] == bucket_id # does it return the same ID twice in a row?
# use some space
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('one')
# TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
(err, out) = rgwadmin(ctx, client, [
'bucket', 'stats', '--bucket', bucket_name], check_status=True)
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 1
assert out['usage']['rgw.main']['size_kb'] > 0
# reclaim it
key.delete()
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
(err, out) = rgwadmin(ctx, client,
['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
check_status=True)
# create a second user to link the bucket to
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user2,
'--display-name', display_name2,
'--access-key', access_key2,
'--secret', secret_key2,
'--max-buckets', '1',
],
check_status=True)
# try creating an object with the first user before the bucket is relinked
denied = False
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('two')
except boto.exception.S3ResponseError:
denied = True
assert not denied
# delete the object
key.delete()
# link the bucket to another user
(err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
check_status=True)
bucket_data = out['data']
assert bucket_data['bucket']['name'] == bucket_name
bucket_id = bucket_data['bucket']['bucket_id']
# link the bucket to another user
(err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id],
check_status=True)
# try to remove user, should fail (has a linked bucket)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
assert err
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
(err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
check_status=True)
# relink the bucket to the first user and delete the second user
(err, out) = rgwadmin(ctx, client,
['bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id],
check_status=True)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2],
check_status=True)
# TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
# upload an object
object_name = 'four'
key = boto.s3.key.Key(bucket, object_name)
key.set_contents_from_string(object_name)
# now delete it
(err, out) = rgwadmin(ctx, client,
['object', 'rm', '--bucket', bucket_name, '--object', object_name],
check_status=True)
# TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx, client, [
'bucket', 'stats', '--bucket', bucket_name],
check_status=True)
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 0
# list log objects
# TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
assert len(out) > 0
for obj in out:
# TESTCASE 'log-show','log','show','after activity','returns expected info'
if obj[:4] == 'meta' or obj[:4] == 'data' or obj[:18] == 'obj_delete_at_hint':
continue
(err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj],
check_status=True)
assert len(rgwlog) > 0
# exempt bucket_name2 from checking as it was only used for multi-region tests
assert rgwlog['bucket'].find(bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
assert rgwlog['bucket'] != bucket_name or rgwlog['bucket_id'] == bucket_id
assert rgwlog['bucket_owner'] == user1 or rgwlog['bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
for entry in rgwlog['log_entries']:
log.debug('checking log entry: ', entry)
assert entry['bucket'] == rgwlog['bucket']
possible_buckets = [bucket_name + '5', bucket_name2]
user = entry['user']
assert user == user1 or user.endswith('system-user') or \
rgwlog['bucket'] in possible_buckets
# TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
(err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj],
check_status=True)
# TODO: show log by bucket+date
# need to wait for all usage data to get flushed, should take up to 30 seconds
timestamp = time.time()
while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes
(err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj']) # last operation we did is delete obj, wait for it to flush
if get_user_successful_ops(out, user1) > 0:
break
time.sleep(1)
assert time.time() - timestamp <= (20 * 60)
# TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
(err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = get_user_summary(out, user1)
total = user_summary['total']
assert total['successful_ops'] > 0
# TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
(err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
check_status=True)
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = out['summary'][0]
for entry in user_summary['categories']:
assert entry['successful_ops'] > 0
assert user_summary['user'] == user1
# TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
for cat in test_categories:
(err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat],
check_status=True)
assert len(out['summary']) > 0
user_summary = out['summary'][0]
assert user_summary['user'] == user1
assert len(user_summary['categories']) == 1
entry = user_summary['categories'][0]
assert entry['category'] == cat
assert entry['successful_ops'] > 0
# the usage flush interval is 30 seconds, wait that much an then some
# to make sure everything has been flushed
time.sleep(35)
# TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
(err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1],
check_status=True)
(err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
check_status=True)
assert len(out['entries']) == 0
assert len(out['summary']) == 0
# TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
check_status=True)
# TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
try:
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('five')
except boto.exception.S3ResponseError as e:
assert e.status == 403
# TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1],
check_status=True)
# TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('six')
# TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
# create an object large enough to be split into multiple parts
test_string = 'foo'*10000000
big_key = boto.s3.key.Key(bucket)
big_key.set_contents_from_string(test_string)
# now delete the head
big_key.delete()
# wait a bit to give the garbage collector time to cycle
time.sleep(15)
(err, out) = rgwadmin(ctx, client, ['gc', 'list'])
assert len(out) > 0
# TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
(err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)
#confirm
(err, out) = rgwadmin(ctx, client, ['gc', 'list'])
assert len(out) == 0
# TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
assert err
# delete should fail because ``key`` still exists
try:
bucket.delete()
except boto.exception.S3ResponseError as e:
assert e.status == 409
key.delete()
bucket.delete()
# TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
bucket = connection.create_bucket(bucket_name)
# create an object
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('seven')
# should be private already but guarantee it
key.set_acl('private')
(err, out) = rgwadmin(ctx, client,
['policy', '--bucket', bucket.name, '--object', key.key],
check_status=True, format='xml')
acl = get_acl(key)
assert acl == out.strip('\n')
# add another grantee by making the object public read
key.set_acl('public-read')
(err, out) = rgwadmin(ctx, client,
['policy', '--bucket', bucket.name, '--object', key.key],
check_status=True, format='xml')
acl = get_acl(key)
assert acl == out.strip('\n')
# TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key_name = ['eight', 'nine', 'ten', 'eleven']
for i in range(4):
key = boto.s3.key.Key(bucket)
key.set_contents_from_string(key_name[i])
(err, out) = rgwadmin(ctx, client,
['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
check_status=True)
# TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
caps='user=read'
(err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps])
assert out['caps'][0]['perm'] == 'read'
# TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
(err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps])
assert not out['caps']
# TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
assert err
# TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('twelve')
(err, out) = rgwadmin(ctx, client,
['user', 'rm', '--uid', user1, '--purge-data' ],
check_status=True)
# TESTCASE 'rm-user3','user','rm','deleted user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
#
(err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
orig_placement_pools = len(out['placement_pools'])
# removed this test, it is not correct to assume that zone has default placement, it really
# depends on how we set it up before
#
# assert len(out) > 0
# assert len(out['placement_pools']) == 1
# default_rule = out['placement_pools'][0]
# assert default_rule['key'] == 'default-placement'
rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}
out['placement_pools'].append(rule)
(err, out) = rgwadmin(ctx, client, ['zone', 'set'],
stdin=StringIO(json.dumps(out)),
check_status=True)
(err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
assert len(out) > 0
assert len(out['placement_pools']) == orig_placement_pools + 1
def task(ctx, config):
"""
Test radosgw-admin functionality through the RESTful interface
"""
assert config is None or isinstance(config, list) \
or isinstance(config, dict), \
"task s3tests only supports a list or dictionary for configuration"
all_clients = ['client.{id}'.format(id=id_)
for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
if config is None:
config = all_clients
if isinstance(config, list):
config = dict.fromkeys(config)
clients = config.keys()
# just use the first client...
client = clients[0]
##
admin_user = '******'
admin_display_name = 'Ms. Admin User'
admin_access_key = 'MH1WC2XQ1S8UISFDZC8W'
admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write'
user1 = 'foo'
user2 = 'fud'
subuser1 = 'foo:foo1'
subuser2 = 'foo:foo2'
display_name1 = 'Foo'
display_name2 = 'Fud'
email = '*****@*****.**'
access_key = '9te6NH5mcdcq0Tc5i8i1'
secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
access_key2 = 'p5YnriCv1nAtykxBrupQ'
secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
bucket_name = 'myfoo'
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'create-admin-user','user','create','administrative user','succeeds'
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', admin_user,
'--display-name', admin_display_name,
'--access-key', admin_access_key,
'--secret', admin_secret_key,
'--max-buckets', '0',
'--caps', admin_caps
])
logging.error(out)
logging.error(err)
assert not err
assert hasattr(ctx, 'rgw'), 'radosgw-admin-rest must run after the rgw task'
endpoint = ctx.rgw.role_endpoints.get(client)
assert endpoint, 'no rgw endpoint for {}'.format(client)
admin_conn = boto.s3.connection.S3Connection(
aws_access_key_id=admin_access_key,
aws_secret_access_key=admin_secret_key,
is_secure=True if endpoint.cert else False,
port=endpoint.port,
host=endpoint.hostname,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1})
assert ret == 404
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(ret, out) = rgwadmin_rest(admin_conn,
['user', 'create'],
{'uid' : user1,
'display-name' : display_name1,
'email' : email,
'access-key' : access_key,
'secret-key' : secret_key,
'max-buckets' : '4'
})
assert ret == 200
# TESTCASE 'list-no-user','user','list','list user keys','user list object'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 0})
assert ret == 200
assert out['count'] == 0
assert out['truncated'] == True
assert len(out['keys']) == 0
assert len(out['marker']) > 0
# TESTCASE 'list-user-without-marker','user','list','list user keys','user list object'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 1})
assert ret == 200
assert out['count'] == 1
assert out['truncated'] == True
assert len(out['keys']) == 1
assert len(out['marker']) > 0
marker = out['marker']
# TESTCASE 'list-user-with-marker','user','list','list user keys','user list object'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {'list' : '', 'max-entries' : 1, 'marker': marker})
assert ret == 200
assert out['count'] == 1
assert out['truncated'] == False
assert len(out['keys']) == 1
# TESTCASE 'info-existing','user','info','existing user','returns correct info'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# TESTCASE 'info-existing','user','info','existing user query with wrong uid but correct access key','returns correct info'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'access-key' : access_key, 'uid': 'uid_not_exist'})
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
assert ret == 200
# TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert out['suspended']
assert out['email'] == email
# TESTCASE 're-enable','user','enable','suspended user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'})
assert not err
# TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert not out['suspended']
# TESTCASE 'add-keys','key','create','w/valid info','succeeds'
(ret, out) = rgwadmin_rest(admin_conn,
['key', 'create'],
{'uid' : user1,
'access-key' : access_key2,
'secret-key' : secret_key2
})
assert ret == 200
# TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert len(out['keys']) == 2
assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
# TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
(ret, out) = rgwadmin_rest(admin_conn,
['key', 'rm'],
{'uid' : user1,
'access-key' : access_key2
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
(ret, out) = rgwadmin_rest(admin_conn,
['subuser', 'create'],
{'subuser' : subuser1,
'secret-key' : swift_secret1,
'key-type' : 'swift'
})
assert ret == 200
# TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert len(out['swift_keys']) == 1
assert out['swift_keys'][0]['user'] == subuser1
assert out['swift_keys'][0]['secret_key'] == swift_secret1
# TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
(ret, out) = rgwadmin_rest(admin_conn,
['subuser', 'create'],
{'subuser' : subuser2,
'secret-key' : swift_secret2,
'key-type' : 'swift'
})
assert ret == 200
# TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 200
assert len(out['swift_keys']) == 2
assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
# TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
(ret, out) = rgwadmin_rest(admin_conn,
['key', 'rm'],
{'subuser' : subuser1,
'key-type' :'swift'
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert len(out['swift_keys']) == 1
# TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
(ret, out) = rgwadmin_rest(admin_conn,
['subuser', 'rm'],
{'subuser' : subuser1
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert len(out['subusers']) == 1
# TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
(ret, out) = rgwadmin_rest(admin_conn,
['subuser', 'rm'],
{'subuser' : subuser2,
'key-type' : 'swift',
'{purge-keys' :True
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert len(out['swift_keys']) == 0
assert len(out['subusers']) == 0
# TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1})
assert ret == 200
assert len(out) == 0
# connect to rgw
connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=True if endpoint.cert else False,
port=endpoint.port,
host=endpoint.hostname,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
assert ret == 200
assert len(out) == 0
# create a first bucket
bucket = connection.create_bucket(bucket_name)
# TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1})
assert ret == 200
assert len(out) == 1
assert out[0] == bucket_name
# TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn,
['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
assert ret == 200
assert out['owner'] == user1
assert out['tenant'] == ''
bucket_id = out['id']
# TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
assert ret == 200
assert len(out) == 1
assert out[0]['id'] == bucket_id # does it return the same ID twice in a row?
# use some space
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('one')
# TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
assert ret == 200
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 1
assert out['usage']['rgw.main']['size_kb'] > 0
# reclaim it
key.delete()
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], {'uid' : user1, 'bucket' : bucket_name})
assert ret == 200
# create a second user to link the bucket to
(ret, out) = rgwadmin_rest(admin_conn,
['user', 'create'],
{'uid' : user2,
'display-name' : display_name2,
'access-key' : access_key2,
'secret-key' : secret_key2,
'max-buckets' : '1',
})
assert ret == 200
# try creating an object with the first user before the bucket is relinked
denied = False
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('two')
except boto.exception.S3ResponseError:
denied = True
assert not denied
# delete the object
key.delete()
# link the bucket to another user
(ret, out) = rgwadmin_rest(admin_conn,
['bucket', 'link'],
{'uid' : user2,
'bucket' : bucket_name,
'bucket-id' : bucket_id,
})
assert ret == 200
# try creating an object with the first user which should cause an error
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('three')
except boto.exception.S3ResponseError:
denied = True
assert denied
# relink the bucket to the first user and delete the second user
(ret, out) = rgwadmin_rest(admin_conn,
['bucket', 'link'],
{'uid' : user1,
'bucket' : bucket_name,
'bucket-id' : bucket_id,
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user2})
assert ret == 200
# TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
# upload an object
object_name = 'four'
key = boto.s3.key.Key(bucket, object_name)
key.set_contents_from_string(object_name)
# now delete it
(ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {'bucket' : bucket_name, 'object' : object_name})
assert ret == 200
# TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
assert ret == 200
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 0
# create a bucket for deletion stats
useless_bucket = connection.create_bucket('useless_bucket')
useless_key = useless_bucket.new_key('useless_key')
useless_key.set_contents_from_string('useless string')
# delete it
useless_key.delete()
useless_bucket.delete()
# wait for the statistics to flush
time.sleep(60)
# need to wait for all usage data to get flushed, should take up to 30 seconds
timestamp = time.time()
while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'categories' : 'delete_obj'}) # last operation we did is delete obj, wait for it to flush
if get_user_successful_ops(out, user1) > 0:
break
time.sleep(1)
assert time.time() - timestamp <= (20 * 60)
# TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'])
assert ret == 200
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = get_user_summary(out, user1)
total = user_summary['total']
assert total['successful_ops'] > 0
# TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
assert ret == 200
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = out['summary'][0]
for entry in user_summary['categories']:
assert entry['successful_ops'] > 0
assert user_summary['user'] == user1
# TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
for cat in test_categories:
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1, 'categories' : cat})
assert ret == 200
assert len(out['summary']) > 0
user_summary = out['summary'][0]
assert user_summary['user'] == user1
assert len(user_summary['categories']) == 1
entry = user_summary['categories'][0]
assert entry['category'] == cat
assert entry['successful_ops'] > 0
# TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid' : user1})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
assert ret == 200
assert len(out['entries']) == 0
assert len(out['summary']) == 0
# TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
assert ret == 200
# TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
try:
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('five')
except boto.exception.S3ResponseError as e:
assert e.status == 403
# TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'})
assert ret == 200
# TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('six')
# TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection'
# create an object large enough to be split into multiple parts
test_string = 'foo'*10000000
big_key = boto.s3.key.Key(bucket)
big_key.set_contents_from_string(test_string)
# now delete the head
big_key.delete()
# TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
assert ret == 409
# delete should fail because ``key`` still exists
try:
bucket.delete()
except boto.exception.S3ResponseError as e:
assert e.status == 409
key.delete()
bucket.delete()
# TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
bucket = connection.create_bucket(bucket_name)
# create an object
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('seven')
# should be private already but guarantee it
key.set_acl('private')
(ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
assert ret == 200
assert len(out['acl']['grant_map']) == 1
# add another grantee by making the object public read
key.set_acl('public-read')
(ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
assert ret == 200
assert len(out['acl']['grant_map']) == 2
# TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key_name = ['eight', 'nine', 'ten', 'eleven']
for i in range(4):
key = boto.s3.key.Key(bucket)
key.set_contents_from_string(key_name[i])
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], {'bucket' : bucket_name, 'purge-objects' : True})
assert ret == 200
# TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
caps = 'usage=read'
(ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {'uid' : user1, 'user-caps' : caps})
assert ret == 200
assert out[0]['perm'] == 'read'
# TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {'uid' : user1, 'user-caps' : caps})
assert ret == 200
assert not out
# TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
assert ret == 409
# TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('twelve')
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1, 'purge-data' : True})
assert ret == 200
# TESTCASE 'rm-user3','user','info','deleted user','fails'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
assert ret == 404
def task(ctx, config):
"""
Test radosgw-admin functionality against a running rgw instance.
"""
global log
assert (
config is None or isinstance(config, list) or isinstance(config, dict)
), "task s3tests only supports a list or dictionary for configuration"
all_clients = ["client.{id}".format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, "client")]
if config is None:
config = all_clients
if isinstance(config, list):
config = dict.fromkeys(config)
clients = config.keys()
multi_region_run = rgw_utils.multi_region_enabled(ctx)
client = clients[0]
# default choice, multi-region code may overwrite this
if multi_region_run:
client = rgw_utils.get_master_client(ctx, clients)
# once the client is chosen, pull the host name and assigned port out of
# the role_endpoints that were assigned by the rgw task
(remote_host, remote_port) = ctx.rgw.role_endpoints[client]
##
user1 = "foo"
user2 = "fud"
subuser1 = "foo:foo1"
subuser2 = "foo:foo2"
display_name1 = "Foo"
display_name2 = "Fud"
email = "*****@*****.**"
email2 = "*****@*****.**"
access_key = "9te6NH5mcdcq0Tc5i8i1"
secret_key = "Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu"
access_key2 = "p5YnriCv1nAtykxBrupQ"
secret_key2 = "Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh"
swift_secret1 = "gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL"
swift_secret2 = "ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy"
bucket_name = "myfoo"
bucket_name2 = "mybar"
# connect to rgw
connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
connection2 = boto.s3.connection.S3Connection(
aws_access_key_id=access_key2,
aws_secret_access_key=secret_key2,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1])
assert err
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(err, out) = rgwadmin(
ctx,
client,
[
"user",
"create",
"--uid",
user1,
"--display-name",
display_name1,
"--email",
email,
"--access-key",
access_key,
"--secret",
secret_key,
"--max-buckets",
"4",
],
check_status=True,
)
# TESTCASE 'duplicate email','user','create','existing user email','fails'
(err, out) = rgwadmin(
ctx, client, ["user", "create", "--uid", user2, "--display-name", display_name2, "--email", email]
)
assert err
# TESTCASE 'info-existing','user','info','existing user','returns correct info'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
assert out["user_id"] == user1
assert out["email"] == email
assert out["display_name"] == display_name1
assert len(out["keys"]) == 1
assert out["keys"][0]["access_key"] == access_key
assert out["keys"][0]["secret_key"] == secret_key
assert not out["suspended"]
# this whole block should only be run if regions have been configured
if multi_region_run:
rgw_utils.radosgw_agent_sync_all(ctx)
# post-sync, validate that user1 exists on the sync destination host
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
dest_client = c_config["dest"]
(err, out) = rgwadmin(ctx, dest_client, ["metadata", "list", "user"])
(err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1], check_status=True)
assert out["user_id"] == user1
assert out["email"] == email
assert out["display_name"] == display_name1
assert len(out["keys"]) == 1
assert out["keys"][0]["access_key"] == access_key
assert out["keys"][0]["secret_key"] == secret_key
assert not out["suspended"]
# compare the metadata between different regions, make sure it matches
log.debug("compare the metadata between different regions, make sure it matches")
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
(err1, out1) = rgwadmin(
ctx, source_client, ["metadata", "get", "user:{uid}".format(uid=user1)], check_status=True
)
(err2, out2) = rgwadmin(
ctx, dest_client, ["metadata", "get", "user:{uid}".format(uid=user1)], check_status=True
)
assert out1 == out2
# suspend a user on the master, then check the status on the destination
log.debug("suspend a user on the master, then check the status on the destination")
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
(err, out) = rgwadmin(ctx, source_client, ["user", "suspend", "--uid", user1])
rgw_utils.radosgw_agent_sync_all(ctx)
(err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1], check_status=True)
assert out["suspended"]
# delete a user on the master, then check that it's gone on the destination
log.debug("delete a user on the master, then check that it's gone on the destination")
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
(err, out) = rgwadmin(ctx, source_client, ["user", "rm", "--uid", user1], check_status=True)
rgw_utils.radosgw_agent_sync_all(ctx)
(err, out) = rgwadmin(ctx, source_client, ["user", "info", "--uid", user1])
assert out is None
(err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1])
assert out is None
# then recreate it so later tests pass
(err, out) = rgwadmin(
ctx,
client,
[
"user",
"create",
"--uid",
user1,
"--display-name",
display_name1,
"--email",
email,
"--access-key",
access_key,
"--secret",
secret_key,
"--max-buckets",
"4",
],
check_status=True,
)
# now do the multi-region bucket tests
log.debug("now do the multi-region bucket tests")
# Create a second user for the following tests
log.debug("Create a second user for the following tests")
(err, out) = rgwadmin(
ctx,
client,
[
"user",
"create",
"--uid",
user2,
"--display-name",
display_name2,
"--email",
email2,
"--access-key",
access_key2,
"--secret",
secret_key2,
"--max-buckets",
"4",
],
check_status=True,
)
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user2], check_status=True)
assert out is not None
# create a bucket and do a sync
log.debug("create a bucket and do a sync")
bucket = connection.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug("compare the metadata for the bucket between different regions, make sure it matches")
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
(err1, out1) = rgwadmin(
ctx,
source_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
(err2, out2) = rgwadmin(
ctx,
dest_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
assert out1 == out2
# get the bucket.instance info and compare that
src_bucket_id = out1["data"]["bucket"]["bucket_id"]
dest_bucket_id = out2["data"]["bucket"]["bucket_id"]
(err1, out1) = rgwadmin(
ctx,
source_client,
[
"metadata",
"get",
"bucket.instance:{bucket_name}:{bucket_instance}".format(
bucket_name=bucket_name2, bucket_instance=src_bucket_id
),
],
check_status=True,
)
(err2, out2) = rgwadmin(
ctx,
dest_client,
[
"metadata",
"get",
"bucket.instance:{bucket_name}:{bucket_instance}".format(
bucket_name=bucket_name2, bucket_instance=dest_bucket_id
),
],
check_status=True,
)
del out1["data"]["bucket_info"]["bucket"]["pool"]
del out1["data"]["bucket_info"]["bucket"]["index_pool"]
del out2["data"]["bucket_info"]["bucket"]["pool"]
del out2["data"]["bucket_info"]["bucket"]["index_pool"]
assert out1 == out2
same_region = 0
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
source_region = rgw_utils.region_for_client(ctx, source_client)
dest_region = rgw_utils.region_for_client(ctx, dest_client)
# 301 is only returned for requests to something in a different region
if source_region == dest_region:
log.debug("301 is only returned for requests to something in a different region")
same_region += 1
continue
# Attempt to create a new connection with user1 to the destination RGW
log.debug("Attempt to create a new connection with user1 to the destination RGW")
# and use that to attempt a delete (that should fail)
exception_encountered = False
try:
(dest_remote_host, dest_remote_port) = ctx.rgw.role_endpoints[dest_client]
connection_dest = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_remote_port,
host=dest_remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# this should fail
connection_dest.delete_bucket(bucket_name2)
except boto.exception.S3ResponseError as e:
assert e.status == 301
exception_encountered = True
# confirm that the expected exception was seen
assert exception_encountered
# now delete the bucket on the source RGW and do another sync
log.debug("now delete the bucket on the source RGW and do another sync")
bucket.delete()
rgw_utils.radosgw_agent_sync_all(ctx)
if same_region == len(ctx.radosgw_agent.config):
bucket.delete()
rgw_utils.radosgw_agent_sync_all(ctx)
# make sure that the bucket no longer exists in either region
log.debug("make sure that the bucket no longer exists in either region")
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
(err1, out1) = rgwadmin(
ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)]
)
(err2, out2) = rgwadmin(
ctx, dest_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)]
)
# Both of the previous calls should have errors due to requesting
# metadata for non-existent buckets
assert err1
assert err2
# create a bucket and then sync it
log.debug("create a bucket and then sync it")
bucket = connection.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug("compare the metadata for the bucket between different regions, make sure it matches")
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
(err1, out1) = rgwadmin(
ctx,
source_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
(err2, out2) = rgwadmin(
ctx,
dest_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
assert out1 == out2
# Now delete the bucket and recreate it with a different user
log.debug("Now delete the bucket and recreate it with a different user")
# within the same window of time and then sync.
bucket.delete()
bucket = connection2.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug("compare the metadata for the bucket between different regions, make sure it matches")
# user2 should own the bucket in both regions
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
(err1, out1) = rgwadmin(
ctx,
source_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
(err2, out2) = rgwadmin(
ctx,
dest_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
assert out1 == out2
assert out1["data"]["owner"] == user2
assert out1["data"]["owner"] != user1
# now we're going to use this bucket to test meta-data update propagation
log.debug("now we're going to use this bucket to test meta-data update propagation")
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
# get the metadata so we can tweak it
log.debug("get the metadata so we can tweak it")
(err, orig_data) = rgwadmin(
ctx,
source_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
# manually edit mtime for this bucket to be 300 seconds in the past
log.debug("manually edit mtime for this bucket to be 300 seconds in the past")
new_data = copy.deepcopy(orig_data)
new_data["mtime"] = orig_data["mtime"] - 300
assert new_data != orig_data
(err, out) = rgwadmin(
ctx,
source_client,
["metadata", "put", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
stdin=StringIO(json.dumps(new_data)),
check_status=True,
)
# get the metadata and make sure that the 'put' worked
log.debug("get the metadata and make sure that the 'put' worked")
(err, out) = rgwadmin(
ctx,
source_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
assert out == new_data
# sync to propagate the new metadata
log.debug("sync to propagate the new metadata")
rgw_utils.radosgw_agent_sync_all(ctx)
# get the metadata from the dest and compare it to what we just set
log.debug("get the metadata from the dest and compare it to what we just set")
# and what the source region has.
(err1, out1) = rgwadmin(
ctx,
source_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
(err2, out2) = rgwadmin(
ctx,
dest_client,
["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
check_status=True,
)
# yeah for the transitive property
assert out1 == out2
assert out1 == new_data
# now we delete the bucket
log.debug("now we delete the bucket")
bucket.delete()
log.debug("sync to propagate the deleted bucket")
rgw_utils.radosgw_agent_sync_all(ctx)
# Delete user2 as later tests do not expect it to exist.
# Verify that it is gone on both regions
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config["src"]
dest_client = c_config["dest"]
(err, out) = rgwadmin(ctx, source_client, ["user", "rm", "--uid", user2], check_status=True)
rgw_utils.radosgw_agent_sync_all(ctx)
# The two 'user info' calls should fail and not return any data
# since we just deleted this user.
(err, out) = rgwadmin(ctx, source_client, ["user", "info", "--uid", user2])
assert out is None
(err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user2])
assert out is None
# Test data sync
# First create a bucket for data sync test purpose
bucket = connection.create_bucket(bucket_name + "data")
# Create a tiny file and check if in sync
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
if c_config.get("metadata-only"):
continue
for full in (True, False):
source_client = c_config["src"]
dest_client = c_config["dest"]
k = boto.s3.key.Key(bucket)
k.key = "tiny_file"
k.set_contents_from_string("123456789")
safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client)
time.sleep(safety_window)
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
(dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
dest_connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_port,
host=dest_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
dest_k = dest_connection.get_bucket(bucket_name + "data").get_key("tiny_file")
assert k.get_contents_as_string() == dest_k.get_contents_as_string()
# check that deleting it removes it from the dest zone
k.delete()
time.sleep(safety_window)
# full sync doesn't handle deleted objects yet
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)
dest_bucket = dest_connection.get_bucket(bucket_name + "data")
dest_k = dest_bucket.get_key("tiny_file")
assert dest_k == None, "object not deleted from destination zone"
# finally we delete the bucket
bucket.delete()
bucket = connection.create_bucket(bucket_name + "data2")
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
if c_config.get("metadata-only"):
continue
for full in (True, False):
source_client = c_config["src"]
dest_client = c_config["dest"]
(dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
dest_connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_port,
host=dest_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
for i in range(20):
k = boto.s3.key.Key(bucket)
k.key = "tiny_file_" + str(i)
k.set_contents_from_string(str(i) * 100)
safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client)
time.sleep(safety_window)
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
for i in range(20):
dest_k = dest_connection.get_bucket(bucket_name + "data2").get_key("tiny_file_" + str(i))
assert (str(i) * 100) == dest_k.get_contents_as_string()
k = boto.s3.key.Key(bucket)
k.key = "tiny_file_" + str(i)
k.delete()
# check that deleting removes the objects from the dest zone
time.sleep(safety_window)
# full sync doesn't delete deleted objects yet
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)
for i in range(20):
dest_bucket = dest_connection.get_bucket(bucket_name + "data2")
dest_k = dest_bucket.get_key("tiny_file_" + str(i))
assert dest_k == None, "object %d not deleted from destination zone" % i
bucket.delete()
# end of 'if multi_region_run:'
# TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
(err, out) = rgwadmin(ctx, client, ["user", "suspend", "--uid", user1], check_status=True)
# TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
assert out["suspended"]
# TESTCASE 're-enable','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx, client, ["user", "enable", "--uid", user1], check_status=True)
# TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
assert not out["suspended"]
# TESTCASE 'add-keys','key','create','w/valid info','succeeds'
(err, out) = rgwadmin(
ctx,
client,
["key", "create", "--uid", user1, "--access-key", access_key2, "--secret", secret_key2],
check_status=True,
)
# TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
assert len(out["keys"]) == 2
assert out["keys"][0]["access_key"] == access_key2 or out["keys"][1]["access_key"] == access_key2
assert out["keys"][0]["secret_key"] == secret_key2 or out["keys"][1]["secret_key"] == secret_key2
# TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
(err, out) = rgwadmin(ctx, client, ["key", "rm", "--uid", user1, "--access-key", access_key2], check_status=True)
assert len(out["keys"]) == 1
assert out["keys"][0]["access_key"] == access_key
assert out["keys"][0]["secret_key"] == secret_key
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
subuser_access = "full"
subuser_perm = "full-control"
(err, out) = rgwadmin(
ctx, client, ["subuser", "create", "--subuser", subuser1, "--access", subuser_access], check_status=True
)
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
(err, out) = rgwadmin(
ctx,
client,
["subuser", "modify", "--subuser", subuser1, "--secret", swift_secret1, "--key-type", "swift"],
check_status=True,
)
# TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1])
assert out["subusers"][0]["permissions"] == subuser_perm
# TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
assert len(out["swift_keys"]) == 1
assert out["swift_keys"][0]["user"] == subuser1
assert out["swift_keys"][0]["secret_key"] == swift_secret1
# TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
(err, out) = rgwadmin(
ctx,
client,
["subuser", "create", "--subuser", subuser2, "--secret", swift_secret2, "--key-type", "swift"],
check_status=True,
)
# TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
assert len(out["swift_keys"]) == 2
assert out["swift_keys"][0]["user"] == subuser2 or out["swift_keys"][1]["user"] == subuser2
assert out["swift_keys"][0]["secret_key"] == swift_secret2 or out["swift_keys"][1]["secret_key"] == swift_secret2
# TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
(err, out) = rgwadmin(ctx, client, ["key", "rm", "--subuser", subuser1, "--key-type", "swift"], check_status=True)
assert len(out["swift_keys"]) == 1
# TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
(err, out) = rgwadmin(ctx, client, ["subuser", "rm", "--subuser", subuser1], check_status=True)
assert len(out["subusers"]) == 1
# TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
(err, out) = rgwadmin(
ctx, client, ["subuser", "rm", "--subuser", subuser2, "--key-type", "swift", "--purge-keys"], check_status=True
)
assert len(out["swift_keys"]) == 0
assert len(out["subusers"]) == 0
# TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--uid", user1], check_status=True)
assert len(out) == 0
if multi_region_run:
rgw_utils.radosgw_agent_sync_all(ctx)
# TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, ["bucket", "list", "--uid", user1], check_status=True)
assert len(out) == 0
# create a first bucket
bucket = connection.create_bucket(bucket_name)
# TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
(err, out) = rgwadmin(ctx, client, ["bucket", "list", "--uid", user1], check_status=True)
assert len(out) == 1
assert out[0] == bucket_name
# TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
(err, out) = rgwadmin(ctx, client, ["bucket", "list"], check_status=True)
assert len(out) >= 1
assert bucket_name in out
# TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
bucket2 = connection.create_bucket(bucket_name + "2")
bucket3 = connection.create_bucket(bucket_name + "3")
bucket4 = connection.create_bucket(bucket_name + "4")
# the 5th should fail.
failed = False
try:
connection.create_bucket(bucket_name + "5")
except Exception:
failed = True
assert failed
# delete the buckets
bucket2.delete()
bucket3.delete()
bucket4.delete()
# TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True)
assert out["owner"] == user1
bucket_id = out["id"]
# TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
(err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--uid", user1], check_status=True)
assert len(out) == 1
assert out[0]["id"] == bucket_id # does it return the same ID twice in a row?
# use some space
key = boto.s3.key.Key(bucket)
key.set_contents_from_string("one")
# TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
(err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True)
assert out["id"] == bucket_id
assert out["usage"]["rgw.main"]["num_objects"] == 1
assert out["usage"]["rgw.main"]["size_kb"] > 0
# reclaim it
key.delete()
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
(err, out) = rgwadmin(ctx, client, ["bucket", "unlink", "--uid", user1, "--bucket", bucket_name], check_status=True)
# create a second user to link the bucket to
(err, out) = rgwadmin(
ctx,
client,
[
"user",
"create",
"--uid",
user2,
"--display-name",
display_name2,
"--access-key",
access_key2,
"--secret",
secret_key2,
"--max-buckets",
"1",
],
check_status=True,
)
# try creating an object with the first user before the bucket is relinked
denied = False
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string("two")
except boto.exception.S3ResponseError:
denied = True
assert not denied
# delete the object
key.delete()
# link the bucket to another user
(err, out) = rgwadmin(ctx, client, ["metadata", "get", "bucket:{n}".format(n=bucket_name)], check_status=True)
bucket_data = out["data"]
assert bucket_data["bucket"]["name"] == bucket_name
bucket_id = bucket_data["bucket"]["bucket_id"]
# link the bucket to another user
(err, out) = rgwadmin(
ctx,
client,
["bucket", "link", "--uid", user2, "--bucket", bucket_name, "--bucket-id", bucket_id],
check_status=True,
)
# try to remove user, should fail (has a linked bucket)
(err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user2])
assert err
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
(err, out) = rgwadmin(ctx, client, ["bucket", "unlink", "--uid", user2, "--bucket", bucket_name], check_status=True)
# relink the bucket to the first user and delete the second user
(err, out) = rgwadmin(
ctx,
client,
["bucket", "link", "--uid", user1, "--bucket", bucket_name, "--bucket-id", bucket_id],
check_status=True,
)
(err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user2], check_status=True)
# TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
# upload an object
object_name = "four"
key = boto.s3.key.Key(bucket, object_name)
key.set_contents_from_string(object_name)
# now delete it
(err, out) = rgwadmin(
ctx, client, ["object", "rm", "--bucket", bucket_name, "--object", object_name], check_status=True
)
# TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True)
assert out["id"] == bucket_id
assert out["usage"]["rgw.main"]["num_objects"] == 0
# list log objects
# TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx, client, ["log", "list"], check_status=True)
assert len(out) > 0
for obj in out:
# TESTCASE 'log-show','log','show','after activity','returns expected info'
if obj[:4] == "meta" or obj[:4] == "data":
continue
(err, rgwlog) = rgwadmin(ctx, client, ["log", "show", "--object", obj], check_status=True)
assert len(rgwlog) > 0
# exempt bucket_name2 from checking as it was only used for multi-region tests
assert rgwlog["bucket"].find(bucket_name) == 0 or rgwlog["bucket"].find(bucket_name2) == 0
assert rgwlog["bucket"] != bucket_name or rgwlog["bucket_id"] == bucket_id
assert (
rgwlog["bucket_owner"] == user1 or rgwlog["bucket"] == bucket_name + "5" or rgwlog["bucket"] == bucket_name2
)
for entry in rgwlog["log_entries"]:
log.debug("checking log entry: ", entry)
assert entry["bucket"] == rgwlog["bucket"]
possible_buckets = [bucket_name + "5", bucket_name2]
user = entry["user"]
assert user == user1 or user.endswith("system-user") or rgwlog["bucket"] in possible_buckets
# TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
(err, out) = rgwadmin(ctx, client, ["log", "rm", "--object", obj], check_status=True)
# TODO: show log by bucket+date
# need to wait for all usage data to get flushed, should take up to 30 seconds
timestamp = time.time()
while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes
(err, out) = rgwadmin(
ctx, client, ["usage", "show", "--categories", "delete_obj"]
) # last operation we did is delete obj, wait for it to flush
if successful_ops(out) > 0:
break
time.sleep(1)
assert time.time() - timestamp <= (20 * 60)
# TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
(err, out) = rgwadmin(ctx, client, ["usage", "show"], check_status=True)
assert len(out["entries"]) > 0
assert len(out["summary"]) > 0
user_summary = out["summary"][0]
total = user_summary["total"]
assert total["successful_ops"] > 0
# TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
(err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1], check_status=True)
assert len(out["entries"]) > 0
assert len(out["summary"]) > 0
user_summary = out["summary"][0]
for entry in user_summary["categories"]:
assert entry["successful_ops"] > 0
assert user_summary["user"] == user1
# TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
test_categories = ["create_bucket", "put_obj", "delete_obj", "delete_bucket"]
for cat in test_categories:
(err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1, "--categories", cat], check_status=True)
assert len(out["summary"]) > 0
user_summary = out["summary"][0]
assert user_summary["user"] == user1
assert len(user_summary["categories"]) == 1
entry = user_summary["categories"][0]
assert entry["category"] == cat
assert entry["successful_ops"] > 0
# the usage flush interval is 30 seconds, wait that much an then some
# to make sure everything has been flushed
time.sleep(35)
# TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
(err, out) = rgwadmin(ctx, client, ["usage", "trim", "--uid", user1], check_status=True)
(err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1], check_status=True)
assert len(out["entries"]) == 0
assert len(out["summary"]) == 0
# TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
(err, out) = rgwadmin(ctx, client, ["user", "suspend", "--uid", user1], check_status=True)
# TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
try:
key = boto.s3.key.Key(bucket)
key.set_contents_from_string("five")
except boto.exception.S3ResponseError as e:
assert e.status == 403
# TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx, client, ["user", "enable", "--uid", user1], check_status=True)
# TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
key = boto.s3.key.Key(bucket)
key.set_contents_from_string("six")
# TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
# create an object large enough to be split into multiple parts
test_string = "foo" * 10000000
big_key = boto.s3.key.Key(bucket)
big_key.set_contents_from_string(test_string)
# now delete the head
big_key.delete()
# wait a bit to give the garbage collector time to cycle
time.sleep(15)
(err, out) = rgwadmin(ctx, client, ["gc", "list"])
assert len(out) > 0
# TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
(err, out) = rgwadmin(ctx, client, ["gc", "process"], check_status=True)
# confirm
(err, out) = rgwadmin(ctx, client, ["gc", "list"])
assert len(out) == 0
# TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
(err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1])
assert err
# delete should fail because ``key`` still exists
try:
bucket.delete()
except boto.exception.S3ResponseError as e:
assert e.status == 409
key.delete()
bucket.delete()
# TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
bucket = connection.create_bucket(bucket_name)
# create an object
key = boto.s3.key.Key(bucket)
key.set_contents_from_string("seven")
# should be private already but guarantee it
key.set_acl("private")
(err, out) = rgwadmin(ctx, client, ["policy", "--bucket", bucket.name, "--object", key.key], check_status=True)
acl = key.get_xml_acl()
assert acl == out.strip("\n")
# add another grantee by making the object public read
key.set_acl("public-read")
(err, out) = rgwadmin(ctx, client, ["policy", "--bucket", bucket.name, "--object", key.key], check_status=True)
acl = key.get_xml_acl()
assert acl == out.strip("\n")
# TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key_name = ["eight", "nine", "ten", "eleven"]
for i in range(4):
key = boto.s3.key.Key(bucket)
key.set_contents_from_string(key_name[i])
(err, out) = rgwadmin(ctx, client, ["bucket", "rm", "--bucket", bucket_name, "--purge-objects"], check_status=True)
# TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
caps = "user=read"
(err, out) = rgwadmin(ctx, client, ["caps", "add", "--uid", user1, "--caps", caps])
assert out["caps"][0]["perm"] == "read"
# TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
(err, out) = rgwadmin(ctx, client, ["caps", "rm", "--uid", user1, "--caps", caps])
assert not out["caps"]
# TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
(err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1])
assert err
# TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
key.set_contents_from_string("twelve")
(err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1, "--purge-data"], check_status=True)
# TESTCASE 'rm-user3','user','rm','deleted user','fails'
(err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1])
assert err
# TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
#
(err, out) = rgwadmin(ctx, client, ["zone", "get"])
orig_placement_pools = len(out["placement_pools"])
# removed this test, it is not correct to assume that zone has default placement, it really
# depends on how we set it up before
#
# assert len(out) > 0
# assert len(out['placement_pools']) == 1
# default_rule = out['placement_pools'][0]
# assert default_rule['key'] == 'default-placement'
rule = {"key": "new-placement", "val": {"data_pool": ".rgw.buckets.2", "index_pool": ".rgw.buckets.index.2"}}
out["placement_pools"].append(rule)
(err, out) = rgwadmin(ctx, client, ["zone", "set"], stdin=StringIO(json.dumps(out)), check_status=True)
(err, out) = rgwadmin(ctx, client, ["zone", "get"])
assert len(out) > 0
assert len(out["placement_pools"]) == orig_placement_pools + 1
def configure_regions_and_zones(ctx, config, regions, role_endpoints, realm):
"""
Configure regions and zones from rados and rgw.
"""
if not regions:
log.debug(
'In rgw.configure_regions_and_zones() and regions is None. '
'Bailing')
configure_compression_in_default_zone(ctx, config)
yield
return
if not realm:
log.debug(
'In rgw.configure_regions_and_zones() and realm is None. '
'Bailing')
configure_compression_in_default_zone(ctx, config)
yield
return
log.info('Configuring regions and zones...')
log.debug('config is %r', config)
log.debug('regions are %r', regions)
log.debug('role_endpoints = %r', role_endpoints)
log.debug('realm is %r', realm)
# extract the zone info
role_zones = dict([(client, extract_zone_info(ctx, client, c_config))
for client, c_config in config.iteritems()])
log.debug('roles_zones = %r', role_zones)
# extract the user info and append it to the payload tuple for the given
# client
for client, c_config in config.iteritems():
if not c_config:
user_info = None
else:
user_info = extract_user_info(c_config)
(region, zone, zone_info) = role_zones[client]
role_zones[client] = (region, zone, zone_info, user_info)
region_info = dict([
(region_name, extract_region_info(region_name, r_config))
for region_name, r_config in regions.iteritems()])
fill_in_endpoints(region_info, role_zones, role_endpoints)
# clear out the old defaults
cluster_name, daemon_type, client_id = teuthology.split_role(client)
first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
(mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
# removing these objects from .rgw.root and the per-zone root pools
# may or may not matter
rados(ctx, mon,
cmd=['-p', '.rgw.root', 'rm', 'region_info.default', '--cluster', cluster_name])
rados(ctx, mon,
cmd=['-p', '.rgw.root', 'rm', 'zone_info.default', '--cluster', cluster_name])
# read master zonegroup and master_zone
for zonegroup, zg_info in region_info.iteritems():
if zg_info['is_master']:
master_zonegroup = zonegroup
master_zone = zg_info['master_zone']
break
for client in config.iterkeys():
(zonegroup, zone, zone_info, user_info) = role_zones[client]
if zonegroup == master_zonegroup and zone == master_zone:
master_client = client
break
log.debug('master zonegroup =%r', master_zonegroup)
log.debug('master zone = %r', master_zone)
log.debug('master client = %r', master_client)
log.debug('config %r ', config)
(ret, out)=rgwadmin(ctx, client,
cmd=['realm', 'create', '--rgw-realm', realm, '--default'])
log.debug('realm create ret %r exists %r', -ret, errno.EEXIST)
assert ret == 0 or ret != -errno.EEXIST
if ret is -errno.EEXIST:
log.debug('realm %r exists', realm)
for client in config.iterkeys():
for role, (zonegroup, zone, zone_info, user_info) in role_zones.iteritems():
rados(ctx, mon,
cmd=['-p', zone_info['domain_root'],
'rm', 'region_info.default', '--cluster', cluster_name])
rados(ctx, mon,
cmd=['-p', zone_info['domain_root'],
'rm', 'zone_info.default', '--cluster', cluster_name])
(remote,) = ctx.cluster.only(role).remotes.keys()
for pool_info in zone_info['placement_pools']:
remote.run(args=['sudo', 'ceph', 'osd', 'pool', 'create',
pool_info['val']['index_pool'], '64', '64', '--cluster', cluster_name])
if ctx.rgw.ec_data_pool:
create_ec_pool(remote, pool_info['val']['data_pool'],
zone, 64, ctx.rgw.erasure_code_profile, cluster_name)
else:
create_replicated_pool(
remote, pool_info['val']['data_pool'],
64, cluster_name)
zone_json = json.dumps(dict(zone_info.items() + user_info.items()))
log.debug('zone info is: %r', zone_json)
rgwadmin(ctx, client,
cmd=['zone', 'set', '--rgw-zonegroup', zonegroup,
'--rgw-zone', zone],
stdin=StringIO(zone_json),
check_status=True)
for region, info in region_info.iteritems():
region_json = json.dumps(info)
log.debug('region info is: %s', region_json)
rgwadmin(ctx, client,
cmd=['zonegroup', 'set'],
stdin=StringIO(region_json),
check_status=True)
if info['is_master']:
rgwadmin(ctx, client,
cmd=['zonegroup', 'default', '--rgw-zonegroup', master_zonegroup],
check_status=True)
(zonegroup, zone, zone_info, user_info) = role_zones[client]
rgwadmin(ctx, client,
cmd=['zone', 'default', '--rgw-zone', zone],
check_status=True)
#this used to take master_client, need to edit that accordingly
rgwadmin(ctx, client,
cmd=['period', 'update', '--commit'],
check_status=True)
yield
def task(ctx, config):
"""
Test radosgw-admin functionality through the RESTful interface
"""
assert config is None or isinstance(config, list) \
or isinstance(config, dict), \
"task s3tests only supports a list or dictionary for configuration"
all_clients = [
'client.{id}'.format(id=id_)
for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')
]
if config is None:
config = all_clients
if isinstance(config, list):
config = dict.fromkeys(config)
clients = config.keys()
# just use the first client...
client = clients[0]
##
admin_user = '******'
admin_display_name = 'Ms. Admin User'
admin_access_key = 'MH1WC2XQ1S8UISFDZC8W'
admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write'
user1 = 'foo'
user2 = 'fud'
subuser1 = 'foo:foo1'
subuser2 = 'foo:foo2'
display_name1 = 'Foo'
display_name2 = 'Fud'
email = '*****@*****.**'
access_key = '9te6NH5mcdcq0Tc5i8i1'
secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
access_key2 = 'p5YnriCv1nAtykxBrupQ'
secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
bucket_name = 'myfoo'
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'create-admin-user','user','create','administrative user','succeeds'
(err, out) = rgwadmin(ctx, client, [
'user', 'create', '--uid', admin_user, '--display-name',
admin_display_name, '--access-key', admin_access_key, '--secret',
admin_secret_key, '--max-buckets', '0', '--caps', admin_caps
])
logging.error(out)
logging.error(err)
assert not err
assert hasattr(ctx,
'rgw'), 'radosgw-admin-rest must run after the rgw task'
endpoint = ctx.rgw.role_endpoints.get(client)
assert endpoint, 'no rgw endpoint for {}'.format(client)
admin_conn = boto.s3.connection.S3Connection(
aws_access_key_id=admin_access_key,
aws_secret_access_key=admin_secret_key,
is_secure=True if endpoint.cert else False,
port=endpoint.port,
host=endpoint.hostname,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1})
assert ret == 404
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(ret, out) = rgwadmin_rest(
admin_conn, ['user', 'create'], {
'uid': user1,
'display-name': display_name1,
'email': email,
'access-key': access_key,
'secret-key': secret_key,
'max-buckets': '4'
})
assert ret == 200
# TESTCASE 'list-no-user','user','list','list user keys','user list object'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {
'list': '',
'max-entries': 0
})
assert ret == 200
assert out['count'] == 0
assert out['truncated'] == True
assert len(out['keys']) == 0
assert len(out['marker']) > 0
# TESTCASE 'list-user-without-marker','user','list','list user keys','user list object'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {
'list': '',
'max-entries': 1
})
assert ret == 200
assert out['count'] == 1
assert out['truncated'] == True
assert len(out['keys']) == 1
assert len(out['marker']) > 0
marker = out['marker']
# TESTCASE 'list-user-with-marker','user','list','list user keys','user list object'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {
'list': '',
'max-entries': 1,
'marker': marker
})
assert ret == 200
assert out['count'] == 1
assert out['truncated'] == False
assert len(out['keys']) == 1
# TESTCASE 'info-existing','user','info','existing user','returns correct info'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
assert out['tenant'] == ''
assert out['max_buckets'] == 4
assert out['caps'] == []
assert out['op_mask'] == 'read, write, delete'
assert out['default_placement'] == ''
assert out['default_storage_class'] == ''
assert out['placement_tags'] == []
assert not out['bucket_quota']['enabled']
assert not out['bucket_quota']['check_on_raw']
assert out['bucket_quota']['max_size'] == -1
assert out['bucket_quota']['max_size_kb'] == 0
assert out['bucket_quota']['max_objects'] == -1
assert not out['user_quota']['enabled']
assert not out['user_quota']['check_on_raw']
assert out['user_quota']['max_size'] == -1
assert out['user_quota']['max_size_kb'] == 0
assert out['user_quota']['max_objects'] == -1
assert out['temp_url_keys'] == []
assert out['type'] == 'rgw'
assert out['mfa_ids'] == []
# TESTCASE 'info-existing','user','info','existing user query with wrong uid but correct access key','returns correct info'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {
'access-key': access_key,
'uid': 'uid_not_exist'
})
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
assert out['tenant'] == ''
assert out['max_buckets'] == 4
assert out['caps'] == []
assert out['op_mask'] == "read, write, delete"
assert out['default_placement'] == ''
assert out['default_storage_class'] == ''
assert out['placement_tags'] == []
assert not out['bucket_quota']['enabled']
assert not out['bucket_quota']['check_on_raw']
assert out['bucket_quota']['max_size'] == -1
assert out['bucket_quota']['max_size_kb'] == 0
assert out['bucket_quota']['max_objects'] == -1
assert not out['user_quota']['enabled']
assert not out['user_quota']['check_on_raw']
assert out['user_quota']['max_size'] == -1
assert out['user_quota']['max_size_kb'] == 0
assert out['user_quota']['max_objects'] == -1
assert out['temp_url_keys'] == []
assert out['type'] == 'rgw'
assert out['mfa_ids'] == []
# TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {
'uid': user1,
'suspended': True
})
assert ret == 200
# TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert ret == 200
assert out['suspended']
assert out['email'] == email
# TESTCASE 're-enable','user','enable','suspended user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {
'uid': user1,
'suspended': 'false'
})
assert not err
# TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert ret == 200
assert not out['suspended']
# TESTCASE 'add-keys','key','create','w/valid info','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['key', 'create'], {
'uid': user1,
'access-key': access_key2,
'secret-key': secret_key2
})
assert ret == 200
# TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert ret == 200
assert len(out['keys']) == 2
assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][
'access_key'] == access_key2
assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][
'secret_key'] == secret_key2
# TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
(ret, out) = rgwadmin_rest(admin_conn, ['key', 'rm'], {
'uid': user1,
'access-key': access_key2
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'create'], {
'subuser': subuser1,
'secret-key': swift_secret1,
'key-type': 'swift'
})
assert ret == 200
# TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert ret == 200
assert len(out['swift_keys']) == 1
assert out['swift_keys'][0]['user'] == subuser1
assert out['swift_keys'][0]['secret_key'] == swift_secret1
# TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'create'], {
'subuser': subuser2,
'secret-key': swift_secret2,
'key-type': 'swift'
})
assert ret == 200
# TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert ret == 200
assert len(out['swift_keys']) == 2
assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][
'user'] == subuser2
assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[
'swift_keys'][1]['secret_key'] == swift_secret2
# TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
(ret, out) = rgwadmin_rest(admin_conn, ['key', 'rm'], {
'subuser': subuser1,
'key-type': 'swift'
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert len(out['swift_keys']) == 1
# TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
(ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'rm'],
{'subuser': subuser1})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert len(out['subusers']) == 1
# TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
(ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'rm'], {
'subuser': subuser2,
'key-type': 'swift',
'{purge-keys': True
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert len(out['swift_keys']) == 0
assert len(out['subusers']) == 0
# TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid': user1})
assert ret == 200
assert len(out) == 0
# connect to rgw
connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=True if endpoint.cert else False,
port=endpoint.port,
host=endpoint.hostname,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
'uid': user1,
'stats': True
})
assert ret == 200
assert len(out) == 0
# create a first bucket
bucket = connection.create_bucket(bucket_name)
# TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid': user1})
assert ret == 200
assert len(out) == 1
assert out[0] == bucket_name
# TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
'bucket': bucket_name,
'stats': True
})
assert ret == 200
assert out['owner'] == user1
assert out['tenant'] == ''
bucket_id = out['id']
# TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
'uid': user1,
'stats': True
})
assert ret == 200
assert len(out) == 1
assert out[0][
'id'] == bucket_id # does it return the same ID twice in a row?
# use some space
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('one')
# TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
'bucket': bucket_name,
'stats': True
})
assert ret == 200
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 1
assert out['usage']['rgw.main']['size_kb'] > 0
# reclaim it
key.delete()
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], {
'uid': user1,
'bucket': bucket_name
})
assert ret == 200
# create a second user to link the bucket to
(ret, out) = rgwadmin_rest(
admin_conn, ['user', 'create'], {
'uid': user2,
'display-name': display_name2,
'access-key': access_key2,
'secret-key': secret_key2,
'max-buckets': '1',
})
assert ret == 200
# try creating an object with the first user before the bucket is relinked
denied = False
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('two')
except boto.exception.S3ResponseError:
denied = True
assert not denied
# delete the object
key.delete()
# link the bucket to another user
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {
'uid': user2,
'bucket': bucket_name,
'bucket-id': bucket_id,
})
assert ret == 200
# try creating an object with the first user which should cause an error
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('three')
except boto.exception.S3ResponseError:
denied = True
assert denied
# relink the bucket to the first user and delete the second user
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {
'uid': user1,
'bucket': bucket_name,
'bucket-id': bucket_id,
})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user2})
assert ret == 200
# TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
# upload an object
object_name = 'four'
key = boto.s3.key.Key(bucket, object_name)
key.set_contents_from_string(object_name)
# now delete it
(ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {
'bucket': bucket_name,
'object': object_name
})
assert ret == 200
# TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
'bucket': bucket_name,
'stats': True
})
assert ret == 200
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 0
# create a bucket for deletion stats
useless_bucket = connection.create_bucket('useless_bucket')
useless_key = useless_bucket.new_key('useless_key')
useless_key.set_contents_from_string('useless string')
# delete it
useless_key.delete()
useless_bucket.delete()
# wait for the statistics to flush
time.sleep(60)
# need to wait for all usage data to get flushed, should take up to 30 seconds
timestamp = time.time()
while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes
(ret, out) = rgwadmin_rest(
admin_conn, ['usage', 'show'],
{'categories': 'delete_obj'
}) # last operation we did is delete obj, wait for it to flush
if get_user_successful_ops(out, user1) > 0:
break
time.sleep(1)
assert time.time() - timestamp <= (20 * 60)
# TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'])
assert ret == 200
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = get_user_summary(out, user1)
total = user_summary['total']
assert total['successful_ops'] > 0
# TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid': user1})
assert ret == 200
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = out['summary'][0]
for entry in user_summary['categories']:
assert entry['successful_ops'] > 0
assert user_summary['user'] == user1
# TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
test_categories = [
'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'
]
for cat in test_categories:
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {
'uid': user1,
'categories': cat
})
assert ret == 200
assert len(out['summary']) > 0
user_summary = out['summary'][0]
assert user_summary['user'] == user1
assert len(user_summary['categories']) == 1
entry = user_summary['categories'][0]
assert entry['category'] == cat
assert entry['successful_ops'] > 0
# TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid': user1})
assert ret == 200
(ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid': user1})
assert ret == 200
assert len(out['entries']) == 0
assert len(out['summary']) == 0
# TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {
'uid': user1,
'suspended': True
})
assert ret == 200
# TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
try:
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('five')
except boto.exception.S3ResponseError as e:
assert e.status == 403
# TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {
'uid': user1,
'suspended': 'false'
})
assert ret == 200
# TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('six')
# TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection'
# create an object large enough to be split into multiple parts
test_string = 'foo' * 10000000
big_key = boto.s3.key.Key(bucket)
big_key.set_contents_from_string(test_string)
# now delete the head
big_key.delete()
# TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user1})
assert ret == 409
# delete should fail because ``key`` still exists
try:
bucket.delete()
except boto.exception.S3ResponseError as e:
assert e.status == 409
key.delete()
bucket.delete()
# TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
bucket = connection.create_bucket(bucket_name)
# create an object
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('seven')
# should be private already but guarantee it
key.set_acl('private')
(ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {
'bucket': bucket.name,
'object': key.key
})
assert ret == 200
assert len(out['acl']['grant_map']) == 1
# add another grantee by making the object public read
key.set_acl('public-read')
(ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {
'bucket': bucket.name,
'object': key.key
})
assert ret == 200
assert len(out['acl']['grant_map']) == 2
# TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key_name = ['eight', 'nine', 'ten', 'eleven']
for i in range(4):
key = boto.s3.key.Key(bucket)
key.set_contents_from_string(key_name[i])
(ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], {
'bucket': bucket_name,
'purge-objects': True
})
assert ret == 200
# TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
caps = 'usage=read'
(ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {
'uid': user1,
'user-caps': caps
})
assert ret == 200
assert out[0]['perm'] == 'read'
# TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
(ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {
'uid': user1,
'user-caps': caps
})
assert ret == 200
assert not out
# TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user1})
assert ret == 409
# TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('twelve')
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {
'uid': user1,
'purge-data': True
})
assert ret == 200
# TESTCASE 'rm-user3','user','info','deleted user','fails'
(ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
assert ret == 404
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
connection2 = boto.s3.connection.S3Connection(
aws_access_key_id=access_key2,
aws_secret_access_key=secret_key2,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user1,
'--display-name', display_name1,
'--email', email,
'--access-key', access_key,
'--secret', secret_key,
'--max-buckets', '4'
],
check_status=True)
# TESTCASE 'duplicate email','user','create','existing user email','fails'
def task(ctx, config):
"""
Test radosgw-admin functionality against a running rgw instance.
"""
global log
assert config is None or isinstance(config, list) \
or isinstance(config, dict), \
"task s3tests only supports a list or dictionary for configuration"
all_clients = ['client.{id}'.format(id=id_)
for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
if config is None:
config = all_clients
if isinstance(config, list):
config = dict.fromkeys(config)
clients = config.keys()
multi_region_run = rgw_utils.multi_region_enabled(ctx)
client = clients[0]; # default choice, multi-region code may overwrite this
if multi_region_run:
client = rgw_utils.get_master_client(ctx, clients)
# once the client is chosen, pull the host name and assigned port out of
# the role_endpoints that were assigned by the rgw task
(remote_host, remote_port) = ctx.rgw.role_endpoints[client]
realm = ctx.rgw.realm
log.debug('radosgw-admin: realm %r', realm)
##
user1='foo'
user2='fud'
subuser1='foo:foo1'
subuser2='foo:foo2'
display_name1='Foo'
display_name2='Fud'
email='*****@*****.**'
email2='*****@*****.**'
access_key='9te6NH5mcdcq0Tc5i8i1'
secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
access_key2='p5YnriCv1nAtykxBrupQ'
secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
bucket_name='myfoo'
bucket_name2='mybar'
# connect to rgw
connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
connection2 = boto.s3.connection.S3Connection(
aws_access_key_id=access_key2,
aws_secret_access_key=secret_key2,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user1,
'--display-name', display_name1,
'--email', email,
'--access-key', access_key,
'--secret', secret_key,
'--max-buckets', '4'
],
check_status=True)
# TESTCASE 'duplicate email','user','create','existing user email','fails'
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user2,
'--display-name', display_name2,
'--email', email,
])
assert err
# TESTCASE 'info-existing','user','info','existing user','returns correct info'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# this whole block should only be run if regions have been configured
if multi_region_run:
rgw_utils.radosgw_agent_sync_all(ctx)
# post-sync, validate that user1 exists on the sync destination host
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
dest_client = c_config['dest']
(err, out) = rgwadmin(ctx, dest_client, ['metadata', 'list', 'user'])
(err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True)
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# compare the metadata between different regions, make sure it matches
log.debug('compare the metadata between different regions, make sure it matches')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(ctx, source_client,
['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True)
(err2, out2) = rgwadmin(ctx, dest_client,
['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True)
assert out1 == out2
# suspend a user on the master, then check the status on the destination
log.debug('suspend a user on the master, then check the status on the destination')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err, out) = rgwadmin(ctx, source_client, ['user', 'suspend', '--uid', user1])
rgw_utils.radosgw_agent_sync_all(ctx)
(err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True)
assert out['suspended']
# delete a user on the master, then check that it's gone on the destination
log.debug('delete a user on the master, then check that it\'s gone on the destination')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err, out) = rgwadmin(ctx, source_client, ['user', 'rm', '--uid', user1], check_status=True)
rgw_utils.radosgw_agent_sync_all(ctx)
(err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user1])
assert out is None
(err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1])
assert out is None
# then recreate it so later tests pass
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user1,
'--display-name', display_name1,
'--email', email,
'--access-key', access_key,
'--secret', secret_key,
'--max-buckets', '4'
],
check_status=True)
# now do the multi-region bucket tests
log.debug('now do the multi-region bucket tests')
# Create a second user for the following tests
log.debug('Create a second user for the following tests')
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user2,
'--display-name', display_name2,
'--email', email2,
'--access-key', access_key2,
'--secret', secret_key2,
'--max-buckets', '4'
],
check_status=True)
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user2], check_status=True)
assert out is not None
# create a bucket and do a sync
log.debug('create a bucket and do a sync')
bucket = connection.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug('compare the metadata for the bucket between different regions, make sure it matches')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(ctx, source_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
(err2, out2) = rgwadmin(ctx, dest_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
assert out1 == out2
# get the bucket.instance info and compare that
src_bucket_id = out1['data']['bucket']['bucket_id']
dest_bucket_id = out2['data']['bucket']['bucket_id']
(err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get',
'bucket.instance:{bucket_name}:{bucket_instance}'.format(
bucket_name=bucket_name2,bucket_instance=src_bucket_id)],
check_status=True)
(err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get',
'bucket.instance:{bucket_name}:{bucket_instance}'.format(
bucket_name=bucket_name2,bucket_instance=dest_bucket_id)],
check_status=True)
del out1['data']['bucket_info']['bucket']['pool']
del out1['data']['bucket_info']['bucket']['index_pool']
del out1['data']['bucket_info']['bucket']['data_extra_pool']
del out2['data']['bucket_info']['bucket']['pool']
del out2['data']['bucket_info']['bucket']['index_pool']
del out2['data']['bucket_info']['bucket']['data_extra_pool']
assert out1 == out2
same_region = 0
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
source_region = rgw_utils.region_for_client(ctx, source_client)
dest_region = rgw_utils.region_for_client(ctx, dest_client)
# 301 is only returned for requests to something in a different region
if source_region == dest_region:
log.debug('301 is only returned for requests to something in a different region')
same_region += 1
continue
# Attempt to create a new connection with user1 to the destination RGW
log.debug('Attempt to create a new connection with user1 to the destination RGW')
# and use that to attempt a delete (that should fail)
(dest_remote_host, dest_remote_port) = ctx.rgw.role_endpoints[dest_client]
connection_dest = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_remote_port,
host=dest_remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# this should fail
r, content = send_raw_http_request(connection_dest, 'DELETE', bucket_name2, '', follow_redirects = False)
assert r.status == 301
# now delete the bucket on the source RGW and do another sync
log.debug('now delete the bucket on the source RGW and do another sync')
bucket.delete()
rgw_utils.radosgw_agent_sync_all(ctx)
if same_region == len(ctx.radosgw_agent.config):
bucket.delete()
rgw_utils.radosgw_agent_sync_all(ctx)
# make sure that the bucket no longer exists in either region
log.debug('make sure that the bucket no longer exists in either region')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)])
(err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get',
'bucket:{bucket_name}'.format(bucket_name=bucket_name2)])
# Both of the previous calls should have errors due to requesting
# metadata for non-existent buckets
assert err1
assert err2
# create a bucket and then sync it
log.debug('create a bucket and then sync it')
bucket = connection.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug('compare the metadata for the bucket between different regions, make sure it matches')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(ctx, source_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
(err2, out2) = rgwadmin(ctx, dest_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
assert out1 == out2
# Now delete the bucket and recreate it with a different user
log.debug('Now delete the bucket and recreate it with a different user')
# within the same window of time and then sync.
bucket.delete()
bucket = connection2.create_bucket(bucket_name2)
rgw_utils.radosgw_agent_sync_all(ctx)
# compare the metadata for the bucket between different regions, make sure it matches
log.debug('compare the metadata for the bucket between different regions, make sure it matches')
# user2 should own the bucket in both regions
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err1, out1) = rgwadmin(ctx, source_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
(err2, out2) = rgwadmin(ctx, dest_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
assert out1 == out2
assert out1['data']['owner'] == user2
assert out1['data']['owner'] != user1
# now we're going to use this bucket to test meta-data update propagation
log.debug('now we\'re going to use this bucket to test meta-data update propagation')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
# get the metadata so we can tweak it
log.debug('get the metadata so we can tweak it')
(err, orig_data) = rgwadmin(ctx, source_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
# manually edit mtime for this bucket to be 300 seconds in the past
log.debug('manually edit mtime for this bucket to be 300 seconds in the past')
new_data = copy.deepcopy(orig_data)
mtime = datetime.datetime.strptime(orig_data['mtime'], "%Y-%m-%d %H:%M:%S.%fZ") - datetime.timedelta(300)
new_data['mtime'] = unicode(mtime.strftime("%Y-%m-%d %H:%M:%S.%fZ"))
log.debug("new mtime ", mtime)
assert new_data != orig_data
(err, out) = rgwadmin(ctx, source_client,
['metadata', 'put', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
stdin=StringIO(json.dumps(new_data)),
check_status=True)
# get the metadata and make sure that the 'put' worked
log.debug('get the metadata and make sure that the \'put\' worked')
(err, out) = rgwadmin(ctx, source_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
assert out == new_data
# sync to propagate the new metadata
log.debug('sync to propagate the new metadata')
rgw_utils.radosgw_agent_sync_all(ctx)
# get the metadata from the dest and compare it to what we just set
log.debug('get the metadata from the dest and compare it to what we just set')
# and what the source region has.
(err1, out1) = rgwadmin(ctx, source_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
(err2, out2) = rgwadmin(ctx, dest_client,
['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
check_status=True)
# yeah for the transitive property
assert out1 == out2
assert out1 == new_data
# now we delete the bucket
log.debug('now we delete the bucket')
bucket.delete()
log.debug('sync to propagate the deleted bucket')
rgw_utils.radosgw_agent_sync_all(ctx)
# Delete user2 as later tests do not expect it to exist.
# Verify that it is gone on both regions
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
source_client = c_config['src']
dest_client = c_config['dest']
(err, out) = rgwadmin(ctx, source_client,
['user', 'rm', '--uid', user2], check_status=True)
rgw_utils.radosgw_agent_sync_all(ctx)
# The two 'user info' calls should fail and not return any data
# since we just deleted this user.
(err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user2])
assert out is None
(err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user2])
assert out is None
# Test data sync
# First create a bucket for data sync test purpose
bucket = connection.create_bucket(bucket_name + 'data')
# Create a tiny file and check if in sync
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
if c_config.get('metadata-only'):
continue
for full in (True, False):
source_client = c_config['src']
dest_client = c_config['dest']
k = boto.s3.key.Key(bucket)
k.key = 'tiny_file'
k.set_contents_from_string("123456789")
safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client)
time.sleep(safety_window)
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
(dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
dest_connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_port,
host=dest_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
dest_k = dest_connection.get_bucket(bucket_name + 'data').get_key('tiny_file')
assert k.get_contents_as_string() == dest_k.get_contents_as_string()
# check that deleting it removes it from the dest zone
k.delete()
time.sleep(safety_window)
# full sync doesn't handle deleted objects yet
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)
dest_bucket = dest_connection.get_bucket(bucket_name + 'data')
dest_k = dest_bucket.get_key('tiny_file')
assert dest_k == None, 'object not deleted from destination zone'
# finally we delete the bucket
bucket.delete()
bucket = connection.create_bucket(bucket_name + 'data2')
for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
if c_config.get('metadata-only'):
continue
for full in (True, False):
source_client = c_config['src']
dest_client = c_config['dest']
(dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
dest_connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=dest_port,
host=dest_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
for i in range(20):
k = boto.s3.key.Key(bucket)
k.key = 'tiny_file_' + str(i)
k.set_contents_from_string(str(i) * 100)
safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client)
time.sleep(safety_window)
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
for i in range(20):
dest_k = dest_connection.get_bucket(bucket_name + 'data2').get_key('tiny_file_' + str(i))
assert (str(i) * 100) == dest_k.get_contents_as_string()
k = boto.s3.key.Key(bucket)
k.key = 'tiny_file_' + str(i)
k.delete()
# check that deleting removes the objects from the dest zone
time.sleep(safety_window)
# full sync doesn't delete deleted objects yet
rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)
for i in range(20):
dest_bucket = dest_connection.get_bucket(bucket_name + 'data2')
dest_k = dest_bucket.get_key('tiny_file_' + str(i))
assert dest_k == None, 'object %d not deleted from destination zone' % i
bucket.delete()
# end of 'if multi_region_run:'
# TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
check_status=True)
# TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert out['suspended']
# TESTCASE 're-enable','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True)
# TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert not out['suspended']
# TESTCASE 'add-keys','key','create','w/valid info','succeeds'
(err, out) = rgwadmin(ctx, client, [
'key', 'create', '--uid', user1,
'--access-key', access_key2, '--secret', secret_key2,
], check_status=True)
# TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1],
check_status=True)
assert len(out['keys']) == 2
assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2
# TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
(err, out) = rgwadmin(ctx, client, [
'key', 'rm', '--uid', user1,
'--access-key', access_key2,
], check_status=True)
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
subuser_access = 'full'
subuser_perm = 'full-control'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'create', '--subuser', subuser1,
'--access', subuser_access
], check_status=True)
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'modify', '--subuser', subuser1,
'--secret', swift_secret1,
'--key-type', 'swift',
], check_status=True)
# TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert out['subusers'][0]['permissions'] == subuser_perm
# TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert len(out['swift_keys']) == 1
assert out['swift_keys'][0]['user'] == subuser1
assert out['swift_keys'][0]['secret_key'] == swift_secret1
# TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'create', '--subuser', subuser2,
'--secret', swift_secret2,
'--key-type', 'swift',
], check_status=True)
# TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
assert len(out['swift_keys']) == 2
assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2
# TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
(err, out) = rgwadmin(ctx, client, [
'key', 'rm', '--subuser', subuser1,
'--key-type', 'swift',
], check_status=True)
assert len(out['swift_keys']) == 1
# TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'rm', '--subuser', subuser1,
], check_status=True)
assert len(out['subusers']) == 1
# TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
(err, out) = rgwadmin(ctx, client, [
'subuser', 'rm', '--subuser', subuser2,
'--key-type', 'swift', '--purge-keys',
], check_status=True)
assert len(out['swift_keys']) == 0
assert len(out['subusers']) == 0
# TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1],
check_status=True)
assert len(out) == 0
if multi_region_run:
rgw_utils.radosgw_agent_sync_all(ctx)
# TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
assert len(out) == 0
# create a first bucket
bucket = connection.create_bucket(bucket_name)
# TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
assert len(out) == 1
assert out[0] == bucket_name
# TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
assert len(out) >= 1
assert bucket_name in out;
# TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
bucket2 = connection.create_bucket(bucket_name + '2')
bucket3 = connection.create_bucket(bucket_name + '3')
bucket4 = connection.create_bucket(bucket_name + '4')
# the 5th should fail.
failed = False
try:
connection.create_bucket(bucket_name + '5')
except Exception:
failed = True
assert failed
# delete the buckets
bucket2.delete()
bucket3.delete()
bucket4.delete()
# TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
(err, out) = rgwadmin(ctx, client, [
'bucket', 'stats', '--bucket', bucket_name], check_status=True)
assert out['owner'] == user1
bucket_id = out['id']
# TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
(err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True)
assert len(out) == 1
assert out[0]['id'] == bucket_id # does it return the same ID twice in a row?
# use some space
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('one')
# TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
(err, out) = rgwadmin(ctx, client, [
'bucket', 'stats', '--bucket', bucket_name], check_status=True)
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 1
assert out['usage']['rgw.main']['size_kb'] > 0
# reclaim it
key.delete()
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
(err, out) = rgwadmin(ctx, client,
['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
check_status=True)
# create a second user to link the bucket to
(err, out) = rgwadmin(ctx, client, [
'user', 'create',
'--uid', user2,
'--display-name', display_name2,
'--access-key', access_key2,
'--secret', secret_key2,
'--max-buckets', '1',
],
check_status=True)
# try creating an object with the first user before the bucket is relinked
denied = False
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('two')
except boto.exception.S3ResponseError:
denied = True
assert not denied
# delete the object
key.delete()
# link the bucket to another user
(err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
check_status=True)
bucket_data = out['data']
assert bucket_data['bucket']['name'] == bucket_name
bucket_id = bucket_data['bucket']['bucket_id']
# link the bucket to another user
(err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id],
check_status=True)
# try to remove user, should fail (has a linked bucket)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
assert err
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
(err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
check_status=True)
# relink the bucket to the first user and delete the second user
(err, out) = rgwadmin(ctx, client,
['bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id],
check_status=True)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2],
check_status=True)
# TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
# upload an object
object_name = 'four'
key = boto.s3.key.Key(bucket, object_name)
key.set_contents_from_string(object_name)
# now delete it
(err, out) = rgwadmin(ctx, client,
['object', 'rm', '--bucket', bucket_name, '--object', object_name],
check_status=True)
# TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx, client, [
'bucket', 'stats', '--bucket', bucket_name],
check_status=True)
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 0
# list log objects
# TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
assert len(out) > 0
for obj in out:
# TESTCASE 'log-show','log','show','after activity','returns expected info'
if obj[:4] == 'meta' or obj[:4] == 'data' or obj[:18] == 'obj_delete_at_hint':
continue
(err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj],
check_status=True)
assert len(rgwlog) > 0
# exempt bucket_name2 from checking as it was only used for multi-region tests
assert rgwlog['bucket'].find(bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
assert rgwlog['bucket'] != bucket_name or rgwlog['bucket_id'] == bucket_id
assert rgwlog['bucket_owner'] == user1 or rgwlog['bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
for entry in rgwlog['log_entries']:
log.debug('checking log entry: ', entry)
assert entry['bucket'] == rgwlog['bucket']
possible_buckets = [bucket_name + '5', bucket_name2]
user = entry['user']
assert user == user1 or user.endswith('system-user') or \
rgwlog['bucket'] in possible_buckets
# TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
(err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj],
check_status=True)
# TODO: show log by bucket+date
# need to wait for all usage data to get flushed, should take up to 30 seconds
timestamp = time.time()
while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes
(err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj']) # last operation we did is delete obj, wait for it to flush
if get_user_successful_ops(out, user1) > 0:
break
time.sleep(1)
assert time.time() - timestamp <= (20 * 60)
# TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
(err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = get_user_summary(out, user1)
total = user_summary['total']
assert total['successful_ops'] > 0
# TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
(err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
check_status=True)
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = out['summary'][0]
for entry in user_summary['categories']:
assert entry['successful_ops'] > 0
assert user_summary['user'] == user1
# TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
for cat in test_categories:
(err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat],
check_status=True)
assert len(out['summary']) > 0
user_summary = out['summary'][0]
assert user_summary['user'] == user1
assert len(user_summary['categories']) == 1
entry = user_summary['categories'][0]
assert entry['category'] == cat
assert entry['successful_ops'] > 0
# the usage flush interval is 30 seconds, wait that much an then some
# to make sure everything has been flushed
time.sleep(35)
# TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
(err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1],
check_status=True)
(err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
check_status=True)
assert len(out['entries']) == 0
assert len(out['summary']) == 0
# TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
check_status=True)
# TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
try:
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('five')
except boto.exception.S3ResponseError as e:
assert e.status == 403
# TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1],
check_status=True)
# TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('six')
# TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
# create an object large enough to be split into multiple parts
test_string = 'foo'*10000000
big_key = boto.s3.key.Key(bucket)
big_key.set_contents_from_string(test_string)
# now delete the head
big_key.delete()
# wait a bit to give the garbage collector time to cycle
time.sleep(15)
(err, out) = rgwadmin(ctx, client, ['gc', 'list'])
assert len(out) > 0
# TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
(err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)
#confirm
(err, out) = rgwadmin(ctx, client, ['gc', 'list'])
assert len(out) == 0
# TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
assert err
# delete should fail because ``key`` still exists
try:
bucket.delete()
except boto.exception.S3ResponseError as e:
assert e.status == 409
key.delete()
bucket.delete()
# TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
bucket = connection.create_bucket(bucket_name)
# create an object
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('seven')
# should be private already but guarantee it
key.set_acl('private')
(err, out) = rgwadmin(ctx, client,
['policy', '--bucket', bucket.name, '--object', key.key],
check_status=True, format='xml')
acl = get_acl(key)
assert acl == out.strip('\n')
# add another grantee by making the object public read
key.set_acl('public-read')
(err, out) = rgwadmin(ctx, client,
['policy', '--bucket', bucket.name, '--object', key.key],
check_status=True, format='xml')
acl = get_acl(key)
assert acl == out.strip('\n')
# TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key_name = ['eight', 'nine', 'ten', 'eleven']
for i in range(4):
key = boto.s3.key.Key(bucket)
key.set_contents_from_string(key_name[i])
(err, out) = rgwadmin(ctx, client,
['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
check_status=True)
# TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
caps='user=read'
(err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps])
assert out['caps'][0]['perm'] == 'read'
# TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
(err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps])
assert not out['caps']
# TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
assert err
# TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('twelve')
(err, out) = rgwadmin(ctx, client,
['user', 'rm', '--uid', user1, '--purge-data' ],
check_status=True)
# TESTCASE 'rm-user3','user','rm','deleted user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
#
if realm is None:
(err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
else:
(err, out) = rgwadmin(ctx, client, ['zone', 'get'])
orig_placement_pools = len(out['placement_pools'])
# removed this test, it is not correct to assume that zone has default placement, it really
# depends on how we set it up before
#
# assert len(out) > 0
# assert len(out['placement_pools']) == 1
# default_rule = out['placement_pools'][0]
# assert default_rule['key'] == 'default-placement'
rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}
out['placement_pools'].append(rule)
(err, out) = rgwadmin(ctx, client, ['zone', 'set'],
stdin=StringIO(json.dumps(out)),
check_status=True)
if realm is None:
(err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
else:
(err, out) = rgwadmin(ctx, client, ['zone', 'get'])
assert len(out) > 0
assert len(out['placement_pools']) == orig_placement_pools + 1
def pull_configuration(ctx, config, regions, role_endpoints, realm, master_client):
"""
Configure regions and zones from rados and rgw.
"""
if not regions:
log.debug(
'In rgw.pull_confguration() and regions is None. '
'Bailing')
yield
return
if not realm:
log.debug(
'In rgw.pull_configuration() and realm is None. '
'Bailing')
yield
return
log.info('Pulling configuration...')
log.debug('config is %r', config)
log.debug('regions are %r', regions)
log.debug('role_endpoints = %r', role_endpoints)
log.debug('realm is %r', realm)
log.debug('master client = %r', master_client)
# extract the zone info
role_zones = dict([(client, extract_zone_info(ctx, client, c_config))
for client, c_config in config.iteritems()])
log.debug('roles_zones = %r', role_zones)
# extract the user info and append it to the payload tuple for the given
# client
for client, c_config in config.iteritems():
if not c_config:
user_info = None
else:
user_info = extract_user_info(c_config)
(region, zone, zone_info) = role_zones[client]
role_zones[client] = (region, zone, zone_info, user_info)
region_info = dict([
(region_name, extract_region_info(region_name, r_config))
for region_name, r_config in regions.iteritems()])
fill_in_endpoints(region_info, role_zones, role_endpoints)
for client in config.iterkeys():
if client != master_client:
host, port = role_endpoints[master_client]
endpoint = 'http://{host}:{port}/'.format(host=host, port=port)
log.debug("endpoint: %s", endpoint)
rgwadmin(ctx, client,
cmd=['-n', client, 'realm', 'pull', '--rgw-realm', realm, '--default', '--url',
endpoint, '--access_key',
user_info['system_key']['access_key'], '--secret',
user_info['system_key']['secret_key']],
check_status=True)
(zonegroup, zone, zone_info, zone_user_info) = role_zones[client]
zone_json = json.dumps(dict(zone_info.items() + zone_user_info.items()))
log.debug("zone info is: %r"), zone_json
rgwadmin(ctx, client,
cmd=['zone', 'set', '--rgw-zonegroup', zonegroup,
'--rgw-zone', zone],
stdin=StringIO(zone_json),
check_status=True)
rgwadmin(ctx, client,
cmd=['period', 'update', '--commit', '--url',
endpoint, '--access_key',
user_info['system_key']['access_key'], '--secret',
user_info['system_key']['secret_key']],
check_status=True)
yield
def configure_multisite_regions_and_zones(ctx, config, regions, role_endpoints, realm, master_client):
"""
Configure multisite regions and zones from rados and rgw.
"""
if not regions:
log.debug(
'In rgw.configure_multisite_regions_and_zones() and regions is None. '
'Bailing')
yield
return
if not realm:
log.debug(
'In rgw.configure_multisite_regions_and_zones() and realm is None. '
'Bailing')
yield
return
log.info('Configuring multisite regions and zones...')
log.debug('config is %r', config)
log.debug('regions are %r', regions)
log.debug('role_endpoints = %r', role_endpoints)
log.debug('realm is %r', realm)
# extract the zone info
role_zones = dict([(client, extract_zone_info(ctx, client, c_config))
for client, c_config in config.iteritems()])
log.debug('role_zones = %r', role_zones)
# extract the user info and append it to the payload tuple for the given
# client
for client, c_config in config.iteritems():
if not c_config:
user_info = None
else:
user_info = extract_user_info(c_config)
(region, zone, zone_info) = role_zones[client]
role_zones[client] = (region, zone, zone_info, user_info)
region_info = dict([
(region_name, extract_region_info(region_name, r_config))
for region_name, r_config in regions.iteritems()])
fill_in_endpoints(region_info, role_zones, role_endpoints)
# clear out the old defaults
first_mon = teuthology.get_first_mon(ctx, config)
(mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
# read master zonegroup and master_zone
for zonegroup, zg_info in region_info.iteritems():
if zg_info['is_master']:
master_zonegroup = zonegroup
master_zone = zg_info['master_zone']
break
log.debug('master zonegroup =%r', master_zonegroup)
log.debug('master zone = %r', master_zone)
log.debug('master client = %r', master_client)
rgwadmin(ctx, master_client,
cmd=['realm', 'create', '--rgw-realm', realm, '--default'],
check_status=True)
for region, info in region_info.iteritems():
region_json = json.dumps(info)
log.debug('region info is: %s', region_json)
rgwadmin(ctx, master_client,
cmd=['zonegroup', 'set'],
stdin=StringIO(region_json),
check_status=True)
rgwadmin(ctx, master_client,
cmd=['zonegroup', 'default', '--rgw-zonegroup', master_zonegroup],
check_status=True)
for role, (zonegroup, zone, zone_info, user_info) in role_zones.iteritems():
(remote,) = ctx.cluster.only(role).remotes.keys()
for pool_info in zone_info['placement_pools']:
remote.run(args=['sudo', 'ceph', 'osd', 'pool', 'create',
pool_info['val']['index_pool'], '64', '64'])
if ctx.rgw.ec_data_pool:
create_ec_pool(remote, pool_info['val']['data_pool'],
zone, 64, ctx.rgw.erasure_code_profile)
else:
create_replicated_pool(remote, pool_info['val']['data_pool'], 64)
(zonegroup, zone, zone_info, user_info) = role_zones[master_client]
zone_json = json.dumps(dict(zone_info.items() + user_info.items()))
log.debug("zone info is: %r", zone_json)
rgwadmin(ctx, master_client,
cmd=['zone', 'set', '--rgw-zonegroup', zonegroup,
'--rgw-zone', zone],
stdin=StringIO(zone_json),
check_status=True)
rgwadmin(ctx, master_client,
cmd=['-n', master_client, 'zone', 'default', zone],
check_status=True)
rgwadmin(ctx, master_client,
cmd=['-n', master_client, 'period', 'update', '--commit'],
check_status=True)
yield
def configure_multisite_regions_and_zones(ctx, config, regions, role_endpoints,
realm, master_client):
"""
Configure multisite regions and zones from rados and rgw.
"""
if not regions:
log.debug(
'In rgw.configure_multisite_regions_and_zones() and regions is None. '
'Bailing')
yield
return
if not realm:
log.debug(
'In rgw.configure_multisite_regions_and_zones() and realm is None. '
'Bailing')
yield
return
log.info('Configuring multisite regions and zones...')
log.debug('config is %r', config)
log.debug('regions are %r', regions)
log.debug('role_endpoints = %r', role_endpoints)
log.debug('realm is %r', realm)
# extract the zone info
role_zones = dict([(client, extract_zone_info(ctx, client, c_config))
for client, c_config in config.iteritems()])
log.debug('role_zones = %r', role_zones)
# extract the user info and append it to the payload tuple for the given
# client
for client, c_config in config.iteritems():
if not c_config:
user_info = None
else:
user_info = extract_user_info(c_config)
(region, zone, zone_info) = role_zones[client]
role_zones[client] = (region, zone, zone_info, user_info)
region_info = dict([(region_name,
extract_region_info(region_name, r_config))
for region_name, r_config in regions.iteritems()])
fill_in_endpoints(region_info, role_zones, role_endpoints)
# clear out the old defaults
cluster_name, daemon_type, client_id = teuthology.split_role(master_client)
first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
(mon, ) = ctx.cluster.only(first_mon).remotes.iterkeys()
# read master zonegroup and master_zone
for zonegroup, zg_info in region_info.iteritems():
if zg_info['is_master']:
master_zonegroup = zonegroup
master_zone = zg_info['master_zone']
break
log.debug('master zonegroup =%r', master_zonegroup)
log.debug('master zone = %r', master_zone)
log.debug('master client = %r', master_client)
rgwadmin(ctx,
master_client,
cmd=['realm', 'create', '--rgw-realm', realm, '--default'],
check_status=True)
for region, info in region_info.iteritems():
region_json = json.dumps(info)
log.debug('region info is: %s', region_json)
rgwadmin(ctx,
master_client,
cmd=['zonegroup', 'set'],
stdin=StringIO(region_json),
check_status=True)
rgwadmin(ctx,
master_client,
cmd=['zonegroup', 'default', '--rgw-zonegroup', master_zonegroup],
check_status=True)
for role, (zonegroup, zone, zone_info,
user_info) in role_zones.iteritems():
(remote, ) = ctx.cluster.only(role).remotes.keys()
for pool_info in zone_info['placement_pools']:
remote.run(args=[
'sudo', 'ceph', 'osd', 'pool', 'create', pool_info['val']
['index_pool'], '64', '64', '--cluster', cluster_name
])
if ctx.rgw.ec_data_pool:
create_ec_pool(remote, pool_info['val']['data_pool'], zone, 64,
ctx.rgw.erasure_code_profile, cluster_name)
else:
create_replicated_pool(remote, pool_info['val']['data_pool'],
64, cluster_name)
(zonegroup, zone, zone_info, user_info) = role_zones[master_client]
zone_json = json.dumps(dict(zone_info.items() + user_info.items()))
log.debug("zone info is: %r", zone_json)
rgwadmin(
ctx,
master_client,
cmd=['zone', 'set', '--rgw-zonegroup', zonegroup, '--rgw-zone', zone],
stdin=StringIO(zone_json),
check_status=True)
rgwadmin(ctx,
master_client,
cmd=['zone', 'default', '--rgw-zone', zone],
check_status=True)
rgwadmin(ctx,
master_client,
cmd=['period', 'update', '--commit'],
check_status=True)
yield
def task(ctx, config):
"""
Test radosgw-admin functionality against a running rgw instance.
"""
global log
assert ctx.rgw.config, \
"radosgw_admin task needs a config passed from the rgw task"
config = ctx.rgw.config
log.debug('config is: %r', config)
clients_from_config = config.keys()
# choose first client as default
client = clients_from_config[0]
# once the client is chosen, pull the host name and assigned port out of
# the role_endpoints that were assigned by the rgw task
(remote_host, remote_port) = ctx.rgw.role_endpoints[client]
##
user1 = 'foo'
user2 = 'fud'
subuser1 = 'foo:foo1'
subuser2 = 'foo:foo2'
display_name1 = 'Foo'
display_name2 = 'Fud'
email = '*****@*****.**'
email2 = '*****@*****.**'
access_key = '9te6NH5mcdcq0Tc5i8i1'
secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
access_key2 = 'p5YnriCv1nAtykxBrupQ'
secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'
bucket_name = 'myfoo'
bucket_name2 = 'mybar'
# connect to rgw
connection = boto.s3.connection.S3Connection(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
connection2 = boto.s3.connection.S3Connection(
aws_access_key_id=access_key2,
aws_secret_access_key=secret_key2,
is_secure=False,
port=remote_port,
host=remote_host,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)
# legend (test cases can be easily grep-ed out)
# TESTCASE 'testname','object','method','operation','assertion'
# TESTCASE 'info-nosuch','user','info','non-existent user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
(err, out) = rgwadmin(
ctx,
client, [
'user', 'create', '--uid', user1, '--display-name', display_name1,
'--email', email, '--access-key', access_key, '--secret',
secret_key, '--max-buckets', '4'
],
check_status=True)
# TESTCASE 'duplicate email','user','create','existing user email','fails'
(err, out) = rgwadmin(ctx, client, [
'user',
'create',
'--uid',
user2,
'--display-name',
display_name2,
'--email',
email,
])
assert err
# TESTCASE 'info-existing','user','info','existing user','returns correct info'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert out['user_id'] == user1
assert out['email'] == email
assert out['display_name'] == display_name1
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
assert not out['suspended']
# TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
(err, out) = rgwadmin(ctx,
client, ['user', 'suspend', '--uid', user1],
check_status=True)
# TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert out['suspended']
# TESTCASE 're-enable','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx,
client, ['user', 'enable', '--uid', user1],
check_status=True)
# TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert not out['suspended']
# TESTCASE 'add-keys','key','create','w/valid info','succeeds'
(err, out) = rgwadmin(ctx,
client, [
'key',
'create',
'--uid',
user1,
'--access-key',
access_key2,
'--secret',
secret_key2,
],
check_status=True)
# TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert len(out['keys']) == 2
assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][
'access_key'] == access_key2
assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][
'secret_key'] == secret_key2
# TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
(err, out) = rgwadmin(ctx,
client, [
'key',
'rm',
'--uid',
user1,
'--access-key',
access_key2,
],
check_status=True)
assert len(out['keys']) == 1
assert out['keys'][0]['access_key'] == access_key
assert out['keys'][0]['secret_key'] == secret_key
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
subuser_access = 'full'
subuser_perm = 'full-control'
(err, out) = rgwadmin(ctx,
client, [
'subuser', 'create', '--subuser', subuser1,
'--access', subuser_access
],
check_status=True)
# TESTCASE 'add-swift-key','key','create','swift key','succeeds'
(err, out) = rgwadmin(ctx,
client, [
'subuser',
'modify',
'--subuser',
subuser1,
'--secret',
swift_secret1,
'--key-type',
'swift',
],
check_status=True)
# TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert out['subusers'][0]['permissions'] == subuser_perm
# TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert len(out['swift_keys']) == 1
assert out['swift_keys'][0]['user'] == subuser1
assert out['swift_keys'][0]['secret_key'] == swift_secret1
# TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
(err, out) = rgwadmin(ctx,
client, [
'subuser',
'create',
'--subuser',
subuser2,
'--secret',
swift_secret2,
'--key-type',
'swift',
],
check_status=True)
# TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
(err, out) = rgwadmin(ctx,
client, ['user', 'info', '--uid', user1],
check_status=True)
assert len(out['swift_keys']) == 2
assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][
'user'] == subuser2
assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[
'swift_keys'][1]['secret_key'] == swift_secret2
# TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
(err, out) = rgwadmin(ctx,
client, [
'key',
'rm',
'--subuser',
subuser1,
'--key-type',
'swift',
],
check_status=True)
assert len(out['swift_keys']) == 1
# TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
(err, out) = rgwadmin(ctx,
client, [
'subuser',
'rm',
'--subuser',
subuser1,
],
check_status=True)
assert len(out['subusers']) == 1
# TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
(err, out) = rgwadmin(ctx,
client, [
'subuser',
'rm',
'--subuser',
subuser2,
'--key-type',
'swift',
'--purge-keys',
],
check_status=True)
assert len(out['swift_keys']) == 0
assert len(out['subusers']) == 0
# TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--uid', user1],
check_status=True)
assert len(out) == 0
# TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'list', '--uid', user1],
check_status=True)
assert len(out) == 0
# create a first bucket
bucket = connection.create_bucket(bucket_name)
# TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'list', '--uid', user1],
check_status=True)
assert len(out) == 1
assert out[0] == bucket_name
# TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
(err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
assert len(out) >= 1
assert bucket_name in out
# TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
bucket2 = connection.create_bucket(bucket_name + '2')
bucket3 = connection.create_bucket(bucket_name + '3')
bucket4 = connection.create_bucket(bucket_name + '4')
# the 5th should fail.
failed = False
try:
connection.create_bucket(bucket_name + '5')
except Exception:
failed = True
assert failed
# delete the buckets
bucket2.delete()
bucket3.delete()
bucket4.delete()
# TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--bucket', bucket_name],
check_status=True)
assert out['owner'] == user1
bucket_id = out['id']
# TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--uid', user1],
check_status=True)
assert len(out) == 1
assert out[0][
'id'] == bucket_id # does it return the same ID twice in a row?
# use some space
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('one')
# TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--bucket', bucket_name],
check_status=True)
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 1
assert out['usage']['rgw.main']['size_kb'] > 0
# reclaim it
key.delete()
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
(err, out) = rgwadmin(
ctx,
client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
check_status=True)
# create a second user to link the bucket to
(err, out) = rgwadmin(ctx,
client, [
'user',
'create',
'--uid',
user2,
'--display-name',
display_name2,
'--access-key',
access_key2,
'--secret',
secret_key2,
'--max-buckets',
'1',
],
check_status=True)
# try creating an object with the first user before the bucket is relinked
denied = False
key = boto.s3.key.Key(bucket)
try:
key.set_contents_from_string('two')
except boto.exception.S3ResponseError:
denied = True
assert not denied
# delete the object
key.delete()
# link the bucket to another user
(err,
out) = rgwadmin(ctx,
client,
['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
check_status=True)
bucket_data = out['data']
assert bucket_data['bucket']['name'] == bucket_name
bucket_id = bucket_data['bucket']['bucket_id']
# link the bucket to another user
(err, out) = rgwadmin(ctx,
client, [
'bucket', 'link', '--uid', user2, '--bucket',
bucket_name, '--bucket-id', bucket_id
],
check_status=True)
# try to remove user, should fail (has a linked bucket)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
assert err
# TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
(err, out) = rgwadmin(
ctx,
client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
check_status=True)
# relink the bucket to the first user and delete the second user
(err, out) = rgwadmin(ctx,
client, [
'bucket', 'link', '--uid', user1, '--bucket',
bucket_name, '--bucket-id', bucket_id
],
check_status=True)
(err, out) = rgwadmin(ctx,
client, ['user', 'rm', '--uid', user2],
check_status=True)
# TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'
# upload an object
object_name = 'four'
key = boto.s3.key.Key(bucket, object_name)
key.set_contents_from_string(object_name)
# now delete it
(err, out) = rgwadmin(
ctx,
client,
['object', 'rm', '--bucket', bucket_name, '--object', object_name],
check_status=True)
# TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx,
client, ['bucket', 'stats', '--bucket', bucket_name],
check_status=True)
assert out['id'] == bucket_id
assert out['usage']['rgw.main']['num_objects'] == 0
# list log objects
# TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
(err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
assert len(out) > 0
for obj in out:
# TESTCASE 'log-show','log','show','after activity','returns expected info'
if obj[:
4] == 'meta' or obj[:
4] == 'data' or obj[:
18] == 'obj_delete_at_hint':
continue
(err, rgwlog) = rgwadmin(ctx,
client, ['log', 'show', '--object', obj],
check_status=True)
assert len(rgwlog) > 0
# exempt bucket_name2 from checking as it was only used for multi-region tests
assert rgwlog['bucket'].find(
bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
assert rgwlog['bucket'] != bucket_name or rgwlog[
'bucket_id'] == bucket_id
assert rgwlog['bucket_owner'] == user1 or rgwlog[
'bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
for entry in rgwlog['log_entries']:
log.debug('checking log entry: ', entry)
assert entry['bucket'] == rgwlog['bucket']
possible_buckets = [bucket_name + '5', bucket_name2]
user = entry['user']
assert user == user1 or user.endswith('system-user') or \
rgwlog['bucket'] in possible_buckets
# TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
(err, out) = rgwadmin(ctx,
client, ['log', 'rm', '--object', obj],
check_status=True)
# TODO: show log by bucket+date
# need to wait for all usage data to get flushed, should take up to 30 seconds
timestamp = time.time()
while time.time() - timestamp <= (20 * 60): # wait up to 20 minutes
(err, out) = rgwadmin(
ctx, client,
['usage', 'show', '--categories', 'delete_obj'
]) # last operation we did is delete obj, wait for it to flush
if get_user_successful_ops(out, user1) > 0:
break
time.sleep(1)
assert time.time() - timestamp <= (20 * 60)
# TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
(err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = get_user_summary(out, user1)
total = user_summary['total']
assert total['successful_ops'] > 0
# TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
(err, out) = rgwadmin(ctx,
client, ['usage', 'show', '--uid', user1],
check_status=True)
assert len(out['entries']) > 0
assert len(out['summary']) > 0
user_summary = out['summary'][0]
for entry in user_summary['categories']:
assert entry['successful_ops'] > 0
assert user_summary['user'] == user1
# TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
test_categories = [
'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'
]
for cat in test_categories:
(err, out) = rgwadmin(
ctx,
client, ['usage', 'show', '--uid', user1, '--categories', cat],
check_status=True)
assert len(out['summary']) > 0
user_summary = out['summary'][0]
assert user_summary['user'] == user1
assert len(user_summary['categories']) == 1
entry = user_summary['categories'][0]
assert entry['category'] == cat
assert entry['successful_ops'] > 0
# the usage flush interval is 30 seconds, wait that much an then some
# to make sure everything has been flushed
time.sleep(35)
# TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
(err, out) = rgwadmin(ctx,
client, ['usage', 'trim', '--uid', user1],
check_status=True)
(err, out) = rgwadmin(ctx,
client, ['usage', 'show', '--uid', user1],
check_status=True)
assert len(out['entries']) == 0
assert len(out['summary']) == 0
# TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
(err, out) = rgwadmin(ctx,
client, ['user', 'suspend', '--uid', user1],
check_status=True)
# TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
try:
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('five')
except boto.exception.S3ResponseError as e:
assert e.status == 403
# TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
(err, out) = rgwadmin(ctx,
client, ['user', 'enable', '--uid', user1],
check_status=True)
# TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('six')
# TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'
# create an object large enough to be split into multiple parts
test_string = 'foo' * 10000000
big_key = boto.s3.key.Key(bucket)
big_key.set_contents_from_string(test_string)
# now delete the head
big_key.delete()
# wait a bit to give the garbage collector time to cycle
time.sleep(15)
(err, out) = rgwadmin(ctx, client, ['gc', 'list'])
assert len(out) > 0
# TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
(err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)
#confirm
(err, out) = rgwadmin(ctx, client, ['gc', 'list'])
assert len(out) == 0
# TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
assert err
# delete should fail because ``key`` still exists
try:
bucket.delete()
except boto.exception.S3ResponseError as e:
assert e.status == 409
key.delete()
bucket.delete()
# TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
bucket = connection.create_bucket(bucket_name)
# create an object
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('seven')
# should be private already but guarantee it
key.set_acl('private')
(err,
out) = rgwadmin(ctx,
client,
['policy', '--bucket', bucket.name, '--object', key.key],
check_status=True,
format='xml')
acl = get_acl(key)
assert acl == out.strip('\n')
# add another grantee by making the object public read
key.set_acl('public-read')
(err,
out) = rgwadmin(ctx,
client,
['policy', '--bucket', bucket.name, '--object', key.key],
check_status=True,
format='xml')
acl = get_acl(key)
assert acl == out.strip('\n')
# TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key_name = ['eight', 'nine', 'ten', 'eleven']
for i in range(4):
key = boto.s3.key.Key(bucket)
key.set_contents_from_string(key_name[i])
(err, out) = rgwadmin(
ctx,
client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
check_status=True)
# TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
caps = 'user=read'
(err, out) = rgwadmin(ctx, client,
['caps', 'add', '--uid', user1, '--caps', caps])
assert out['caps'][0]['perm'] == 'read'
# TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
(err, out) = rgwadmin(ctx, client,
['caps', 'rm', '--uid', user1, '--caps', caps])
assert not out['caps']
# TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
(err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
assert err
# TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
bucket = connection.create_bucket(bucket_name)
key = boto.s3.key.Key(bucket)
key.set_contents_from_string('twelve')
(err, out) = rgwadmin(ctx,
client,
['user', 'rm', '--uid', user1, '--purge-data'],
check_status=True)
# TESTCASE 'rm-user3','user','rm','deleted user','fails'
(err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
assert err
# TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
#
(err, out) = rgwadmin(ctx, client,
['zone', 'get', '--rgw-zone', 'default'])
orig_placement_pools = len(out['placement_pools'])
# removed this test, it is not correct to assume that zone has default placement, it really
# depends on how we set it up before
#
# assert len(out) > 0
# assert len(out['placement_pools']) == 1
# default_rule = out['placement_pools'][0]
# assert default_rule['key'] == 'default-placement'
rule = {
'key': 'new-placement',
'val': {
'data_pool': '.rgw.buckets.2',
'index_pool': '.rgw.buckets.index.2'
}
}
out['placement_pools'].append(rule)
(err, out) = rgwadmin(ctx,
client, ['zone', 'set'],
stdin=StringIO(json.dumps(out)),
check_status=True)
(err, out) = rgwadmin(ctx, client,
['zone', 'get', '--rgw-zone', 'default'])
assert len(out) > 0
assert len(out['placement_pools']) == orig_placement_pools + 1
def configure_regions_and_zones(ctx, config, regions, role_endpoints, realm):
"""
Configure regions and zones from rados and rgw.
"""
if not regions:
log.debug(
'In rgw.configure_regions_and_zones() and regions is None. '
'Bailing')
yield
return
if not realm:
log.debug(
'In rgw.configure_regions_and_zones() and realm is None. '
'Bailing')
yield
return
log.info('Configuring regions and zones...')
log.debug('config is %r', config)
log.debug('regions are %r', regions)
log.debug('role_endpoints = %r', role_endpoints)
log.debug('realm is %r', realm)
# extract the zone info
role_zones = dict([(client, extract_zone_info(ctx, client, c_config))
for client, c_config in config.iteritems()])
log.debug('roles_zones = %r', role_zones)
# extract the user info and append it to the payload tuple for the given
# client
for client, c_config in config.iteritems():
if not c_config:
user_info = None
else:
user_info = extract_user_info(c_config)
(region, zone, zone_info) = role_zones[client]
role_zones[client] = (region, zone, zone_info, user_info)
region_info = dict([
(region_name, extract_region_info(region_name, r_config))
for region_name, r_config in regions.iteritems()])
fill_in_endpoints(region_info, role_zones, role_endpoints)
# clear out the old defaults
first_mon = teuthology.get_first_mon(ctx, config)
(mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
# removing these objects from .rgw.root and the per-zone root pools
# may or may not matter
rados(ctx, mon,
cmd=['-p', '.rgw.root', 'rm', 'region_info.default'])
rados(ctx, mon,
cmd=['-p', '.rgw.root', 'rm', 'zone_info.default'])
# read master zonegroup and master_zone
for zonegroup, zg_info in region_info.iteritems():
if zg_info['is_master']:
master_zonegroup = zonegroup
master_zone = zg_info['master_zone']
break
for client in config.iterkeys():
(zonegroup, zone, zone_info, user_info) = role_zones[client]
if zonegroup == master_zonegroup and zone == master_zone:
master_client = client
break
log.debug('master zonegroup =%r', master_zonegroup)
log.debug('master zone = %r', master_zone)
log.debug('master client = %r', master_client)
log.debug('config %r ', config)
(ret, out)=rgwadmin(ctx, master_client,
cmd=['realm', 'create', '--rgw-realm', realm, '--default'])
log.debug('realm create ret %r exists %r', -ret, errno.EEXIST)
assert ret == 0 or ret != -errno.EEXIST
if ret is -errno.EEXIST:
log.debug('realm %r exists', realm)
for client in config.iterkeys():
for role, (zonegroup, zone, zone_info, user_info) in role_zones.iteritems():
rados(ctx, mon,
cmd=['-p', zone_info['domain_root'],
'rm', 'region_info.default'])
rados(ctx, mon,
cmd=['-p', zone_info['domain_root'],
'rm', 'zone_info.default'])
(remote,) = ctx.cluster.only(role).remotes.keys()
for pool_info in zone_info['placement_pools']:
remote.run(args=['sudo', 'ceph', 'osd', 'pool', 'create',
pool_info['val']['index_pool'], '64', '64'])
if ctx.rgw.ec_data_pool:
create_ec_pool(remote, pool_info['val']['data_pool'],
zone, 64, ctx.rgw.erasure_code_profile)
else:
create_replicated_pool(
remote, pool_info['val']['data_pool'],
64)
zone_json = json.dumps(dict(zone_info.items() + user_info.items()))
log.debug('zone info is: %r', zone_json)
rgwadmin(ctx, client,
cmd=['zone', 'set', '--rgw-zonegroup', zonegroup,
'--rgw-zone', zone],
stdin=StringIO(zone_json),
check_status=True)
for region, info in region_info.iteritems():
region_json = json.dumps(info)
log.debug('region info is: %s', region_json)
rgwadmin(ctx, client,
cmd=['zonegroup', 'set'],
stdin=StringIO(region_json),
check_status=True)
if info['is_master']:
rgwadmin(ctx, client,
cmd=['zonegroup', 'default', '--rgw-zonegroup', master_zonegroup],
check_status=True)
(zonegroup, zone, zone_info, user_info) = role_zones[client]
rgwadmin(ctx, client,
cmd=['zone', 'default', zone],
check_status=True)
rgwadmin(ctx, master_client,
cmd=['-n', master_client, 'period', 'update', '--commit'],
check_status=True)
yield