Exemple #1
0
def store_uploaded_file(f, name):
    img_dir = '/tmp/image/'
    if not os.path.exists(img_dir):
        os.mkdir(img_dir)
    file_name = id_generator()
    path = '%s%s' % (img_dir, file_name)
    with open(path, 'wb+') as destination:
        for chunk in f.chunks():
            destination.write(chunk)

    calling_format = boto.s3.connection.OrdinaryCallingFormat()
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=settings.IMAGECRUD['access_key'],
        aws_secret_access_key=settings.IMAGECRUD['secret_key'],
        is_secure=False,
        host=settings.IMAGECRUD['s3_host'],
        port=settings.IMAGECRUD['s3_port'],
        calling_format=calling_format,
        path=settings.IMAGECRUD['s3_path'])

    try:
        bucket = connection.get_bucket(settings.IMAGECRUD['img_bucket'])
    except:
        bucket = connection.create_bucket(settings.IMAGECRUD['img_bucket'])

    key_name = '%s.jpg' % id_generator()
    key = bucket.new_key(key_name)
    key.set_contents_from_filename(path)
    key.set_canned_acl('public-read')
    key.close()

    return 'http://%s:%s%s/%s/%s' % (
        settings.IMAGECRUD['s3_host'], settings.IMAGECRUD['s3_port'],
        settings.IMAGECRUD['s3_path'], settings.IMAGECRUD['img_bucket'],
        key_name)
Exemple #2
0
def store_uploaded_file(f, name):
    img_dir='/tmp/image/'
    if not os.path.exists(img_dir):
        os.mkdir(img_dir)
    file_name = id_generator()
    path = '%s%s' % (img_dir,file_name)
    with open(path , 'wb+') as destination:
        for chunk in f.chunks():
            destination.write(chunk)

    calling_format=boto.s3.connection.OrdinaryCallingFormat()
    connection = boto.s3.connection.S3Connection(aws_access_key_id=settings.IMAGECRUD['access_key'],
                      aws_secret_access_key=settings.IMAGECRUD['secret_key'],
                      is_secure=False,
                      host=settings.IMAGECRUD['s3_host'],
                      port=settings.IMAGECRUD['s3_port'],
                      calling_format=calling_format,
                      path=settings.IMAGECRUD['s3_path'])

    try:
        bucket = connection.get_bucket(settings.IMAGECRUD['img_bucket'])
    except:
        bucket = connection.create_bucket(settings.IMAGECRUD['img_bucket'])

    key_name= '%s.jpg' % id_generator()
    key = bucket.new_key(key_name)
    key.set_contents_from_filename(path)
    key.set_canned_acl('public-read')
    key.close()

    return 'http://%s:%s%s/%s/%s' % (settings.IMAGECRUD['s3_host'], settings.IMAGECRUD['s3_port'],settings.IMAGECRUD['s3_path'],settings.IMAGECRUD['img_bucket'],key_name)
Exemple #3
0
def get_new_bucket(connection=None):
    """
    Get a bucket that exists and is empty.

    Always recreates a bucket from scratch. This is useful to also
    reset ACLs and such.
    """
    if connection is None:
        connection = s3.main
    name = get_new_bucket_name()
    # the only way for this to fail with a pre-existing bucket is if
    # someone raced us between setup nuke_prefixed_buckets and here;
    # ignore that as astronomically unlikely
    bucket = connection.create_bucket(name)
    return bucket
Exemple #4
0
def get_new_bucket(connection=None):
    """
    Get a bucket that exists and is empty.

    Always recreates a bucket from scratch. This is useful to also
    reset ACLs and such.
    """
    if connection is None:
        connection = s3.main
    name = get_new_bucket_name()
    # the only way for this to fail with a pre-existing bucket is if
    # someone raced us between setup nuke_prefixed_buckets and here;
    # ignore that as astronomically unlikely
    bucket = connection.create_bucket(name)
    return bucket
Exemple #5
0
def get_new_bucket(target=None, name=None, headers=None):
    """
    Get a bucket that exists and is empty.

    Always recreates a bucket from scratch. This is useful to also
    reset ACLs and such.
    """
    if target is None:
        target = targets.main.default
    connection = target.connection
    if name is None:
        name = get_new_bucket_name()
    # the only way for this to fail with a pre-existing bucket is if
    # someone raced us between setup nuke_prefixed_buckets and here;
    # ignore that as astronomically unlikely
    bucket = connection.create_bucket(name, location=target.conf.api_name, headers=headers)
    return bucket
Exemple #6
0
def test_bucket():

    s3_access_key=os.environ['S3_ACCESS_KEY_ID']
    s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY']
    
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id= s3_access_key,
        aws_secret_access_key=s3_secret_key,
        is_secure=False,
        port=8000,
        host='localhost',
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    print 'create a bucket'
    bucket = connection.create_bucket('bucket1')

    keys = ['mykey1', 'mykey2', 'mykey3', 'mykey4', 'obj1']
    keys_to_delete = ['obj1']

    for key in keys:
        obj = boto.s3.key.Key(bucket)
        obj.key = key
        obj.set_contents_from_string('This is ' + key)

    print 'bucket content:'
    for k in bucket.list():
        print k

    for o in bucket.list(prefix='obj'):
        print(o.key)

    result = bucket.delete_keys(keys_to_delete)
    print result

    for o in bucket.list(prefix='obj'):
        print(o.key)

    print 'after delete_keys'
    for k in bucket.list():
        print k
    
    for k in bucket.list():
        bucket.delete_key(k)
    
    connection.delete_bucket(bucket)
Exemple #7
0
def get_new_bucket(target=None, name=None, headers=None):
    """
    Get a bucket that exists and is empty.

    Always recreates a bucket from scratch. This is useful to also
    reset ACLs and such.
    """
    if target is None:
        target = targets.main.default
    connection = target.connection
    if name is None:
        name = get_new_bucket_name()
    # the only way for this to fail with a pre-existing bucket is if
    # someone raced us between setup nuke_prefixed_buckets and here;
    # ignore that as astronomically unlikely
    bucket = connection.create_bucket(name, location=target.conf.api_name, headers=headers)
    return bucket
def test_bucket_create_special_key_names():

    s3_access_key=os.environ['S3_ACCESS_KEY_ID']
    s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY']

    connection = boto.s3.connection.S3Connection(
        aws_access_key_id= s3_access_key,
        aws_secret_access_key=s3_secret_key,
        is_secure=False,
        port=8000,
        host='localhost',
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    bucket = connection.create_bucket('version1')
    bucket.configure_versioning(True)

    objname = 'goodobj'
    i = 0
    c = []
    k = []

    c.append('content-{i}'.format(i=i))

    key = bucket.new_key(objname)
    key.set_contents_from_string(c[i])

    key = bucket.get_key(objname)
    s = key.get_contents_as_string()

    print key, ' contents ' , s
    objname = '_testobj'
    
    key = bucket.new_key(objname)
    key.set_contents_from_string(c[i])

    key = bucket.get_key(objname)
    s = key.get_contents_as_string()

    print key, ' contents ' , s
Exemple #9
0
def test_bucket():

    s3_access_key=os.environ['S3_ACCESS_KEY_ID']
    s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY']
    
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id= s3_access_key,
        aws_secret_access_key=s3_secret_key,
        is_secure=False,
        port=8000,
        host='localhost',
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket_name = 'myfoo'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')

    # should be private already but guarantee it
    key.set_acl('private')

    
    print 'key=' + key.key

    acl = get_acl(key)

    print acl

    (err, out) = rgwadmin(['policy', '--bucket', bucket.name, '--object', key.key],
        check_status=True, format='xml')

    print "rgwadmin: ", err
    print out

    assert acl == out.strip('\n')
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    assert config is None or isinstance(config, list) \
        or isinstance(config, dict), \
        "task s3tests only supports a list or dictionary for configuration"
    all_clients = ['client.{id}'.format(id=id_)
                   for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients = config.keys()

    multi_region_run = rgw_utils.multi_region_enabled(ctx)

    client = clients[0]; # default choice, multi-region code may overwrite this
    if multi_region_run:
        client = rgw_utils.get_master_client(ctx, clients)

    # once the client is chosen, pull the host name and  assigned port out of
    # the role_endpoints that were assigned by the rgw task
    (remote_host, remote_port) = ctx.rgw.role_endpoints[client]

    ##
    user1='foo'
    user2='fud'
    subuser1='foo:foo1'
    subuser2='foo:foo2'
    display_name1='Foo'
    display_name2='Fud'
    email='*****@*****.**'
    email2='*****@*****.**'
    access_key='9te6NH5mcdcq0Tc5i8i1'
    secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2='p5YnriCv1nAtykxBrupQ'
    secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name='myfoo'
    bucket_name2='mybar'

    # connect to rgw
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )
    connection2 = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key2,
        aws_secret_access_key=secret_key2,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user1,
            '--display-name', display_name1,
            '--email', email,
            '--access-key', access_key,
            '--secret', secret_key,
            '--max-buckets', '4'
            ],
            check_status=True)

    # TESTCASE 'duplicate email','user','create','existing user email','fails'
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user2,
            '--display-name', display_name2,
            '--email', email,
            ])
    assert err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # this whole block should only be run if regions have been configured
    if multi_region_run:
        rgw_utils.radosgw_agent_sync_all(ctx)
        # post-sync, validate that user1 exists on the sync destination host
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            dest_client = c_config['dest']
            (err, out) = rgwadmin(ctx, dest_client, ['metadata', 'list', 'user'])
            (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True)
            assert out['user_id'] == user1
            assert out['email'] == email
            assert out['display_name'] == display_name1
            assert len(out['keys']) == 1
            assert out['keys'][0]['access_key'] == access_key
            assert out['keys'][0]['secret_key'] == secret_key
            assert not out['suspended']

        # compare the metadata between different regions, make sure it matches
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(ctx, source_client,
                ['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True)
            (err2, out2) = rgwadmin(ctx, dest_client,
                ['metadata', 'get', 'user:{uid}'.format(uid=user1)], check_status=True)
            assert out1 == out2

        # suspend a user on the master, then check the status on the destination
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err, out) = rgwadmin(ctx, source_client, ['user', 'suspend', '--uid', user1])
            rgw_utils.radosgw_agent_sync_all(ctx)
            (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1], check_status=True)
            assert out['suspended']

        # delete a user on the master, then check that it's gone on the destination
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err, out) = rgwadmin(ctx, source_client, ['user', 'rm', '--uid', user1], check_status=True)
            rgw_utils.radosgw_agent_sync_all(ctx)
            (err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user1])
            assert out is None
            (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user1])
            assert out is None

            # then recreate it so later tests pass
            (err, out) = rgwadmin(ctx, client, [
                'user', 'create',
                '--uid', user1,
                '--display-name', display_name1,
                '--email', email,
                '--access-key', access_key,
                '--secret', secret_key,
                '--max-buckets', '4'
                ],
                check_status=True)

        # now do the multi-region bucket tests

        # Create a second user for the following tests
        (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user2,
            '--display-name', display_name2,
            '--email', email2,
            '--access-key', access_key2,
            '--secret', secret_key2,
            '--max-buckets', '4'
            ],
            check_status=True)
        (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user2], check_status=True)
        assert out is not None

        # create a bucket and do a sync
        bucket = connection.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(ctx, source_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            (err2, out2) = rgwadmin(ctx, dest_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            assert out1 == out2

            # get the bucket.instance info and compare that
            src_bucket_id = out1['data']['bucket']['bucket_id']
            dest_bucket_id = out2['data']['bucket']['bucket_id']
            (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get',
                'bucket.instance:{bucket_name}:{bucket_instance}'.format(
                bucket_name=bucket_name2,bucket_instance=src_bucket_id)],
                check_status=True)
            (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get',
                'bucket.instance:{bucket_name}:{bucket_instance}'.format(
                bucket_name=bucket_name2,bucket_instance=dest_bucket_id)],
                check_status=True)
            assert out1 == out2

        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']

            # Attempt to create a new connection with user1 to the destination RGW
            # and use that to attempt a delete (that should fail)
            exception_encountered = False
            try:
                (dest_remote_host, dest_remote_port) = ctx.rgw.role_endpoints[dest_client]
                connection_dest = boto.s3.connection.S3Connection(
                    aws_access_key_id=access_key,
                    aws_secret_access_key=secret_key,
                    is_secure=False,
                    port=dest_remote_port,
                    host=dest_remote_host,
                    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
                    )

                # this should fail
                connection_dest.delete_bucket(bucket_name2)
            except boto.exception.S3ResponseError as e:
                assert e.status == 301
                exception_encountered = True

            # confirm that the expected exception was seen
            assert exception_encountered

            # now delete the bucket on the source RGW and do another sync
            bucket.delete()
            rgw_utils.radosgw_agent_sync_all(ctx)

        # make sure that the bucket no longer exists in either region
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(ctx, source_client, ['metadata', 'get',
                'bucket:{bucket_name}'.format(bucket_name=bucket_name2)])
            (err2, out2) = rgwadmin(ctx, dest_client, ['metadata', 'get',
                'bucket:{bucket_name}'.format(bucket_name=bucket_name2)])
            # Both of the previous calls should have errors due to requesting
            # metadata for non-existent buckets
            assert err1
            assert err2

        # create a bucket and then sync it
        bucket = connection.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(ctx, source_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            (err2, out2) = rgwadmin(ctx, dest_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            assert out1 == out2

        # Now delete the bucket and recreate it with a different user
        # within the same window of time and then sync.
        bucket.delete()
        bucket = connection2.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        # user2 should own the bucket in both regions
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(ctx, source_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            (err2, out2) = rgwadmin(ctx, dest_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            assert out1 == out2
            assert out1['data']['owner'] == user2
            assert out1['data']['owner'] != user1

        # now we're going to use this bucket to test meta-data update propagation
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']

            # get the metadata so we can tweak it
            (err, orig_data) = rgwadmin(ctx, source_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)

            # manually edit mtime for this bucket to be 300 seconds in the past
            new_data = copy.deepcopy(orig_data)
            new_data['mtime'] =  orig_data['mtime'] - 300
            assert new_data != orig_data
            (err, out) = rgwadmin(ctx, source_client,
                ['metadata', 'put', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                stdin=StringIO(json.dumps(new_data)),
                check_status=True)

            # get the metadata and make sure that the 'put' worked
            (err, out) = rgwadmin(ctx, source_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            assert out == new_data

            # sync to propagate the new metadata
            rgw_utils.radosgw_agent_sync_all(ctx)

            # get the metadata from the dest and compare it to what we just set
            # and what the source region has.
            (err1, out1) = rgwadmin(ctx, source_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            (err2, out2) = rgwadmin(ctx, dest_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            # yeah for the transitive property
            assert out1 == out2
            assert out1 == new_data

        # now we delete the bucket
        bucket.delete()

        # Delete user2 as later tests do not expect it to exist.
        # Verify that it is gone on both regions
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err, out) = rgwadmin(ctx, source_client,
                ['user', 'rm', '--uid', user2], check_status=True)
            rgw_utils.radosgw_agent_sync_all(ctx)
            # The two 'user info' calls should fail and not return any data
            # since we just deleted this user.
            (err, out) = rgwadmin(ctx, source_client, ['user', 'info', '--uid', user2])
            assert out is None
            (err, out) = rgwadmin(ctx, dest_client, ['user', 'info', '--uid', user2])
            assert out is None

    # end of 'if multi_region_run:'

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
        check_status=True)

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True)

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'create', '--uid', user1,
            '--access-key', access_key2, '--secret', secret_key2,
            ], check_status=True)

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1],
        check_status=True)
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'rm', '--uid', user1,
            '--access-key', access_key2,
            ], check_status=True)
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    subuser_access = 'full'
    subuser_perm = 'full-control'

    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'create', '--subuser', subuser1,
            '--access', subuser_access
            ], check_status=True)

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'modify', '--subuser', subuser1,
            '--secret', swift_secret1,
            '--key-type', 'swift',
            ], check_status=True)

    # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])

    assert out['subusers'][0]['permissions'] == subuser_perm

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'create', '--subuser', subuser2,
            '--secret', swift_secret2,
            '--key-type', 'swift',
            ], check_status=True)

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'rm', '--subuser', subuser1,
            '--key-type', 'swift',
            ], check_status=True)
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'rm', '--subuser', subuser1,
            ], check_status=True)
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'rm', '--subuser', subuser2,
            '--key-type', 'swift', '--purge-keys',
            ], check_status=True)
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1],
        check_status=True)
    assert len(out) == 0

    if multi_region_run:
        rgw_utils.radosgw_agent_sync_all(ctx)

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
    assert len(out) >= 1
    assert bucket_name in out;

    # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
    bucket2 = connection.create_bucket(bucket_name + '2')
    bucket3 = connection.create_bucket(bucket_name + '3')
    bucket4 = connection.create_bucket(bucket_name + '4')
    # the 5th should fail.
    failed = False
    try:
        connection.create_bucket(bucket_name + '5')
    except Exception:
        failed = True
    assert failed

    # delete the buckets
    bucket2.delete()
    bucket3.delete()
    bucket4.delete()

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name], check_status=True)
    assert out['owner'] == user1
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True)
    assert len(out) == 1
    assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name], check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (err, out) = rgwadmin(ctx, client,
        ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
        check_status=True)

    # create a second user to link the bucket to
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user2,
            '--display-name', display_name2,
            '--access-key', access_key2,
            '--secret', secret_key2,
            '--max-buckets', '1',
            ],
            check_status=True)

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied

    # delete the object
    key.delete()

    # link the bucket to another user
    (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name],
        check_status=True)

    # try to remove user, should fail (has a linked bucket)
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
    assert err

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
        check_status=True)

    # relink the bucket to the first user and delete the second user
    (err, out) = rgwadmin(ctx, client,
        ['bucket', 'link', '--uid', user1, '--bucket', bucket_name],
        check_status=True)

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2],
        check_status=True)

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = 'four'
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)

    # now delete it
    (err, out) = rgwadmin(ctx, client,
        ['object', 'rm', '--bucket', bucket_name, '--object', object_name],
        check_status=True)

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name],
            check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        if obj[:4] == 'meta' or obj[:4] == 'data':
            continue

        (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj],
            check_status=True)
        assert len(log) > 0

        # exempt bucket_name2 from checking as it was only used for multi-region tests
        assert log['bucket'].find(bucket_name) == 0 or log['bucket'].find(bucket_name2) == 0
        assert log['bucket'] != bucket_name or log['bucket_id'] == bucket_id
        assert log['bucket_owner'] == user1 or log['bucket'] == bucket_name + '5' or log['bucket'] == bucket_name2
        for entry in log['log_entries']:
            assert entry['bucket'] == log['bucket']
            assert entry['user'] == user1 or log['bucket'] == bucket_name + '5' or log['bucket'] == bucket_name2

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj],
            check_status=True)

    # TODO: show log by bucket+date

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (20 * 60):      # wait up to 20 minutes
        (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj'])  # last operation we did is delete obj, wait for it to flush
        if successful_ops(out) > 0:
            break;
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    total = user_summary['total']
    assert total['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
        check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    for entry in user_summary['categories']:
        assert entry['successful_ops'] > 0
    assert user_summary['user'] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
    for cat in test_categories:
        (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat],
            check_status=True)
        assert len(out['summary']) > 0
        user_summary = out['summary'][0]
        assert user_summary['user'] == user1
        assert len(user_summary['categories']) == 1
        entry = user_summary['categories'][0]
        assert entry['category'] == cat
        assert entry['successful_ops'] > 0

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1],
        check_status=True)
    (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
        check_status=True)
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
        check_status=True)

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('five')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1],
        check_status=True)

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('six')

    # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = 'foo'*10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)

    # now delete the head
    big_key.delete()

    # wait a bit to give the garbage collector time to cycle
    time.sleep(15)

    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) > 0

    # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
    (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)

    #confirm
    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) == 0

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # delete should fail because ``key`` still exists
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409

    key.delete()
    bucket.delete()

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')

    # should be private already but guarantee it
    key.set_acl('private')

    (err, out) = rgwadmin(ctx, client,
        ['policy', '--bucket', bucket.name, '--object', key.key],
        check_status=True)

    acl = key.get_xml_acl()

    assert acl == out.strip('\n')

    # add another grantee by making the object public read
    key.set_acl('public-read')

    (err, out) = rgwadmin(ctx, client,
        ['policy', '--bucket', bucket.name, '--object', key.key],
        check_status=True)

    acl = key.get_xml_acl()
    assert acl == out.strip('\n')

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key_name = ['eight', 'nine', 'ten', 'eleven']
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])

    (err, out) = rgwadmin(ctx, client,
        ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
        check_status=True)

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps='user=read'
    (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps])

    assert out['caps'][0]['perm'] == 'read'

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps])

    assert not out['caps']

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('twelve')

    (err, out) = rgwadmin(ctx, client,
        ['user', 'rm', '--uid', user1, '--purge-data' ],
        check_status=True)

    # TESTCASE 'rm-user3','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
    #

    (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
    orig_placement_pools = len(out['placement_pools'])

    # removed this test, it is not correct to assume that zone has default placement, it really
    # depends on how we set it up before
    #
    # assert len(out) > 0
    # assert len(out['placement_pools']) == 1

    # default_rule = out['placement_pools'][0]
    # assert default_rule['key'] == 'default-placement'

    rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}

    out['placement_pools'].append(rule)

    (err, out) = rgwadmin(ctx, client, ['zone', 'set'],
        stdin=StringIO(json.dumps(out)),
        check_status=True)

    (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
    assert len(out) > 0
    assert len(out['placement_pools']) == orig_placement_pools + 1
  finally:
    count_ins += 1

f.close()
print("Successfully created %s instances" % (count_success_ins))

connection = boto.s3.connection.S3Connection(
          aws_access_key_id='adf938e8df4841129a5ca7089ab6b0ee',
          aws_secret_access_key='66f187d931f04af5805e899421bbfbef',
          port=8888,
          host='swift.rc.nectar.org.au',
          is_secure=True,
          validate_certs=False,
          calling_format=boto.s3.connection.OrdinaryCallingFormat()
        )

# buckets = connection.get_all_buckets()

container_name = "twitter_container"
try :
    b = connection.create_bucket(container_name)
except boto.exception.S3CreateError as e:
    if e.status == 409:
        print ("already exists the container: %s" % (container_name))
        pass 
except:
    print "Unexpected error:", sys.exc_info()[0]

b = connection.get_bucket('twitter_container', validate=False)
print ("Bucket: %s is ready" %(b.name))
Exemple #12
0
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    global log
    assert config is None or isinstance(config, list) \
        or isinstance(config, dict), \
        "task s3tests only supports a list or dictionary for configuration"
    all_clients = [
        'client.{id}'.format(id=id_)
        for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')
    ]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients = config.keys()

    multi_region_run = rgw_utils.multi_region_enabled(ctx)

    client = clients[0]
    # default choice, multi-region code may overwrite this
    if multi_region_run:
        client = rgw_utils.get_master_client(ctx, clients)

    # once the client is chosen, pull the host name and  assigned port out of
    # the role_endpoints that were assigned by the rgw task
    (remote_host, remote_port) = ctx.rgw.role_endpoints[client]

    ##
    user1 = 'foo'
    user2 = 'fud'
    subuser1 = 'foo:foo1'
    subuser2 = 'foo:foo2'
    display_name1 = 'Foo'
    display_name2 = 'Fud'
    email = '*****@*****.**'
    email2 = '*****@*****.**'
    access_key = '9te6NH5mcdcq0Tc5i8i1'
    secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2 = 'p5YnriCv1nAtykxBrupQ'
    secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name = 'myfoo'
    bucket_name2 = 'mybar'

    # connect to rgw
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )
    connection2 = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key2,
        aws_secret_access_key=secret_key2,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(
        ctx,
        client, [
            'user', 'create', '--uid', user1, '--display-name', display_name1,
            '--email', email, '--access-key', access_key, '--secret',
            secret_key, '--max-buckets', '4'
        ],
        check_status=True)

    # TESTCASE 'duplicate email','user','create','existing user email','fails'
    (err, out) = rgwadmin(ctx, client, [
        'user',
        'create',
        '--uid',
        user2,
        '--display-name',
        display_name2,
        '--email',
        email,
    ])
    assert err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # this whole block should only be run if regions have been configured
    if multi_region_run:
        rgw_utils.radosgw_agent_sync_all(ctx)
        # post-sync, validate that user1 exists on the sync destination host
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            dest_client = c_config['dest']
            (err, out) = rgwadmin(ctx, dest_client,
                                  ['metadata', 'list', 'user'])
            (err, out) = rgwadmin(ctx,
                                  dest_client,
                                  ['user', 'info', '--uid', user1],
                                  check_status=True)
            assert out['user_id'] == user1
            assert out['email'] == email
            assert out['display_name'] == display_name1
            assert len(out['keys']) == 1
            assert out['keys'][0]['access_key'] == access_key
            assert out['keys'][0]['secret_key'] == secret_key
            assert not out['suspended']

        # compare the metadata between different regions, make sure it matches
        log.debug(
            'compare the metadata between different regions, make sure it matches'
        )
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(
                ctx,
                source_client,
                ['metadata', 'get', 'user:{uid}'.format(uid=user1)],
                check_status=True)
            (err2, out2) = rgwadmin(
                ctx,
                dest_client,
                ['metadata', 'get', 'user:{uid}'.format(uid=user1)],
                check_status=True)
            assert out1 == out2

        # suspend a user on the master, then check the status on the destination
        log.debug(
            'suspend a user on the master, then check the status on the destination'
        )
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err, out) = rgwadmin(ctx, source_client,
                                  ['user', 'suspend', '--uid', user1])
            rgw_utils.radosgw_agent_sync_all(ctx)
            (err, out) = rgwadmin(ctx,
                                  dest_client,
                                  ['user', 'info', '--uid', user1],
                                  check_status=True)
            assert out['suspended']

        # delete a user on the master, then check that it's gone on the destination
        log.debug(
            'delete a user on the master, then check that it\'s gone on the destination'
        )
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err, out) = rgwadmin(ctx,
                                  source_client,
                                  ['user', 'rm', '--uid', user1],
                                  check_status=True)
            rgw_utils.radosgw_agent_sync_all(ctx)
            (err, out) = rgwadmin(ctx, source_client,
                                  ['user', 'info', '--uid', user1])
            assert out is None
            (err, out) = rgwadmin(ctx, dest_client,
                                  ['user', 'info', '--uid', user1])
            assert out is None

            # then recreate it so later tests pass
            (err, out) = rgwadmin(
                ctx,
                client, [
                    'user', 'create', '--uid', user1, '--display-name',
                    display_name1, '--email', email, '--access-key',
                    access_key, '--secret', secret_key, '--max-buckets', '4'
                ],
                check_status=True)

        # now do the multi-region bucket tests
        log.debug('now do the multi-region bucket tests')

        # Create a second user for the following tests
        log.debug('Create a second user for the following tests')
        (err, out) = rgwadmin(
            ctx,
            client, [
                'user', 'create', '--uid', user2, '--display-name',
                display_name2, '--email', email2, '--access-key', access_key2,
                '--secret', secret_key2, '--max-buckets', '4'
            ],
            check_status=True)
        (err, out) = rgwadmin(ctx,
                              client, ['user', 'info', '--uid', user2],
                              check_status=True)
        assert out is not None

        # create a bucket and do a sync
        log.debug('create a bucket and do a sync')
        bucket = connection.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        log.debug(
            'compare the metadata for the bucket between different regions, make sure it matches'
        )
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(
                ctx,
                source_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            (err2, out2) = rgwadmin(
                ctx,
                dest_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            assert out1 == out2

            # get the bucket.instance info and compare that
            src_bucket_id = out1['data']['bucket']['bucket_id']
            dest_bucket_id = out2['data']['bucket']['bucket_id']
            (err1, out1) = rgwadmin(
                ctx,
                source_client, [
                    'metadata', 'get',
                    'bucket.instance:{bucket_name}:{bucket_instance}'.format(
                        bucket_name=bucket_name2,
                        bucket_instance=src_bucket_id)
                ],
                check_status=True)
            (err2, out2) = rgwadmin(
                ctx,
                dest_client, [
                    'metadata', 'get',
                    'bucket.instance:{bucket_name}:{bucket_instance}'.format(
                        bucket_name=bucket_name2,
                        bucket_instance=dest_bucket_id)
                ],
                check_status=True)
            del out1['data']['bucket_info']['bucket']['pool']
            del out1['data']['bucket_info']['bucket']['index_pool']
            del out2['data']['bucket_info']['bucket']['pool']
            del out2['data']['bucket_info']['bucket']['index_pool']
            assert out1 == out2

        same_region = 0
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']

            source_region = rgw_utils.region_for_client(ctx, source_client)
            dest_region = rgw_utils.region_for_client(ctx, dest_client)

            # 301 is only returned for requests to something in a different region
            if source_region == dest_region:
                log.debug(
                    '301 is only returned for requests to something in a different region'
                )
                same_region += 1
                continue

            # Attempt to create a new connection with user1 to the destination RGW
            log.debug(
                'Attempt to create a new connection with user1 to the destination RGW'
            )
            # and use that to attempt a delete (that should fail)
            exception_encountered = False
            try:
                (dest_remote_host,
                 dest_remote_port) = ctx.rgw.role_endpoints[dest_client]
                connection_dest = boto.s3.connection.S3Connection(
                    aws_access_key_id=access_key,
                    aws_secret_access_key=secret_key,
                    is_secure=False,
                    port=dest_remote_port,
                    host=dest_remote_host,
                    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
                )

                # this should fail
                connection_dest.delete_bucket(bucket_name2)
            except boto.exception.S3ResponseError as e:
                assert e.status == 301
                exception_encountered = True

            # confirm that the expected exception was seen
            assert exception_encountered

            # now delete the bucket on the source RGW and do another sync
            log.debug(
                'now delete the bucket on the source RGW and do another sync')
            bucket.delete()
            rgw_utils.radosgw_agent_sync_all(ctx)

        if same_region == len(ctx.radosgw_agent.config):
            bucket.delete()
            rgw_utils.radosgw_agent_sync_all(ctx)

        # make sure that the bucket no longer exists in either region
        log.debug(
            'make sure that the bucket no longer exists in either region')
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(ctx, source_client, [
                'metadata', 'get',
                'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
            ])
            (err2, out2) = rgwadmin(ctx, dest_client, [
                'metadata', 'get',
                'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
            ])
            # Both of the previous calls should have errors due to requesting
            # metadata for non-existent buckets
            assert err1
            assert err2

        # create a bucket and then sync it
        log.debug('create a bucket and then sync it')
        bucket = connection.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        log.debug(
            'compare the metadata for the bucket between different regions, make sure it matches'
        )
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(
                ctx,
                source_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            (err2, out2) = rgwadmin(
                ctx,
                dest_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            assert out1 == out2

        # Now delete the bucket and recreate it with a different user
        log.debug(
            'Now delete the bucket and recreate it with a different user')
        # within the same window of time and then sync.
        bucket.delete()
        bucket = connection2.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        log.debug(
            'compare the metadata for the bucket between different regions, make sure it matches'
        )
        # user2 should own the bucket in both regions
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(
                ctx,
                source_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            (err2, out2) = rgwadmin(
                ctx,
                dest_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            assert out1 == out2
            assert out1['data']['owner'] == user2
            assert out1['data']['owner'] != user1

        # now we're going to use this bucket to test meta-data update propagation
        log.debug(
            'now we\'re going to use this bucket to test meta-data update propagation'
        )
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']

            # get the metadata so we can tweak it
            log.debug('get the metadata so we can tweak it')
            (err, orig_data) = rgwadmin(
                ctx,
                source_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)

            # manually edit mtime for this bucket to be 300 seconds in the past
            log.debug(
                'manually edit mtime for this bucket to be 300 seconds in the past'
            )
            new_data = copy.deepcopy(orig_data)
            new_data['mtime'] = orig_data['mtime'] - 300
            assert new_data != orig_data
            (err, out) = rgwadmin(
                ctx,
                source_client, [
                    'metadata', 'put',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                stdin=StringIO(json.dumps(new_data)),
                check_status=True)

            # get the metadata and make sure that the 'put' worked
            log.debug('get the metadata and make sure that the \'put\' worked')
            (err, out) = rgwadmin(
                ctx,
                source_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            assert out == new_data

            # sync to propagate the new metadata
            log.debug('sync to propagate the new metadata')
            rgw_utils.radosgw_agent_sync_all(ctx)

            # get the metadata from the dest and compare it to what we just set
            log.debug(
                'get the metadata from the dest and compare it to what we just set'
            )
            # and what the source region has.
            (err1, out1) = rgwadmin(
                ctx,
                source_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            (err2, out2) = rgwadmin(
                ctx,
                dest_client, [
                    'metadata', 'get',
                    'bucket:{bucket_name}'.format(bucket_name=bucket_name2)
                ],
                check_status=True)
            # yeah for the transitive property
            assert out1 == out2
            assert out1 == new_data

        # now we delete the bucket
        log.debug('now we delete the bucket')
        bucket.delete()

        log.debug('sync to propagate the deleted bucket')
        rgw_utils.radosgw_agent_sync_all(ctx)

        # Delete user2 as later tests do not expect it to exist.
        # Verify that it is gone on both regions
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err, out) = rgwadmin(ctx,
                                  source_client,
                                  ['user', 'rm', '--uid', user2],
                                  check_status=True)
            rgw_utils.radosgw_agent_sync_all(ctx)
            # The two 'user info' calls should fail and not return any data
            # since we just deleted this user.
            (err, out) = rgwadmin(ctx, source_client,
                                  ['user', 'info', '--uid', user2])
            assert out is None
            (err, out) = rgwadmin(ctx, dest_client,
                                  ['user', 'info', '--uid', user2])
            assert out is None

        # Test data sync

        # First create a bucket for data sync test purpose
        bucket = connection.create_bucket(bucket_name + 'data')

        # Create a tiny file and check if in sync
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            if c_config.get('metadata-only'):
                continue

            source_client = c_config['src']
            dest_client = c_config['dest']
            k = boto.s3.key.Key(bucket)
            k.key = 'tiny_file'
            k.set_contents_from_string("123456789")
            time.sleep(rgw_utils.radosgw_data_log_window(ctx, source_client))
            rgw_utils.radosgw_agent_sync_all(ctx, data=True)
            (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
            dest_connection = boto.s3.connection.S3Connection(
                aws_access_key_id=access_key,
                aws_secret_access_key=secret_key,
                is_secure=False,
                port=dest_port,
                host=dest_host,
                calling_format=boto.s3.connection.OrdinaryCallingFormat(),
            )
            dest_k = dest_connection.get_bucket(bucket_name +
                                                'data').get_key('tiny_file')
            assert k.get_contents_as_string() == dest_k.get_contents_as_string(
            )

            # check that deleting it removes it from the dest zone
            k.delete()
            time.sleep(rgw_utils.radosgw_data_log_window(ctx, source_client))
            rgw_utils.radosgw_agent_sync_all(ctx, data=True)

            dest_bucket = dest_connection.get_bucket(bucket_name + 'data')
            dest_k = dest_bucket.get_key('tiny_file')
            assert dest_k == None, 'object not deleted from destination zone'

        # finally we delete the bucket
        bucket.delete()

        bucket = connection.create_bucket(bucket_name + 'data2')
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            if c_config.get('metadata-only'):
                continue

            source_client = c_config['src']
            dest_client = c_config['dest']
            (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
            dest_connection = boto.s3.connection.S3Connection(
                aws_access_key_id=access_key,
                aws_secret_access_key=secret_key,
                is_secure=False,
                port=dest_port,
                host=dest_host,
                calling_format=boto.s3.connection.OrdinaryCallingFormat(),
            )
            for i in range(20):
                k = boto.s3.key.Key(bucket)
                k.key = 'tiny_file_' + str(i)
                k.set_contents_from_string(str(i) * 100)

            time.sleep(rgw_utils.radosgw_data_log_window(ctx, source_client))
            rgw_utils.radosgw_agent_sync_all(ctx, data=True)

            for i in range(20):
                dest_k = dest_connection.get_bucket(
                    bucket_name + 'data2').get_key('tiny_file_' + str(i))
                assert (str(i) * 100) == dest_k.get_contents_as_string()
                k = boto.s3.key.Key(bucket)
                k.key = 'tiny_file_' + str(i)
                k.delete()

            # check that deleting removes the objects from the dest zone
            time.sleep(rgw_utils.radosgw_data_log_window(ctx, source_client))
            rgw_utils.radosgw_agent_sync_all(ctx, data=True)

            for i in range(20):
                dest_bucket = dest_connection.get_bucket(bucket_name + 'data2')
                dest_k = dest_bucket.get_key('tiny_file_' + str(i))
                assert dest_k == None, 'object %d not deleted from destination zone' % i
        bucket.delete()

    # end of 'if multi_region_run:'

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'suspend', '--uid', user1],
                          check_status=True)

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'enable', '--uid', user1],
                          check_status=True)

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'create',
                              '--uid',
                              user1,
                              '--access-key',
                              access_key2,
                              '--secret',
                              secret_key2,
                          ],
                          check_status=True)

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][
        'access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][
        'secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'rm',
                              '--uid',
                              user1,
                              '--access-key',
                              access_key2,
                          ],
                          check_status=True)
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    subuser_access = 'full'
    subuser_perm = 'full-control'

    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser', 'create', '--subuser', subuser1,
                              '--access', subuser_access
                          ],
                          check_status=True)

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'modify',
                              '--subuser',
                              subuser1,
                              '--secret',
                              swift_secret1,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)

    # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])

    assert out['subusers'][0]['permissions'] == subuser_perm

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'create',
                              '--subuser',
                              subuser2,
                              '--secret',
                              swift_secret2,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][
        'user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[
        'swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'rm',
                              '--subuser',
                              subuser1,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'rm',
                              '--subuser',
                              subuser1,
                          ],
                          check_status=True)
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'rm',
                              '--subuser',
                              subuser2,
                              '--key-type',
                              'swift',
                              '--purge-keys',
                          ],
                          check_status=True)
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--uid', user1],
                          check_status=True)
    assert len(out) == 0

    if multi_region_run:
        rgw_utils.radosgw_agent_sync_all(ctx)

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'list', '--uid', user1],
                          check_status=True)
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'list', '--uid', user1],
                          check_status=True)
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
    assert len(out) >= 1
    assert bucket_name in out

    # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
    bucket2 = connection.create_bucket(bucket_name + '2')
    bucket3 = connection.create_bucket(bucket_name + '3')
    bucket4 = connection.create_bucket(bucket_name + '4')
    # the 5th should fail.
    failed = False
    try:
        connection.create_bucket(bucket_name + '5')
    except Exception:
        failed = True
    assert failed

    # delete the buckets
    bucket2.delete()
    bucket3.delete()
    bucket4.delete()

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['owner'] == user1
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--uid', user1],
                          check_status=True)
    assert len(out) == 1
    assert out[0][
        'id'] == bucket_id  # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
        check_status=True)

    # create a second user to link the bucket to
    (err, out) = rgwadmin(ctx,
                          client, [
                              'user',
                              'create',
                              '--uid',
                              user2,
                              '--display-name',
                              display_name2,
                              '--access-key',
                              access_key2,
                              '--secret',
                              secret_key2,
                              '--max-buckets',
                              '1',
                          ],
                          check_status=True)

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied

    # delete the object
    key.delete()

    # link the bucket to another user
    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name],
        check_status=True)

    # try to remove user, should fail (has a linked bucket)
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
    assert err

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
        check_status=True)

    # relink the bucket to the first user and delete the second user
    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'link', '--uid', user1, '--bucket', bucket_name],
        check_status=True)

    (err, out) = rgwadmin(ctx,
                          client, ['user', 'rm', '--uid', user2],
                          check_status=True)

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = 'four'
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)

    # now delete it
    (err, out) = rgwadmin(
        ctx,
        client,
        ['object', 'rm', '--bucket', bucket_name, '--object', object_name],
        check_status=True)

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        if obj[:4] == 'meta' or obj[:4] == 'data':
            continue

        (err, rgwlog) = rgwadmin(ctx,
                                 client, ['log', 'show', '--object', obj],
                                 check_status=True)
        assert len(rgwlog) > 0

        # exempt bucket_name2 from checking as it was only used for multi-region tests
        assert rgwlog['bucket'].find(
            bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
        assert rgwlog['bucket'] != bucket_name or rgwlog[
            'bucket_id'] == bucket_id
        assert rgwlog['bucket_owner'] == user1 or rgwlog[
            'bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
        for entry in rgwlog['log_entries']:
            log.debug('checking log entry: ', entry)
            assert entry['bucket'] == rgwlog['bucket']
            possible_buckets = [bucket_name + '5', bucket_name2]
            user = entry['user']
            assert user == user1 or user.endswith('system-user') or \
                rgwlog['bucket'] in possible_buckets

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx,
                              client, ['log', 'rm', '--object', obj],
                              check_status=True)

    # TODO: show log by bucket+date

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (20 * 60):  # wait up to 20 minutes
        (err, out) = rgwadmin(
            ctx, client,
            ['usage', 'show', '--categories', 'delete_obj'
             ])  # last operation we did is delete obj, wait for it to flush
        if successful_ops(out) > 0:
            break
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    total = user_summary['total']
    assert total['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'show', '--uid', user1],
                          check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    for entry in user_summary['categories']:
        assert entry['successful_ops'] > 0
    assert user_summary['user'] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = [
        'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'
    ]
    for cat in test_categories:
        (err, out) = rgwadmin(
            ctx,
            client, ['usage', 'show', '--uid', user1, '--categories', cat],
            check_status=True)
        assert len(out['summary']) > 0
        user_summary = out['summary'][0]
        assert user_summary['user'] == user1
        assert len(user_summary['categories']) == 1
        entry = user_summary['categories'][0]
        assert entry['category'] == cat
        assert entry['successful_ops'] > 0

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'trim', '--uid', user1],
                          check_status=True)
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'show', '--uid', user1],
                          check_status=True)
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'suspend', '--uid', user1],
                          check_status=True)

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('five')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'enable', '--uid', user1],
                          check_status=True)

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('six')

    # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = 'foo' * 10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)

    # now delete the head
    big_key.delete()

    # wait a bit to give the garbage collector time to cycle
    time.sleep(15)

    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) > 0

    # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
    (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)

    #confirm
    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) == 0

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # delete should fail because ``key`` still exists
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409

    key.delete()
    bucket.delete()

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')

    # should be private already but guarantee it
    key.set_acl('private')

    (err,
     out) = rgwadmin(ctx,
                     client,
                     ['policy', '--bucket', bucket.name, '--object', key.key],
                     check_status=True)

    acl = key.get_xml_acl()

    assert acl == out.strip('\n')

    # add another grantee by making the object public read
    key.set_acl('public-read')

    (err,
     out) = rgwadmin(ctx,
                     client,
                     ['policy', '--bucket', bucket.name, '--object', key.key],
                     check_status=True)

    acl = key.get_xml_acl()
    assert acl == out.strip('\n')

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key_name = ['eight', 'nine', 'ten', 'eleven']
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])

    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
        check_status=True)

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps = 'user=read'
    (err, out) = rgwadmin(ctx, client,
                          ['caps', 'add', '--uid', user1, '--caps', caps])

    assert out['caps'][0]['perm'] == 'read'

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (err, out) = rgwadmin(ctx, client,
                          ['caps', 'rm', '--uid', user1, '--caps', caps])

    assert not out['caps']

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('twelve')

    (err, out) = rgwadmin(ctx,
                          client,
                          ['user', 'rm', '--uid', user1, '--purge-data'],
                          check_status=True)

    # TESTCASE 'rm-user3','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
    #

    (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
    orig_placement_pools = len(out['placement_pools'])

    # removed this test, it is not correct to assume that zone has default placement, it really
    # depends on how we set it up before
    #
    # assert len(out) > 0
    # assert len(out['placement_pools']) == 1

    # default_rule = out['placement_pools'][0]
    # assert default_rule['key'] == 'default-placement'

    rule = {
        'key': 'new-placement',
        'val': {
            'data_pool': '.rgw.buckets.2',
            'index_pool': '.rgw.buckets.index.2'
        }
    }

    out['placement_pools'].append(rule)

    (err, out) = rgwadmin(ctx,
                          client, ['zone', 'set'],
                          stdin=StringIO(json.dumps(out)),
                          check_status=True)

    (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
    assert len(out) > 0
    assert len(out['placement_pools']) == orig_placement_pools + 1
Exemple #13
0
        (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user2,
            '--display-name', display_name2,
            '--email', email2,
            '--access-key', access_key2,
            '--secret', secret_key2,
            '--max-buckets', '4'
            ],
            check_status=True)
        (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user2], check_status=True)
        assert out is not None

        # create a bucket and do a sync
        log.debug('create a bucket and do a sync')
        bucket = connection.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        log.debug('compare the metadata for the bucket between different regions, make sure it matches')
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config['src']
            dest_client = c_config['dest']
            (err1, out1) = rgwadmin(ctx, source_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            (err2, out2) = rgwadmin(ctx, dest_client,
                ['metadata', 'get', 'bucket:{bucket_name}'.format(bucket_name=bucket_name2)],
                check_status=True)
            log.debug('metadata 1 %r', out1)
            log.debug('metadata 2 %r', out2)
Exemple #14
0
import argparse
import os

from boto.connection import AWSAuthConnection

s3_access_key=os.environ['S3_ACCESS_KEY_ID']
s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY']

parser = argparse.ArgumentParser(description='s3_tenant_test')
parser.add_argument('--port', type=int, action='store', default=8000 )
parser.add_argument('--key',type=str, action='store', default=s3_access_key)
parser.add_argument('--secret',type=str, action='store', default=s3_secret_key)
parser.add_argument('--tenant',type=str, action='store', default='testx')
parser.add_argument('--bucket',type=str, action='store', default='bucket1')
args = parser.parse_args()

print "connecting"
connection = boto.s3.connection.S3Connection(
    aws_access_key_id= args.key,
    aws_secret_access_key=args.secret,
    is_secure=False,
    port=args.port,
    host="localhost",
    calling_format = boto.s3.connection.OrdinaryCallingFormat())

print "bucket " + args.bucket + " tenant " + args.tenant
bucket = connection.create_bucket(args.tenant + ":" + args.bucket)
print bucket

bucket = connection.get_bucket(args.tenant + ":" + args.bucket)
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    global log
    assert (
        config is None or isinstance(config, list) or isinstance(config, dict)
    ), "task s3tests only supports a list or dictionary for configuration"
    all_clients = ["client.{id}".format(id=id_) for id_ in teuthology.all_roles_of_type(ctx.cluster, "client")]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients = config.keys()

    multi_region_run = rgw_utils.multi_region_enabled(ctx)

    client = clients[0]
    # default choice, multi-region code may overwrite this
    if multi_region_run:
        client = rgw_utils.get_master_client(ctx, clients)

    # once the client is chosen, pull the host name and  assigned port out of
    # the role_endpoints that were assigned by the rgw task
    (remote_host, remote_port) = ctx.rgw.role_endpoints[client]

    ##
    user1 = "foo"
    user2 = "fud"
    subuser1 = "foo:foo1"
    subuser2 = "foo:foo2"
    display_name1 = "Foo"
    display_name2 = "Fud"
    email = "*****@*****.**"
    email2 = "*****@*****.**"
    access_key = "9te6NH5mcdcq0Tc5i8i1"
    secret_key = "Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu"
    access_key2 = "p5YnriCv1nAtykxBrupQ"
    secret_key2 = "Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh"
    swift_secret1 = "gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL"
    swift_secret2 = "ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy"

    bucket_name = "myfoo"
    bucket_name2 = "mybar"

    # connect to rgw
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )
    connection2 = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key2,
        aws_secret_access_key=secret_key2,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(
        ctx,
        client,
        [
            "user",
            "create",
            "--uid",
            user1,
            "--display-name",
            display_name1,
            "--email",
            email,
            "--access-key",
            access_key,
            "--secret",
            secret_key,
            "--max-buckets",
            "4",
        ],
        check_status=True,
    )

    # TESTCASE 'duplicate email','user','create','existing user email','fails'
    (err, out) = rgwadmin(
        ctx, client, ["user", "create", "--uid", user2, "--display-name", display_name2, "--email", email]
    )
    assert err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
    assert out["user_id"] == user1
    assert out["email"] == email
    assert out["display_name"] == display_name1
    assert len(out["keys"]) == 1
    assert out["keys"][0]["access_key"] == access_key
    assert out["keys"][0]["secret_key"] == secret_key
    assert not out["suspended"]

    # this whole block should only be run if regions have been configured
    if multi_region_run:
        rgw_utils.radosgw_agent_sync_all(ctx)
        # post-sync, validate that user1 exists on the sync destination host
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            dest_client = c_config["dest"]
            (err, out) = rgwadmin(ctx, dest_client, ["metadata", "list", "user"])
            (err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1], check_status=True)
            assert out["user_id"] == user1
            assert out["email"] == email
            assert out["display_name"] == display_name1
            assert len(out["keys"]) == 1
            assert out["keys"][0]["access_key"] == access_key
            assert out["keys"][0]["secret_key"] == secret_key
            assert not out["suspended"]

        # compare the metadata between different regions, make sure it matches
        log.debug("compare the metadata between different regions, make sure it matches")
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]
            (err1, out1) = rgwadmin(
                ctx, source_client, ["metadata", "get", "user:{uid}".format(uid=user1)], check_status=True
            )
            (err2, out2) = rgwadmin(
                ctx, dest_client, ["metadata", "get", "user:{uid}".format(uid=user1)], check_status=True
            )
            assert out1 == out2

        # suspend a user on the master, then check the status on the destination
        log.debug("suspend a user on the master, then check the status on the destination")
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]
            (err, out) = rgwadmin(ctx, source_client, ["user", "suspend", "--uid", user1])
            rgw_utils.radosgw_agent_sync_all(ctx)
            (err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1], check_status=True)
            assert out["suspended"]

        # delete a user on the master, then check that it's gone on the destination
        log.debug("delete a user on the master, then check that it's gone on the destination")
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]
            (err, out) = rgwadmin(ctx, source_client, ["user", "rm", "--uid", user1], check_status=True)
            rgw_utils.radosgw_agent_sync_all(ctx)
            (err, out) = rgwadmin(ctx, source_client, ["user", "info", "--uid", user1])
            assert out is None
            (err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user1])
            assert out is None

            # then recreate it so later tests pass
            (err, out) = rgwadmin(
                ctx,
                client,
                [
                    "user",
                    "create",
                    "--uid",
                    user1,
                    "--display-name",
                    display_name1,
                    "--email",
                    email,
                    "--access-key",
                    access_key,
                    "--secret",
                    secret_key,
                    "--max-buckets",
                    "4",
                ],
                check_status=True,
            )

        # now do the multi-region bucket tests
        log.debug("now do the multi-region bucket tests")

        # Create a second user for the following tests
        log.debug("Create a second user for the following tests")
        (err, out) = rgwadmin(
            ctx,
            client,
            [
                "user",
                "create",
                "--uid",
                user2,
                "--display-name",
                display_name2,
                "--email",
                email2,
                "--access-key",
                access_key2,
                "--secret",
                secret_key2,
                "--max-buckets",
                "4",
            ],
            check_status=True,
        )
        (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user2], check_status=True)
        assert out is not None

        # create a bucket and do a sync
        log.debug("create a bucket and do a sync")
        bucket = connection.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        log.debug("compare the metadata for the bucket between different regions, make sure it matches")
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]
            (err1, out1) = rgwadmin(
                ctx,
                source_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            (err2, out2) = rgwadmin(
                ctx,
                dest_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            assert out1 == out2

            # get the bucket.instance info and compare that
            src_bucket_id = out1["data"]["bucket"]["bucket_id"]
            dest_bucket_id = out2["data"]["bucket"]["bucket_id"]
            (err1, out1) = rgwadmin(
                ctx,
                source_client,
                [
                    "metadata",
                    "get",
                    "bucket.instance:{bucket_name}:{bucket_instance}".format(
                        bucket_name=bucket_name2, bucket_instance=src_bucket_id
                    ),
                ],
                check_status=True,
            )
            (err2, out2) = rgwadmin(
                ctx,
                dest_client,
                [
                    "metadata",
                    "get",
                    "bucket.instance:{bucket_name}:{bucket_instance}".format(
                        bucket_name=bucket_name2, bucket_instance=dest_bucket_id
                    ),
                ],
                check_status=True,
            )
            del out1["data"]["bucket_info"]["bucket"]["pool"]
            del out1["data"]["bucket_info"]["bucket"]["index_pool"]
            del out2["data"]["bucket_info"]["bucket"]["pool"]
            del out2["data"]["bucket_info"]["bucket"]["index_pool"]
            assert out1 == out2

        same_region = 0
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]

            source_region = rgw_utils.region_for_client(ctx, source_client)
            dest_region = rgw_utils.region_for_client(ctx, dest_client)

            # 301 is only returned for requests to something in a different region
            if source_region == dest_region:
                log.debug("301 is only returned for requests to something in a different region")
                same_region += 1
                continue

            # Attempt to create a new connection with user1 to the destination RGW
            log.debug("Attempt to create a new connection with user1 to the destination RGW")
            # and use that to attempt a delete (that should fail)
            exception_encountered = False
            try:
                (dest_remote_host, dest_remote_port) = ctx.rgw.role_endpoints[dest_client]
                connection_dest = boto.s3.connection.S3Connection(
                    aws_access_key_id=access_key,
                    aws_secret_access_key=secret_key,
                    is_secure=False,
                    port=dest_remote_port,
                    host=dest_remote_host,
                    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
                )

                # this should fail
                connection_dest.delete_bucket(bucket_name2)
            except boto.exception.S3ResponseError as e:
                assert e.status == 301
                exception_encountered = True

            # confirm that the expected exception was seen
            assert exception_encountered

            # now delete the bucket on the source RGW and do another sync
            log.debug("now delete the bucket on the source RGW and do another sync")
            bucket.delete()
            rgw_utils.radosgw_agent_sync_all(ctx)

        if same_region == len(ctx.radosgw_agent.config):
            bucket.delete()
            rgw_utils.radosgw_agent_sync_all(ctx)

        # make sure that the bucket no longer exists in either region
        log.debug("make sure that the bucket no longer exists in either region")
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]
            (err1, out1) = rgwadmin(
                ctx, source_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)]
            )
            (err2, out2) = rgwadmin(
                ctx, dest_client, ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)]
            )
            # Both of the previous calls should have errors due to requesting
            # metadata for non-existent buckets
            assert err1
            assert err2

        # create a bucket and then sync it
        log.debug("create a bucket and then sync it")
        bucket = connection.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        log.debug("compare the metadata for the bucket between different regions, make sure it matches")
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]
            (err1, out1) = rgwadmin(
                ctx,
                source_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            (err2, out2) = rgwadmin(
                ctx,
                dest_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            assert out1 == out2

        # Now delete the bucket and recreate it with a different user
        log.debug("Now delete the bucket and recreate it with a different user")
        # within the same window of time and then sync.
        bucket.delete()
        bucket = connection2.create_bucket(bucket_name2)
        rgw_utils.radosgw_agent_sync_all(ctx)

        # compare the metadata for the bucket between different regions, make sure it matches
        log.debug("compare the metadata for the bucket between different regions, make sure it matches")
        # user2 should own the bucket in both regions
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]
            (err1, out1) = rgwadmin(
                ctx,
                source_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            (err2, out2) = rgwadmin(
                ctx,
                dest_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            assert out1 == out2
            assert out1["data"]["owner"] == user2
            assert out1["data"]["owner"] != user1

        # now we're going to use this bucket to test meta-data update propagation
        log.debug("now we're going to use this bucket to test meta-data update propagation")
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]

            # get the metadata so we can tweak it
            log.debug("get the metadata so we can tweak it")
            (err, orig_data) = rgwadmin(
                ctx,
                source_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )

            # manually edit mtime for this bucket to be 300 seconds in the past
            log.debug("manually edit mtime for this bucket to be 300 seconds in the past")
            new_data = copy.deepcopy(orig_data)
            new_data["mtime"] = orig_data["mtime"] - 300
            assert new_data != orig_data
            (err, out) = rgwadmin(
                ctx,
                source_client,
                ["metadata", "put", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                stdin=StringIO(json.dumps(new_data)),
                check_status=True,
            )

            # get the metadata and make sure that the 'put' worked
            log.debug("get the metadata and make sure that the 'put' worked")
            (err, out) = rgwadmin(
                ctx,
                source_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            assert out == new_data

            # sync to propagate the new metadata
            log.debug("sync to propagate the new metadata")
            rgw_utils.radosgw_agent_sync_all(ctx)

            # get the metadata from the dest and compare it to what we just set
            log.debug("get the metadata from the dest and compare it to what we just set")
            # and what the source region has.
            (err1, out1) = rgwadmin(
                ctx,
                source_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            (err2, out2) = rgwadmin(
                ctx,
                dest_client,
                ["metadata", "get", "bucket:{bucket_name}".format(bucket_name=bucket_name2)],
                check_status=True,
            )
            # yeah for the transitive property
            assert out1 == out2
            assert out1 == new_data

        # now we delete the bucket
        log.debug("now we delete the bucket")
        bucket.delete()

        log.debug("sync to propagate the deleted bucket")
        rgw_utils.radosgw_agent_sync_all(ctx)

        # Delete user2 as later tests do not expect it to exist.
        # Verify that it is gone on both regions
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            source_client = c_config["src"]
            dest_client = c_config["dest"]
            (err, out) = rgwadmin(ctx, source_client, ["user", "rm", "--uid", user2], check_status=True)
            rgw_utils.radosgw_agent_sync_all(ctx)
            # The two 'user info' calls should fail and not return any data
            # since we just deleted this user.
            (err, out) = rgwadmin(ctx, source_client, ["user", "info", "--uid", user2])
            assert out is None
            (err, out) = rgwadmin(ctx, dest_client, ["user", "info", "--uid", user2])
            assert out is None

        # Test data sync

        # First create a bucket for data sync test purpose
        bucket = connection.create_bucket(bucket_name + "data")

        # Create a tiny file and check if in sync
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            if c_config.get("metadata-only"):
                continue

            for full in (True, False):
                source_client = c_config["src"]
                dest_client = c_config["dest"]
                k = boto.s3.key.Key(bucket)
                k.key = "tiny_file"
                k.set_contents_from_string("123456789")
                safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client)
                time.sleep(safety_window)
                rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)
                (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
                dest_connection = boto.s3.connection.S3Connection(
                    aws_access_key_id=access_key,
                    aws_secret_access_key=secret_key,
                    is_secure=False,
                    port=dest_port,
                    host=dest_host,
                    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
                )
                dest_k = dest_connection.get_bucket(bucket_name + "data").get_key("tiny_file")
                assert k.get_contents_as_string() == dest_k.get_contents_as_string()

                # check that deleting it removes it from the dest zone
                k.delete()
                time.sleep(safety_window)
                # full sync doesn't handle deleted objects yet
                rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)

                dest_bucket = dest_connection.get_bucket(bucket_name + "data")
                dest_k = dest_bucket.get_key("tiny_file")
                assert dest_k == None, "object not deleted from destination zone"

        # finally we delete the bucket
        bucket.delete()

        bucket = connection.create_bucket(bucket_name + "data2")
        for agent_client, c_config in ctx.radosgw_agent.config.iteritems():
            if c_config.get("metadata-only"):
                continue

            for full in (True, False):
                source_client = c_config["src"]
                dest_client = c_config["dest"]
                (dest_host, dest_port) = ctx.rgw.role_endpoints[dest_client]
                dest_connection = boto.s3.connection.S3Connection(
                    aws_access_key_id=access_key,
                    aws_secret_access_key=secret_key,
                    is_secure=False,
                    port=dest_port,
                    host=dest_host,
                    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
                )
                for i in range(20):
                    k = boto.s3.key.Key(bucket)
                    k.key = "tiny_file_" + str(i)
                    k.set_contents_from_string(str(i) * 100)

                safety_window = rgw_utils.radosgw_data_log_window(ctx, source_client)
                time.sleep(safety_window)
                rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=full)

                for i in range(20):
                    dest_k = dest_connection.get_bucket(bucket_name + "data2").get_key("tiny_file_" + str(i))
                    assert (str(i) * 100) == dest_k.get_contents_as_string()
                    k = boto.s3.key.Key(bucket)
                    k.key = "tiny_file_" + str(i)
                    k.delete()

                # check that deleting removes the objects from the dest zone
                time.sleep(safety_window)
                # full sync doesn't delete deleted objects yet
                rgw_utils.radosgw_agent_sync_all(ctx, data=True, full=False)

                for i in range(20):
                    dest_bucket = dest_connection.get_bucket(bucket_name + "data2")
                    dest_k = dest_bucket.get_key("tiny_file_" + str(i))
                    assert dest_k == None, "object %d not deleted from destination zone" % i
        bucket.delete()

    # end of 'if multi_region_run:'

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx, client, ["user", "suspend", "--uid", user1], check_status=True)

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
    assert out["suspended"]

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ["user", "enable", "--uid", user1], check_status=True)

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
    assert not out["suspended"]

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(
        ctx,
        client,
        ["key", "create", "--uid", user1, "--access-key", access_key2, "--secret", secret_key2],
        check_status=True,
    )

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
    assert len(out["keys"]) == 2
    assert out["keys"][0]["access_key"] == access_key2 or out["keys"][1]["access_key"] == access_key2
    assert out["keys"][0]["secret_key"] == secret_key2 or out["keys"][1]["secret_key"] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx, client, ["key", "rm", "--uid", user1, "--access-key", access_key2], check_status=True)
    assert len(out["keys"]) == 1
    assert out["keys"][0]["access_key"] == access_key
    assert out["keys"][0]["secret_key"] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    subuser_access = "full"
    subuser_perm = "full-control"

    (err, out) = rgwadmin(
        ctx, client, ["subuser", "create", "--subuser", subuser1, "--access", subuser_access], check_status=True
    )

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(
        ctx,
        client,
        ["subuser", "modify", "--subuser", subuser1, "--secret", swift_secret1, "--key-type", "swift"],
        check_status=True,
    )

    # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1])

    assert out["subusers"][0]["permissions"] == subuser_perm

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
    assert len(out["swift_keys"]) == 1
    assert out["swift_keys"][0]["user"] == subuser1
    assert out["swift_keys"][0]["secret_key"] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(
        ctx,
        client,
        ["subuser", "create", "--subuser", subuser2, "--secret", swift_secret2, "--key-type", "swift"],
        check_status=True,
    )

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1], check_status=True)
    assert len(out["swift_keys"]) == 2
    assert out["swift_keys"][0]["user"] == subuser2 or out["swift_keys"][1]["user"] == subuser2
    assert out["swift_keys"][0]["secret_key"] == swift_secret2 or out["swift_keys"][1]["secret_key"] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx, client, ["key", "rm", "--subuser", subuser1, "--key-type", "swift"], check_status=True)
    assert len(out["swift_keys"]) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (err, out) = rgwadmin(ctx, client, ["subuser", "rm", "--subuser", subuser1], check_status=True)
    assert len(out["subusers"]) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (err, out) = rgwadmin(
        ctx, client, ["subuser", "rm", "--subuser", subuser2, "--key-type", "swift", "--purge-keys"], check_status=True
    )
    assert len(out["swift_keys"]) == 0
    assert len(out["subusers"]) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--uid", user1], check_status=True)
    assert len(out) == 0

    if multi_region_run:
        rgw_utils.radosgw_agent_sync_all(ctx)

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ["bucket", "list", "--uid", user1], check_status=True)
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ["bucket", "list", "--uid", user1], check_status=True)
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ["bucket", "list"], check_status=True)
    assert len(out) >= 1
    assert bucket_name in out

    # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
    bucket2 = connection.create_bucket(bucket_name + "2")
    bucket3 = connection.create_bucket(bucket_name + "3")
    bucket4 = connection.create_bucket(bucket_name + "4")
    # the 5th should fail.
    failed = False
    try:
        connection.create_bucket(bucket_name + "5")
    except Exception:
        failed = True
    assert failed

    # delete the buckets
    bucket2.delete()
    bucket3.delete()
    bucket4.delete()

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True)
    assert out["owner"] == user1
    bucket_id = out["id"]

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--uid", user1], check_status=True)
    assert len(out) == 1
    assert out[0]["id"] == bucket_id  # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string("one")

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True)
    assert out["id"] == bucket_id
    assert out["usage"]["rgw.main"]["num_objects"] == 1
    assert out["usage"]["rgw.main"]["size_kb"] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (err, out) = rgwadmin(ctx, client, ["bucket", "unlink", "--uid", user1, "--bucket", bucket_name], check_status=True)

    # create a second user to link the bucket to
    (err, out) = rgwadmin(
        ctx,
        client,
        [
            "user",
            "create",
            "--uid",
            user2,
            "--display-name",
            display_name2,
            "--access-key",
            access_key2,
            "--secret",
            secret_key2,
            "--max-buckets",
            "1",
        ],
        check_status=True,
    )

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string("two")
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied

    # delete the object
    key.delete()

    # link the bucket to another user
    (err, out) = rgwadmin(ctx, client, ["metadata", "get", "bucket:{n}".format(n=bucket_name)], check_status=True)

    bucket_data = out["data"]
    assert bucket_data["bucket"]["name"] == bucket_name

    bucket_id = bucket_data["bucket"]["bucket_id"]

    # link the bucket to another user
    (err, out) = rgwadmin(
        ctx,
        client,
        ["bucket", "link", "--uid", user2, "--bucket", bucket_name, "--bucket-id", bucket_id],
        check_status=True,
    )

    # try to remove user, should fail (has a linked bucket)
    (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user2])
    assert err

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
    (err, out) = rgwadmin(ctx, client, ["bucket", "unlink", "--uid", user2, "--bucket", bucket_name], check_status=True)

    # relink the bucket to the first user and delete the second user
    (err, out) = rgwadmin(
        ctx,
        client,
        ["bucket", "link", "--uid", user1, "--bucket", bucket_name, "--bucket-id", bucket_id],
        check_status=True,
    )

    (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user2], check_status=True)

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = "four"
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)

    # now delete it
    (err, out) = rgwadmin(
        ctx, client, ["object", "rm", "--bucket", bucket_name, "--object", object_name], check_status=True
    )

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ["bucket", "stats", "--bucket", bucket_name], check_status=True)
    assert out["id"] == bucket_id
    assert out["usage"]["rgw.main"]["num_objects"] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ["log", "list"], check_status=True)
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        if obj[:4] == "meta" or obj[:4] == "data":
            continue

        (err, rgwlog) = rgwadmin(ctx, client, ["log", "show", "--object", obj], check_status=True)
        assert len(rgwlog) > 0

        # exempt bucket_name2 from checking as it was only used for multi-region tests
        assert rgwlog["bucket"].find(bucket_name) == 0 or rgwlog["bucket"].find(bucket_name2) == 0
        assert rgwlog["bucket"] != bucket_name or rgwlog["bucket_id"] == bucket_id
        assert (
            rgwlog["bucket_owner"] == user1 or rgwlog["bucket"] == bucket_name + "5" or rgwlog["bucket"] == bucket_name2
        )
        for entry in rgwlog["log_entries"]:
            log.debug("checking log entry: ", entry)
            assert entry["bucket"] == rgwlog["bucket"]
            possible_buckets = [bucket_name + "5", bucket_name2]
            user = entry["user"]
            assert user == user1 or user.endswith("system-user") or rgwlog["bucket"] in possible_buckets

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx, client, ["log", "rm", "--object", obj], check_status=True)

    # TODO: show log by bucket+date

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (20 * 60):  # wait up to 20 minutes
        (err, out) = rgwadmin(
            ctx, client, ["usage", "show", "--categories", "delete_obj"]
        )  # last operation we did is delete obj, wait for it to flush
        if successful_ops(out) > 0:
            break
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ["usage", "show"], check_status=True)
    assert len(out["entries"]) > 0
    assert len(out["summary"]) > 0
    user_summary = out["summary"][0]
    total = user_summary["total"]
    assert total["successful_ops"] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1], check_status=True)
    assert len(out["entries"]) > 0
    assert len(out["summary"]) > 0
    user_summary = out["summary"][0]
    for entry in user_summary["categories"]:
        assert entry["successful_ops"] > 0
    assert user_summary["user"] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = ["create_bucket", "put_obj", "delete_obj", "delete_bucket"]
    for cat in test_categories:
        (err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1, "--categories", cat], check_status=True)
        assert len(out["summary"]) > 0
        user_summary = out["summary"][0]
        assert user_summary["user"] == user1
        assert len(user_summary["categories"]) == 1
        entry = user_summary["categories"][0]
        assert entry["category"] == cat
        assert entry["successful_ops"] > 0

    # the usage flush interval is 30 seconds, wait that much an then some
    # to make sure everything has been flushed
    time.sleep(35)

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (err, out) = rgwadmin(ctx, client, ["usage", "trim", "--uid", user1], check_status=True)
    (err, out) = rgwadmin(ctx, client, ["usage", "show", "--uid", user1], check_status=True)
    assert len(out["entries"]) == 0
    assert len(out["summary"]) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx, client, ["user", "suspend", "--uid", user1], check_status=True)

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string("five")
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ["user", "enable", "--uid", user1], check_status=True)

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string("six")

    # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = "foo" * 10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)

    # now delete the head
    big_key.delete()

    # wait a bit to give the garbage collector time to cycle
    time.sleep(15)

    (err, out) = rgwadmin(ctx, client, ["gc", "list"])

    assert len(out) > 0

    # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
    (err, out) = rgwadmin(ctx, client, ["gc", "process"], check_status=True)

    # confirm
    (err, out) = rgwadmin(ctx, client, ["gc", "list"])

    assert len(out) == 0

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1])
    assert err

    # delete should fail because ``key`` still exists
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409

    key.delete()
    bucket.delete()

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string("seven")

    # should be private already but guarantee it
    key.set_acl("private")

    (err, out) = rgwadmin(ctx, client, ["policy", "--bucket", bucket.name, "--object", key.key], check_status=True)

    acl = key.get_xml_acl()

    assert acl == out.strip("\n")

    # add another grantee by making the object public read
    key.set_acl("public-read")

    (err, out) = rgwadmin(ctx, client, ["policy", "--bucket", bucket.name, "--object", key.key], check_status=True)

    acl = key.get_xml_acl()
    assert acl == out.strip("\n")

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key_name = ["eight", "nine", "ten", "eleven"]
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])

    (err, out) = rgwadmin(ctx, client, ["bucket", "rm", "--bucket", bucket_name, "--purge-objects"], check_status=True)

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps = "user=read"
    (err, out) = rgwadmin(ctx, client, ["caps", "add", "--uid", user1, "--caps", caps])

    assert out["caps"][0]["perm"] == "read"

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ["caps", "rm", "--uid", user1, "--caps", caps])

    assert not out["caps"]

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)

    (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1])
    assert err

    # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string("twelve")

    (err, out) = rgwadmin(ctx, client, ["user", "rm", "--uid", user1, "--purge-data"], check_status=True)

    # TESTCASE 'rm-user3','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ["user", "info", "--uid", user1])
    assert err

    # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
    #

    (err, out) = rgwadmin(ctx, client, ["zone", "get"])
    orig_placement_pools = len(out["placement_pools"])

    # removed this test, it is not correct to assume that zone has default placement, it really
    # depends on how we set it up before
    #
    # assert len(out) > 0
    # assert len(out['placement_pools']) == 1

    # default_rule = out['placement_pools'][0]
    # assert default_rule['key'] == 'default-placement'

    rule = {"key": "new-placement", "val": {"data_pool": ".rgw.buckets.2", "index_pool": ".rgw.buckets.index.2"}}

    out["placement_pools"].append(rule)

    (err, out) = rgwadmin(ctx, client, ["zone", "set"], stdin=StringIO(json.dumps(out)), check_status=True)

    (err, out) = rgwadmin(ctx, client, ["zone", "get"])
    assert len(out) > 0
    assert len(out["placement_pools"]) == orig_placement_pools + 1
def task(ctx, config):
    """
    Test radosgw-admin functionality through the RESTful interface
    """
    assert config is None or isinstance(config, list) \
        or isinstance(config, dict), \
        "task s3tests only supports a list or dictionary for configuration"
    all_clients = ['client.{id}'.format(id=id_)
                   for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients = config.keys()

    # just use the first client...
    client = clients[0]

    ##
    admin_user = '******'
    admin_display_name = 'Ms. Admin User'
    admin_access_key = 'MH1WC2XQ1S8UISFDZC8W'
    admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
    admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write'

    user1 = 'foo'
    user2 = 'fud'
    subuser1 = 'foo:foo1'
    subuser2 = 'foo:foo2'
    display_name1 = 'Foo'
    display_name2 = 'Fud'
    email = '*****@*****.**'
    access_key = '9te6NH5mcdcq0Tc5i8i1'
    secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2 = 'p5YnriCv1nAtykxBrupQ'
    secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name = 'myfoo'

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'create-admin-user','user','create','administrative user','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', admin_user,
            '--display-name', admin_display_name,
            '--access-key', admin_access_key,
            '--secret', admin_secret_key,
            '--max-buckets', '0',
            '--caps', admin_caps
            ])
    logging.error(out)
    logging.error(err)
    assert not err

    (remote,) = ctx.cluster.only(client).remotes.iterkeys()
    remote_host = remote.name.split('@')[1]
    admin_conn = boto.s3.connection.S3Connection(
        aws_access_key_id=admin_access_key,
        aws_secret_access_key=admin_secret_key,
        is_secure=False,
        port=7280,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1})
    assert ret == 404

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['user', 'create'],
            {'uid' : user1,
             'display-name' :  display_name1,
             'email' : email,
             'access-key' : access_key,
             'secret-key' : secret_key,
             'max-buckets' : '4'
            })

    assert ret == 200

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})

    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
    assert ret == 200

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
    assert ret == 200
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : 'false'})
    assert not err

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
    assert ret == 200
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['key', 'create'],
            {'uid' : user1,
             'access-key' : access_key2,
             'secret-key' : secret_key2
            })


    assert ret == 200

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
    assert ret == 200
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['key', 'rm'],
            {'uid' : user1,
             'access-key' : access_key2
            })

    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})

    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['subuser', 'create'],
            {'subuser' : subuser1,
             'secret-key' : swift_secret1,
             'key-type' : 'swift'
            })

    assert ret == 200

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' : user1})
    assert ret == 200
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['subuser', 'create'],
            {'subuser' : subuser2,
             'secret-key' : swift_secret2,
             'key-type' : 'swift'
            })

    assert ret == 200

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
    assert ret == 200
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['key', 'rm'],
            {'subuser' : subuser1,
             'key-type' :'swift'
            })

    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['subuser', 'rm'],
            {'subuser' : subuser1
            })

    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['subuser', 'rm'],
            {'subuser' : subuser2,
             'key-type' : 'swift',
             '{purge-keys' :True
            })

    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' :  user1})
    assert ret == 200
    assert len(out) == 0

    # connect to rgw
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=7280,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
    assert ret == 200
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1})
    assert ret == 200
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (ret, out) = rgwadmin_rest(admin_conn,
            ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})

    assert ret == 200
    assert out['owner'] == user1
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid' : user1, 'stats' : True})
    assert ret == 200
    assert len(out) == 1
    assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
    assert ret == 200
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], {'uid' : user1, 'bucket' : bucket_name})

    assert ret == 200

    # create a second user to link the bucket to
    (ret, out) = rgwadmin_rest(admin_conn,
            ['user', 'create'],
            {'uid' : user2,
            'display-name' :  display_name2,
            'access-key' : access_key2,
            'secret-key' : secret_key2,
            'max-buckets' : '1',
            })

    assert ret == 200

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied

    # delete the object
    key.delete()

    # link the bucket to another user
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {'uid' : user2, 'bucket' : bucket_name})

    assert ret == 200

    # try creating an object with the first user which should cause an error
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('three')
    except boto.exception.S3ResponseError:
        denied = True

    assert denied

    # relink the bucket to the first user and delete the second user
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {'uid' : user1, 'bucket' : bucket_name})
    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user2})
    assert ret == 200

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = 'four'
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)

    # now delete it
    (ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {'bucket' : bucket_name, 'object' : object_name})
    assert ret == 200

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'bucket' : bucket_name, 'stats' : True})
    assert ret == 200
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # create a bucket for deletion stats
    useless_bucket = connection.create_bucket('useless_bucket')
    useless_key = useless_bucket.new_key('useless_key')
    useless_key.set_contents_from_string('useless string')

    # delete it
    useless_key.delete()
    useless_bucket.delete()

    # wait for the statistics to flush
    time.sleep(60)

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (20 * 60):      # wait up to 20 minutes
        (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'categories' : 'delete_obj'})  # last operation we did is delete obj, wait for it to flush

        if successful_ops(out) > 0:
            break
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'])
    assert ret == 200
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    total = user_summary['total']
    assert total['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
    assert ret == 200
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    for entry in user_summary['categories']:
        assert entry['successful_ops'] > 0
    assert user_summary['user'] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
    for cat in test_categories:
        (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1, 'categories' : cat})
        assert ret == 200
        assert len(out['summary']) > 0
        user_summary = out['summary'][0]
        assert user_summary['user'] == user1
        assert len(user_summary['categories']) == 1
        entry = user_summary['categories'][0]
        assert entry['category'] == cat
        assert entry['successful_ops'] > 0

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid' : user1})
    assert ret == 200
    (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid' : user1})
    assert ret == 200
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' : user1, 'suspended' : True})
    assert ret == 200

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('five')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {'uid' :  user1, 'suspended' : 'false'})
    assert ret == 200

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('six')

    # TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = 'foo'*10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)

    # now delete the head
    big_key.delete()

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
    assert ret == 409

    # delete should fail because ``key`` still exists
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409

    key.delete()
    bucket.delete()

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')

    # should be private already but guarantee it
    key.set_acl('private')

    (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
    assert ret == 200

    acl = key.get_xml_acl()
    assert acl == out.strip('\n')

    # add another grantee by making the object public read
    key.set_acl('public-read')

    (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {'bucket' : bucket.name, 'object' : key.key})
    assert ret == 200

    acl = key.get_xml_acl()
    assert acl == out.strip('\n')

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key_name = ['eight', 'nine', 'ten', 'eleven']
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])

    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], {'bucket' : bucket_name, 'purge-objects' : True})
    assert ret == 200

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps = 'usage=read'
    (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {'uid' :  user1, 'user-caps' : caps})
    assert ret == 200
    assert out[0]['perm'] == 'read'

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {'uid' :  user1, 'user-caps' : caps})
    assert ret == 200
    assert not out

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1})
    assert ret == 409

    # TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('twelve')

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid' : user1, 'purge-data' : True})
    assert ret == 200

    # TESTCASE 'rm-user3','user','info','deleted user','fails'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid' :  user1})
    assert ret == 404
Exemple #17
0
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    global log

    assert ctx.rgw.config, \
        "radosgw_admin task needs a config passed from the rgw task"
    config = ctx.rgw.config
    log.debug('config is: %r', config)

    clients_from_config = config.keys()

    # choose first client as default
    client = clients_from_config[0]

    # once the client is chosen, pull the host name and  assigned port out of
    # the role_endpoints that were assigned by the rgw task
    (remote_host, remote_port) = ctx.rgw.role_endpoints[client]

    ##
    user1='foo'
    user2='fud'
    subuser1='foo:foo1'
    subuser2='foo:foo2'
    display_name1='Foo'
    display_name2='Fud'
    email='*****@*****.**'
    email2='*****@*****.**'
    access_key='9te6NH5mcdcq0Tc5i8i1'
    secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2='p5YnriCv1nAtykxBrupQ'
    secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name='myfoo'
    bucket_name2='mybar'

    # connect to rgw
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )
    connection2 = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key2,
        aws_secret_access_key=secret_key2,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user1,
            '--display-name', display_name1,
            '--email', email,
            '--access-key', access_key,
            '--secret', secret_key,
            '--max-buckets', '4'
            ],
            check_status=True)

    # TESTCASE 'duplicate email','user','create','existing user email','fails'
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user2,
            '--display-name', display_name2,
            '--email', email,
            ])
    assert err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
        check_status=True)

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1], check_status=True)

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'create', '--uid', user1,
            '--access-key', access_key2, '--secret', secret_key2,
            ], check_status=True)

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1],
        check_status=True)
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'rm', '--uid', user1,
            '--access-key', access_key2,
            ], check_status=True)
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    subuser_access = 'full'
    subuser_perm = 'full-control'

    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'create', '--subuser', subuser1,
            '--access', subuser_access
            ], check_status=True)

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'modify', '--subuser', subuser1,
            '--secret', swift_secret1,
            '--key-type', 'swift',
            ], check_status=True)

    # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])

    assert out['subusers'][0]['permissions'] == subuser_perm

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'create', '--subuser', subuser2,
            '--secret', swift_secret2,
            '--key-type', 'swift',
            ], check_status=True)

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1], check_status=True)
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'rm', '--subuser', subuser1,
            '--key-type', 'swift',
            ], check_status=True)
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'rm', '--subuser', subuser1,
            ], check_status=True)
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'rm', '--subuser', subuser2,
            '--key-type', 'swift', '--purge-keys',
            ], check_status=True)
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1],
        check_status=True)
    assert len(out) == 0

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1], check_status=True)
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
    assert len(out) >= 1
    assert bucket_name in out;

    # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
    bucket2 = connection.create_bucket(bucket_name + '2')
    bucket3 = connection.create_bucket(bucket_name + '3')
    bucket4 = connection.create_bucket(bucket_name + '4')
    # the 5th should fail.
    failed = False
    try:
        connection.create_bucket(bucket_name + '5')
    except Exception:
        failed = True
    assert failed

    # delete the buckets
    bucket2.delete()
    bucket3.delete()
    bucket4.delete()

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name], check_status=True)
    assert out['owner'] == user1
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1], check_status=True)
    assert len(out) == 1
    assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name], check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (err, out) = rgwadmin(ctx, client,
        ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
        check_status=True)

    # create a second user to link the bucket to
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user2,
            '--display-name', display_name2,
            '--access-key', access_key2,
            '--secret', secret_key2,
            '--max-buckets', '1',
            ],
            check_status=True)

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied

    # delete the object
    key.delete()

    # link the bucket to another user
    (err, out) = rgwadmin(ctx, client, ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
        check_status=True)

    bucket_data = out['data']
    assert bucket_data['bucket']['name'] == bucket_name

    bucket_id = bucket_data['bucket']['bucket_id']

    # link the bucket to another user
    (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name, '--bucket-id', bucket_id],
        check_status=True)

    # try to remove user, should fail (has a linked bucket)
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
    assert err

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
        check_status=True)

    # relink the bucket to the first user and delete the second user
    (err, out) = rgwadmin(ctx, client,
        ['bucket', 'link', '--uid', user1, '--bucket', bucket_name, '--bucket-id', bucket_id],
        check_status=True)

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2],
        check_status=True)

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = 'four'
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)

    # now delete it
    (err, out) = rgwadmin(ctx, client,
        ['object', 'rm', '--bucket', bucket_name, '--object', object_name],
        check_status=True)

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name],
            check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        if obj[:4] == 'meta' or obj[:4] == 'data' or obj[:18] == 'obj_delete_at_hint':
            continue

        (err, rgwlog) = rgwadmin(ctx, client, ['log', 'show', '--object', obj],
            check_status=True)
        assert len(rgwlog) > 0

        # exempt bucket_name2 from checking as it was only used for multi-region tests
        assert rgwlog['bucket'].find(bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
        assert rgwlog['bucket'] != bucket_name or rgwlog['bucket_id'] == bucket_id
        assert rgwlog['bucket_owner'] == user1 or rgwlog['bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
        for entry in rgwlog['log_entries']:
            log.debug('checking log entry: ', entry)
            assert entry['bucket'] == rgwlog['bucket']
            possible_buckets = [bucket_name + '5', bucket_name2]
            user = entry['user']
            assert user == user1 or user.endswith('system-user') or \
                rgwlog['bucket'] in possible_buckets

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj],
            check_status=True)

    # TODO: show log by bucket+date

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (20 * 60):      # wait up to 20 minutes
        (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj'])  # last operation we did is delete obj, wait for it to flush
        if get_user_successful_ops(out, user1) > 0:
            break
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0

    user_summary = get_user_summary(out, user1)

    total = user_summary['total']
    assert total['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
        check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    for entry in user_summary['categories']:
        assert entry['successful_ops'] > 0
    assert user_summary['user'] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
    for cat in test_categories:
        (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat],
            check_status=True)
        assert len(out['summary']) > 0
        user_summary = out['summary'][0]
        assert user_summary['user'] == user1
        assert len(user_summary['categories']) == 1
        entry = user_summary['categories'][0]
        assert entry['category'] == cat
        assert entry['successful_ops'] > 0

    # the usage flush interval is 30 seconds, wait that much an then some
    # to make sure everything has been flushed
    time.sleep(35)

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1],
        check_status=True)
    (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1],
        check_status=True)
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1],
        check_status=True)

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('five')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1],
        check_status=True)

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('six')

    # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = 'foo'*10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)

    # now delete the head
    big_key.delete()

    # wait a bit to give the garbage collector time to cycle
    time.sleep(15)

    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) > 0

    # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
    (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)

    #confirm
    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) == 0

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # delete should fail because ``key`` still exists
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409

    key.delete()
    bucket.delete()

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')

    # should be private already but guarantee it
    key.set_acl('private')

    (err, out) = rgwadmin(ctx, client,
        ['policy', '--bucket', bucket.name, '--object', key.key],
        check_status=True, format='xml')

    acl = get_acl(key)

    assert acl == out.strip('\n')

    # add another grantee by making the object public read
    key.set_acl('public-read')

    (err, out) = rgwadmin(ctx, client,
        ['policy', '--bucket', bucket.name, '--object', key.key],
        check_status=True, format='xml')

    acl = get_acl(key)

    assert acl == out.strip('\n')

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key_name = ['eight', 'nine', 'ten', 'eleven']
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])

    (err, out) = rgwadmin(ctx, client,
        ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
        check_status=True)

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps='user=read'
    (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps])

    assert out['caps'][0]['perm'] == 'read'

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps])

    assert not out['caps']

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('twelve')

    (err, out) = rgwadmin(ctx, client,
        ['user', 'rm', '--uid', user1, '--purge-data' ],
        check_status=True)

    # TESTCASE 'rm-user3','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
    #

    (err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
    orig_placement_pools = len(out['placement_pools'])

    # removed this test, it is not correct to assume that zone has default placement, it really
    # depends on how we set it up before
    #
    # assert len(out) > 0
    # assert len(out['placement_pools']) == 1

    # default_rule = out['placement_pools'][0]
    # assert default_rule['key'] == 'default-placement'

    rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}

    out['placement_pools'].append(rule)

    (err, out) = rgwadmin(ctx, client, ['zone', 'set'],
        stdin=StringIO(json.dumps(out)),
        check_status=True)

    (err, out) = rgwadmin(ctx, client, ['zone', 'get','--rgw-zone','default'])
    assert len(out) > 0
    assert len(out['placement_pools']) == orig_placement_pools + 1
Exemple #18
0
        count_success_ins += 1
    finally:
        count_ins += 1

f.close()
print("Successfully created %s instances" % (count_success_ins))

connection = boto.s3.connection.S3Connection(
    aws_access_key_id='adf938e8df4841129a5ca7089ab6b0ee',
    aws_secret_access_key='66f187d931f04af5805e899421bbfbef',
    port=8888,
    host='swift.rc.nectar.org.au',
    is_secure=True,
    validate_certs=False,
    calling_format=boto.s3.connection.OrdinaryCallingFormat())

# buckets = connection.get_all_buckets()

container_name = "twitter_container"
try:
    b = connection.create_bucket(container_name)
except boto.exception.S3CreateError as e:
    if e.status == 409:
        print("already exists the container: %s" % (container_name))
        pass
except:
    print "Unexpected error:", sys.exc_info()[0]

b = connection.get_bucket('twitter_container', validate=False)
print("Bucket: %s is ready" % (b.name))
Exemple #19
0
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    assert config is None or isinstance(config, list) \
        or isinstance(config, dict), \
        "task s3tests only supports a list or dictionary for configuration"
    all_clients = [
        'client.{id}'.format(id=id_)
        for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')
    ]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients = config.keys()

    # just use the first client...
    client = clients[0]

    ##
    user = '******'
    subuser1 = 'foo:foo1'
    subuser2 = 'foo:foo2'
    display_name = 'Foo'
    email = '*****@*****.**'
    access_key = '9te6NH5mcdcq0Tc5i8i1'
    secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2 = 'p5YnriCv1nAtykxBrupQ'
    secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name = 'myfoo'

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
        'user', 'create', '--uid', user, '--display-name', display_name,
        '--email', email, '--access-key', access_key, '--secret', secret_key
    ])
    assert not err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert out['user_id'] == user
    assert out['email'] == email
    assert out['display_name'] == display_name
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user])
    assert not err

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user])
    assert not err

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
        'key',
        'create',
        '--uid',
        user,
        '--access-key',
        access_key2,
        '--secret',
        secret_key2,
    ])
    assert not err

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][
        'access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][
        'secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx, client, [
        'key',
        'rm',
        '--uid',
        user,
        '--access-key',
        access_key2,
    ])
    assert not err
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
        'key',
        'create',
        '--subuser',
        subuser1,
        '--secret',
        swift_secret1,
        '--key-type',
        'swift',
    ])
    assert not err

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
        'key',
        'create',
        '--subuser',
        subuser2,
        '--secret',
        swift_secret2,
        '--key-type',
        'swift',
    ])
    assert not err

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][
        'user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[
        'swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx, client, [
        'key',
        'rm',
        '--subuser',
        subuser1,
        '--key-type',
        'swift',
    ])
    assert not err
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-swift-key2','key','rm','subuser','succeeds, second key is removed'
    (err, out) = rgwadmin(ctx, client, [
        'key',
        'rm',
        '--subuser',
        subuser2,
        '--key-type',
        'swift',
    ])
    assert not err
    assert len(out['swift_keys']) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user])
    assert not err
    assert len(out) == 0

    # connect to rgw
    (remote, ) = ctx.cluster.only(client).remotes.iterkeys()
    (remote_user, remote_host) = remote.name.split('@')
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=7280,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user])
    assert not err
    assert len(out) == 0

    # creat a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user])
    assert not err
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client,
                          ['bucket', 'stats', '--bucket', bucket_name])
    assert not err
    assert out['owner'] == user
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user])
    assert not err
    assert len(out) == 1
    assert out[0][
        'id'] == bucket_id  # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx, client,
                          ['bucket', 'stats', '--bucket-id',
                           '%d' % bucket_id])
    assert not err
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client,
                          ['bucket', 'stats', '--bucket-id',
                           '%d' % bucket_id])
    assert not err
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ['log', 'list'])
    assert not err
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj])
        assert not err
        assert len(log) > 0
        assert log['bucket'] == bucket_name
        assert log['bucket_id'] == bucket_id
        assert log['bucket_owner'] == user
        for entry in log['log_entries']:
            assert entry['bucket'] == bucket_name
            assert entry['user'] == user

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj])
        assert not err

    # TODO: show log by bucket+date

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user])
    assert not err

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user])
    assert not err

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('three')

    # TESTCASE 'rm-user','user','rm','existing user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user])
    assert not err

    # TESTCASE 'rm-user2','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert err
Exemple #20
0
 def setUpClass(cls):
     connection = boto.connect_s3(aws_access_key_id=key_id, aws_secret_access_key=access_key)
     connection.create_bucket(TestIntegrationS3.mart)
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    assert config is None or isinstance(config, list) \
        or isinstance(config, dict), \
        "task s3tests only supports a list or dictionary for configuration"
    all_clients = ['client.{id}'.format(id=id_)
                   for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients = config.keys()

    # just use the first client...
    client = clients[0];

    ##
    user='******'
    subuser1='foo:foo1'
    subuser2='foo:foo2'
    display_name='Foo'
    email='*****@*****.**'
    access_key='9te6NH5mcdcq0Tc5i8i1'
    secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2='p5YnriCv1nAtykxBrupQ'
    secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name='myfoo'

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user,
            '--display-name', display_name,
            '--email', email,
            '--access-key', access_key,
            '--secret', secret_key,
            '--max-buckets', '4',
            ])
    assert not err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert out['user_id'] == user
    assert out['email'] == email
    assert out['display_name'] == display_name
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user])
    assert not err

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user])
    assert not err

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'create', '--uid', user,
            '--access-key', access_key2, '--secret', secret_key2,
            ])
    assert not err

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'rm', '--uid', user,
            '--access-key', access_key2,
            ])
    assert not err
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'create', '--subuser', subuser1,
            '--secret', swift_secret1,
            '--key-type', 'swift',
            ])
    assert not err

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'create', '--subuser', subuser2,
            '--secret', swift_secret2,
            '--key-type', 'swift',
            ])
    assert not err

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert not err
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'rm', '--subuser', subuser1,
            '--key-type', 'swift',
            ])
    assert not err
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'rm', '--subuser', subuser1,
            ])
    assert not err
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'rm', '--subuser', subuser2,
            '--key-type', 'swift', '--purge-keys',
            ])
    assert not err
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user])
    assert not err
    assert len(out) == 0

    # connect to rgw
    (remote,) = ctx.cluster.only(client).remotes.iterkeys()
    (remote_user, remote_host) = remote.name.split('@')
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=7280,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user])
    assert not err
    assert len(out) == 0

    # creat a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user])
    assert not err
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
    bucket2 = connection.create_bucket(bucket_name + '2')
    bucket3 = connection.create_bucket(bucket_name + '3')
    bucket4 = connection.create_bucket(bucket_name + '4')
    # the 5th should fail.
    failed = False
    try:
        bucket5 = connection.create_bucket(bucket_name + '5')
    except:
        failed = True
    assert failed

    # delete the buckets
    bucket2.delete()
    bucket3.delete()
    bucket4.delete()
   
    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name])
    assert not err
    assert out['owner'] == user
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user])
    assert not err
    assert len(out) == 1
    assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket-id', bucket_id])
    assert not err
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket-id', bucket_id])
    assert not err
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ['log', 'list'])
    assert not err
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj])
        assert not err
        assert len(log) > 0
        assert log['bucket'].find(bucket_name) == 0
        assert log['bucket'] != bucket_name or log['bucket_id'] == bucket_id 
        assert log['bucket_owner'] == user or log['bucket'] == bucket_name + '5'
        for entry in log['log_entries']:
            assert entry['bucket'] == log['bucket']
            assert entry['user'] == user or log['bucket'] == bucket_name + '5'

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj])
        assert not err

    # TODO: show log by bucket+date

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= 45:
        (err, out) = rgwadmin(ctx, client, ['usage', 'show'])
        if len(out['entries']) > 0:
            break;
        time.sleep(1)

    assert time.time() - timestamp <= 45

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show'])
    assert not err
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    for entry in out['summary']:
        assert entry['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user])
    assert not err
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    for entry in out['summary']:
        assert entry['successful_ops'] > 0
        assert entry['user'] == user

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user])
    assert not err
    (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user])
    assert not err
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user])
    assert not err

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user])
    assert not err

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('three')

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user])
    assert err

    # delete should fail
    fails = False
    try:
        bucket.delete()
    except:
        fails = True
    assert fails

    key.delete()
    bucket.delete()
    
    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user])
    assert not err

    # TESTCASE 'rm-user2','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user])
    assert err
Exemple #22
0
args = parser.parse_args()

s3_access_key=os.environ['S3_ACCESS_KEY_ID']
s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY']

connection = boto.s3.connection.S3Connection(
    aws_access_key_id= s3_access_key,
    aws_secret_access_key=s3_secret_key,
    is_secure=False,
    port=8000,
    host='localhost',
    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)

print 'create a bucket'
bucket = connection.create_bucket(args.name)

obj1 = boto.s3.key.Key(bucket)
obj1.key = 'foo'
obj1.set_contents_from_string('This is a test of S3')

obj2 = boto.s3.key.Key(bucket)
obj2.key='abc'
obj2.delete()

b = connection.get_bucket(args.name)
print 'bucket list'
for k in b.list():
    print k

print "second time"
Exemple #23
0
s3_access_key=os.environ['S3_ACCESS_KEY_ID']
s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY']

parser = argparse.ArgumentParser(description='create buckets')
parser.add_argument('--num', type=int, action='store', default=100 )
parser.add_argument('--port', type=int, action='store', default=8000 )
parser.add_argument('--key',type=str, action='store', default=s3_access_key)
parser.add_argument('--secret',type=str, action='store', default=s3_secret_key)
args = parser.parse_args()


#client_access_key=os.environ['SYSTEM_ACCESS_KEY_ID']
#client_secret_key=os.environ['SYSTEM_SECRET_ACCESS_KEY']

connection = boto.s3.connection.S3Connection(
    aws_access_key_id= args.key,
    aws_secret_access_key=args.secret,
    is_secure=False,
    port=args.port,
    host='localhost',
    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)


base_name='bucket'

for i in range(args.num):
    print 'creating '+ `i`
    bucket = connection.create_bucket(base_name + `i`)
    print bucket
Exemple #24
0
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    global log

    assert ctx.rgw.config, \
        "radosgw_admin task needs a config passed from the rgw task"
    config = ctx.rgw.config
    log.debug('config is: %r', config)

    clients_from_config = config.keys()

    # choose first client as default
    client = clients_from_config[0]

    # once the client is chosen, pull the host name and  assigned port out of
    # the role_endpoints that were assigned by the rgw task
    (remote_host, remote_port) = ctx.rgw.role_endpoints[client]

    ##
    user1 = 'foo'
    user2 = 'fud'
    subuser1 = 'foo:foo1'
    subuser2 = 'foo:foo2'
    display_name1 = 'Foo'
    display_name2 = 'Fud'
    email = '*****@*****.**'
    email2 = '*****@*****.**'
    access_key = '9te6NH5mcdcq0Tc5i8i1'
    secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2 = 'p5YnriCv1nAtykxBrupQ'
    secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name = 'myfoo'
    bucket_name2 = 'mybar'

    # connect to rgw
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )
    connection2 = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key2,
        aws_secret_access_key=secret_key2,
        is_secure=False,
        port=remote_port,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(
        ctx,
        client, [
            'user', 'create', '--uid', user1, '--display-name', display_name1,
            '--email', email, '--access-key', access_key, '--secret',
            secret_key, '--max-buckets', '4'
        ],
        check_status=True)

    # TESTCASE 'duplicate email','user','create','existing user email','fails'
    (err, out) = rgwadmin(ctx, client, [
        'user',
        'create',
        '--uid',
        user2,
        '--display-name',
        display_name2,
        '--email',
        email,
    ])
    assert err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'suspend', '--uid', user1],
                          check_status=True)

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'enable', '--uid', user1],
                          check_status=True)

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'create',
                              '--uid',
                              user1,
                              '--access-key',
                              access_key2,
                              '--secret',
                              secret_key2,
                          ],
                          check_status=True)

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][
        'access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][
        'secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'rm',
                              '--uid',
                              user1,
                              '--access-key',
                              access_key2,
                          ],
                          check_status=True)
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    subuser_access = 'full'
    subuser_perm = 'full-control'

    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser', 'create', '--subuser', subuser1,
                              '--access', subuser_access
                          ],
                          check_status=True)

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'modify',
                              '--subuser',
                              subuser1,
                              '--secret',
                              swift_secret1,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)

    # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])

    assert out['subusers'][0]['permissions'] == subuser_perm

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'create',
                              '--subuser',
                              subuser2,
                              '--secret',
                              swift_secret2,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][
        'user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[
        'swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'rm',
                              '--subuser',
                              subuser1,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'rm',
                              '--subuser',
                              subuser1,
                          ],
                          check_status=True)
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'rm',
                              '--subuser',
                              subuser2,
                              '--key-type',
                              'swift',
                              '--purge-keys',
                          ],
                          check_status=True)
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--uid', user1],
                          check_status=True)
    assert len(out) == 0

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'list', '--uid', user1],
                          check_status=True)
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'list', '--uid', user1],
                          check_status=True)
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
    assert len(out) >= 1
    assert bucket_name in out

    # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
    bucket2 = connection.create_bucket(bucket_name + '2')
    bucket3 = connection.create_bucket(bucket_name + '3')
    bucket4 = connection.create_bucket(bucket_name + '4')
    # the 5th should fail.
    failed = False
    try:
        connection.create_bucket(bucket_name + '5')
    except Exception:
        failed = True
    assert failed

    # delete the buckets
    bucket2.delete()
    bucket3.delete()
    bucket4.delete()

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['owner'] == user1
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--uid', user1],
                          check_status=True)
    assert len(out) == 1
    assert out[0][
        'id'] == bucket_id  # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
        check_status=True)

    # create a second user to link the bucket to
    (err, out) = rgwadmin(ctx,
                          client, [
                              'user',
                              'create',
                              '--uid',
                              user2,
                              '--display-name',
                              display_name2,
                              '--access-key',
                              access_key2,
                              '--secret',
                              secret_key2,
                              '--max-buckets',
                              '1',
                          ],
                          check_status=True)

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied

    # delete the object
    key.delete()

    # link the bucket to another user
    (err,
     out) = rgwadmin(ctx,
                     client,
                     ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
                     check_status=True)

    bucket_data = out['data']
    assert bucket_data['bucket']['name'] == bucket_name

    bucket_id = bucket_data['bucket']['bucket_id']

    # link the bucket to another user
    (err, out) = rgwadmin(ctx,
                          client, [
                              'bucket', 'link', '--uid', user2, '--bucket',
                              bucket_name, '--bucket-id', bucket_id
                          ],
                          check_status=True)

    # try to remove user, should fail (has a linked bucket)
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
    assert err

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
        check_status=True)

    # relink the bucket to the first user and delete the second user
    (err, out) = rgwadmin(ctx,
                          client, [
                              'bucket', 'link', '--uid', user1, '--bucket',
                              bucket_name, '--bucket-id', bucket_id
                          ],
                          check_status=True)

    (err, out) = rgwadmin(ctx,
                          client, ['user', 'rm', '--uid', user2],
                          check_status=True)

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = 'four'
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)

    # now delete it
    (err, out) = rgwadmin(
        ctx,
        client,
        ['object', 'rm', '--bucket', bucket_name, '--object', object_name],
        check_status=True)

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        if obj[:
               4] == 'meta' or obj[:
                                   4] == 'data' or obj[:
                                                       18] == 'obj_delete_at_hint':
            continue

        (err, rgwlog) = rgwadmin(ctx,
                                 client, ['log', 'show', '--object', obj],
                                 check_status=True)
        assert len(rgwlog) > 0

        # exempt bucket_name2 from checking as it was only used for multi-region tests
        assert rgwlog['bucket'].find(
            bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
        assert rgwlog['bucket'] != bucket_name or rgwlog[
            'bucket_id'] == bucket_id
        assert rgwlog['bucket_owner'] == user1 or rgwlog[
            'bucket'] == bucket_name + '5' or rgwlog['bucket'] == bucket_name2
        for entry in rgwlog['log_entries']:
            log.debug('checking log entry: ', entry)
            assert entry['bucket'] == rgwlog['bucket']
            possible_buckets = [bucket_name + '5', bucket_name2]
            user = entry['user']
            assert user == user1 or user.endswith('system-user') or \
                rgwlog['bucket'] in possible_buckets

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx,
                              client, ['log', 'rm', '--object', obj],
                              check_status=True)

    # TODO: show log by bucket+date

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (20 * 60):  # wait up to 20 minutes
        (err, out) = rgwadmin(
            ctx, client,
            ['usage', 'show', '--categories', 'delete_obj'
             ])  # last operation we did is delete obj, wait for it to flush
        if get_user_successful_ops(out, user1) > 0:
            break
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0

    user_summary = get_user_summary(out, user1)

    total = user_summary['total']
    assert total['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'show', '--uid', user1],
                          check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    for entry in user_summary['categories']:
        assert entry['successful_ops'] > 0
    assert user_summary['user'] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = [
        'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'
    ]
    for cat in test_categories:
        (err, out) = rgwadmin(
            ctx,
            client, ['usage', 'show', '--uid', user1, '--categories', cat],
            check_status=True)
        assert len(out['summary']) > 0
        user_summary = out['summary'][0]
        assert user_summary['user'] == user1
        assert len(user_summary['categories']) == 1
        entry = user_summary['categories'][0]
        assert entry['category'] == cat
        assert entry['successful_ops'] > 0

    # the usage flush interval is 30 seconds, wait that much an then some
    # to make sure everything has been flushed
    time.sleep(35)

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'trim', '--uid', user1],
                          check_status=True)
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'show', '--uid', user1],
                          check_status=True)
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'suspend', '--uid', user1],
                          check_status=True)

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('five')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'enable', '--uid', user1],
                          check_status=True)

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('six')

    # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = 'foo' * 10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)

    # now delete the head
    big_key.delete()

    # wait a bit to give the garbage collector time to cycle
    time.sleep(15)

    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) > 0

    # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
    (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)

    #confirm
    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) == 0

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # delete should fail because ``key`` still exists
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409

    key.delete()
    bucket.delete()

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')

    # should be private already but guarantee it
    key.set_acl('private')

    (err,
     out) = rgwadmin(ctx,
                     client,
                     ['policy', '--bucket', bucket.name, '--object', key.key],
                     check_status=True,
                     format='xml')

    acl = get_acl(key)

    assert acl == out.strip('\n')

    # add another grantee by making the object public read
    key.set_acl('public-read')

    (err,
     out) = rgwadmin(ctx,
                     client,
                     ['policy', '--bucket', bucket.name, '--object', key.key],
                     check_status=True,
                     format='xml')

    acl = get_acl(key)

    assert acl == out.strip('\n')

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key_name = ['eight', 'nine', 'ten', 'eleven']
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])

    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
        check_status=True)

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps = 'user=read'
    (err, out) = rgwadmin(ctx, client,
                          ['caps', 'add', '--uid', user1, '--caps', caps])

    assert out['caps'][0]['perm'] == 'read'

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (err, out) = rgwadmin(ctx, client,
                          ['caps', 'rm', '--uid', user1, '--caps', caps])

    assert not out['caps']

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('twelve')

    (err, out) = rgwadmin(ctx,
                          client,
                          ['user', 'rm', '--uid', user1, '--purge-data'],
                          check_status=True)

    # TESTCASE 'rm-user3','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
    #

    (err, out) = rgwadmin(ctx, client,
                          ['zone', 'get', '--rgw-zone', 'default'])
    orig_placement_pools = len(out['placement_pools'])

    # removed this test, it is not correct to assume that zone has default placement, it really
    # depends on how we set it up before
    #
    # assert len(out) > 0
    # assert len(out['placement_pools']) == 1

    # default_rule = out['placement_pools'][0]
    # assert default_rule['key'] == 'default-placement'

    rule = {
        'key': 'new-placement',
        'val': {
            'data_pool': '.rgw.buckets.2',
            'index_pool': '.rgw.buckets.index.2'
        }
    }

    out['placement_pools'].append(rule)

    (err, out) = rgwadmin(ctx,
                          client, ['zone', 'set'],
                          stdin=StringIO(json.dumps(out)),
                          check_status=True)

    (err, out) = rgwadmin(ctx, client,
                          ['zone', 'get', '--rgw-zone', 'default'])
    assert len(out) > 0
    assert len(out['placement_pools']) == orig_placement_pools + 1
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    assert config is None or isinstance(config, list) \
        or isinstance(config, dict), \
        "task s3tests only supports a list or dictionary for configuration"
    all_clients = ['client.{id}'.format(id=id_)
                   for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients = config.keys()

    # just use the first client...
    client = clients[0];

    ##
    user1='foo'
    user2='fud'
    subuser1='foo:foo1'
    subuser2='foo:foo2'
    display_name1='Foo'
    display_name2='Fud'
    email='*****@*****.**'
    access_key='9te6NH5mcdcq0Tc5i8i1'
    secret_key='Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2='p5YnriCv1nAtykxBrupQ'
    secret_key2='Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1='gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2='ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name='myfoo'

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user1,
            '--display-name', display_name1,
            '--email', email,
            '--access-key', access_key,
            '--secret', secret_key,
            '--max-buckets', '4'
            ])
    assert not err

    # TESTCASE 'duplicate email','user','create','existing user email','fails'
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user2,
            '--display-name', display_name2,
            '--email', email,
            ])
    assert err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert not err
    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1])
    assert not err

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert not err
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1])
    assert not err

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert not err
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'create', '--uid', user1,
            '--access-key', access_key2, '--secret', secret_key2,
            ])
    assert not err

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert not err
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1]['access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1]['secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'rm', '--uid', user1,
            '--access-key', access_key2,
            ])
    assert not err
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    subuser_access = 'full'
    subuser_perm = 'full-control'

    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'create', '--subuser', subuser1,
            '--access', subuser_access
            ])
    assert not err

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'modify', '--subuser', subuser1,
            '--secret', swift_secret1,
            '--key-type', 'swift',
            ])
    assert not err

    # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])

    assert out['subusers'][0]['permissions'] == subuser_perm

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert not err
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'create', '--subuser', subuser2,
            '--secret', swift_secret2,
            '--key-type', 'swift',
            ])
    assert not err

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert not err
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1]['user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out['swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'key', 'rm', '--subuser', subuser1,
            '--key-type', 'swift',
            ])
    assert not err
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'rm', '--subuser', subuser1,
            ])
    assert not err
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (err, out) = rgwadmin(ctx, client, [
            'subuser', 'rm', '--subuser', subuser2,
            '--key-type', 'swift', '--purge-keys',
            ])
    assert not err
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1])
    assert not err
    assert len(out) == 0

    # connect to rgw
    (remote,) = ctx.cluster.only(client).remotes.iterkeys()
    (remote_user, remote_host) = remote.name.split('@')
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=7280,
        host=remote_host,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
        )

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1])
    assert not err
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list', '--uid', user1])
    assert not err
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list'])
    assert not err
    assert len(out) >= 1
    assert bucket_name in out;

    # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
    bucket2 = connection.create_bucket(bucket_name + '2')
    bucket3 = connection.create_bucket(bucket_name + '3')
    bucket4 = connection.create_bucket(bucket_name + '4')
    # the 5th should fail.
    failed = False
    try:
        connection.create_bucket(bucket_name + '5')
    except:
        failed = True
    assert failed

    # delete the buckets
    bucket2.delete()
    bucket3.delete()
    bucket4.delete()
   
    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name])
    assert not err
    assert out['owner'] == user1
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'stats', '--uid', user1])
    assert not err
    assert len(out) == 1
    assert out[0]['id'] == bucket_id    # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name])
    assert not err
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # reclaim it
    key.delete()

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name])
    assert not err

    # create a second user to link the bucket to
    (err, out) = rgwadmin(ctx, client, [
            'user', 'create',
            '--uid', user2,
            '--display-name', display_name2,
            '--access-key', access_key2,
            '--secret', secret_key2,
            '--max-buckets', '1',
            ])
    assert not err

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied

    # delete the object
    key.delete()

    # link the bucket to another user
    (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user2, '--bucket', bucket_name])
    assert not err

    # try to remove user, should fail (has a linked bucket)
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
    assert err

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name])
    assert not err

    # relink the bucket to the first user and delete the second user
    (err, out) = rgwadmin(ctx, client, ['bucket', 'link', '--uid', user1, '--bucket', bucket_name])
    assert not err

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
    assert not err

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = 'four'
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)

    # now delete it
    (err, out) = rgwadmin(ctx, client, ['object', 'rm', '--bucket', bucket_name, '--object', object_name])

    assert not err

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, [
            'bucket', 'stats', '--bucket', bucket_name])
    assert not err
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ['log', 'list'])
    assert not err
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        (err, log) = rgwadmin(ctx, client, ['log', 'show', '--object', obj])
        assert not err
        assert len(log) > 0
        assert log['bucket'].find(bucket_name) == 0
        assert log['bucket'] != bucket_name or log['bucket_id'] == bucket_id 
        assert log['bucket_owner'] == user1 or log['bucket'] == bucket_name + '5'
        for entry in log['log_entries']:
            assert entry['bucket'] == log['bucket']
            assert entry['user'] == user1 or log['bucket'] == bucket_name + '5'

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx, client, ['log', 'rm', '--object', obj])
        assert not err

    # TODO: show log by bucket+date

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (20 * 60):      # wait up to 20 minutes
        (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--categories', 'delete_obj'])  # last operation we did is delete obj, wait for it to flush
        if successful_ops(out) > 0:
            break;
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show'])
    assert not err
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    total = user_summary['total']
    assert total['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1])
    assert not err
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    for entry in user_summary['categories']:
        assert entry['successful_ops'] > 0
    assert user_summary['user'] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = ['create_bucket', 'put_obj', 'delete_obj', 'delete_bucket']
    for cat in test_categories:
        (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1, '--categories', cat])
        assert not err
        assert len(out['summary']) > 0
        user_summary = out['summary'][0]
        assert user_summary['user'] == user1
        assert len(user_summary['categories']) == 1
        entry = user_summary['categories'][0]
        assert entry['category'] == cat
        assert entry['successful_ops'] > 0

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (err, out) = rgwadmin(ctx, client, ['usage', 'trim', '--uid', user1])
    assert not err
    (err, out) = rgwadmin(ctx, client, ['usage', 'show', '--uid', user1])
    assert not err
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'suspend', '--uid', user1])
    assert not err

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('five')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'enable', '--uid', user1])
    assert not err

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('six')

    # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = 'foo'*10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)

    # now delete the head
    big_key.delete()

    # wait a bit to give the garbage collector time to cycle
    time.sleep(15)

    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) > 0

    # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
    (err, out) = rgwadmin(ctx, client, ['gc', 'process'])

    assert not err

    #confirm
    (err, out) = rgwadmin(ctx, client, ['gc', 'list'])

    assert len(out) == 0

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # delete should fail because ``key`` still exists
    fails = False
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409

    key.delete()
    bucket.delete()

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')

    # should be private already but guarantee it
    key.set_acl('private')

    (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key])

    assert not err

    acl = key.get_xml_acl()

    assert acl == out.strip('\n')

    # add another grantee by making the object public read
    key.set_acl('public-read')

    (err, out) = rgwadmin(ctx, client, ['policy', '--bucket', bucket.name, '--object', key.key])

    assert not err

    acl = key.get_xml_acl()
    assert acl == out.strip('\n')

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key_name = ['eight', 'nine', 'ten', 'eleven']
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])

    (err, out) = rgwadmin(ctx, client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'])
    assert not err

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps='user=read'
    (err, out) = rgwadmin(ctx, client, ['caps', 'add', '--uid', user1, '--caps', caps])

    assert out['caps'][0]['perm'] == 'read'

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['caps', 'rm', '--uid', user1, '--caps', caps])

    assert not out['caps']

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('twelve')

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1, '--purge-data' ])
    assert not err

    # TESTCASE 'rm-user3','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
    (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
    assert len(out) > 0
    assert len(out['placement_pools']) == 1

    default_rule = out['placement_pools'][0]
    assert default_rule['key'] == 'default-placement'

    rule={'key': 'new-placement', 'val': {'data_pool': '.rgw.buckets.2', 'index_pool': '.rgw.buckets.index.2'}}

    out['placement_pools'].append(rule)

    (err, out) = rgwadmin(ctx, client, ['zone', 'set'], stdin=StringIO(json.dumps(out)))
    assert not err

    (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
    assert len(out) > 0
    assert len(out['placement_pools']) == 2
Exemple #26
0
def task(ctx, config):
    """
    Test radosgw-admin functionality through the RESTful interface
    """
    assert config is None or isinstance(config, list) \
        or isinstance(config, dict), \
        "task s3tests only supports a list or dictionary for configuration"
    all_clients = [
        'client.{id}'.format(id=id_)
        for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')
    ]
    if config is None:
        config = all_clients
    if isinstance(config, list):
        config = dict.fromkeys(config)
    clients = config.keys()

    # just use the first client...
    client = next(iter(clients))

    ##
    admin_user = '******'
    admin_display_name = 'Ms. Admin User'
    admin_access_key = 'MH1WC2XQ1S8UISFDZC8W'
    admin_secret_key = 'dQyrTPA0s248YeN5bBv4ukvKU0kh54LWWywkrpoG'
    admin_caps = 'users=read, write; usage=read, write; buckets=read, write; zone=read, write; info=read'

    user1 = 'foo'
    user2 = 'fud'
    subuser1 = 'foo:foo1'
    subuser2 = 'foo:foo2'
    display_name1 = 'Foo'
    display_name2 = 'Fud'
    email = '*****@*****.**'
    access_key = '9te6NH5mcdcq0Tc5i8i1'
    secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2 = 'p5YnriCv1nAtykxBrupQ'
    secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name = 'myfoo'

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'
    # TESTCASE 'create-admin-user','user','create','administrative user','succeeds'
    (err, out) = rgwadmin(ctx, client, [
        'user', 'create', '--uid', admin_user, '--display-name',
        admin_display_name, '--access-key', admin_access_key, '--secret',
        admin_secret_key, '--max-buckets', '0', '--caps', admin_caps
    ])
    logging.error(out)
    logging.error(err)
    assert not err

    assert hasattr(ctx,
                   'rgw'), 'radosgw-admin-rest must run after the rgw task'
    endpoint = ctx.rgw.role_endpoints.get(client)
    assert endpoint, 'no rgw endpoint for {}'.format(client)

    admin_conn = boto.s3.connection.S3Connection(
        aws_access_key_id=admin_access_key,
        aws_secret_access_key=admin_secret_key,
        is_secure=True if endpoint.cert else False,
        port=endpoint.port,
        host=endpoint.hostname,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )

    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {"uid": user1})
    assert ret == 404

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (ret, out) = rgwadmin_rest(
        admin_conn, ['user', 'create'], {
            'uid': user1,
            'display-name': display_name1,
            'email': email,
            'access-key': access_key,
            'secret-key': secret_key,
            'max-buckets': '4'
        })

    assert ret == 200

    # TESTCASE 'list-no-user','user','list','list user keys','user list object'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {
        'list': '',
        'max-entries': 0
    })
    assert ret == 200
    assert out['count'] == 0
    assert out['truncated'] == True
    assert len(out['keys']) == 0
    assert len(out['marker']) > 0

    # TESTCASE 'list-user-without-marker','user','list','list user keys','user list object'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {
        'list': '',
        'max-entries': 1
    })
    assert ret == 200
    assert out['count'] == 1
    assert out['truncated'] == True
    assert len(out['keys']) == 1
    assert len(out['marker']) > 0
    marker = out['marker']

    # TESTCASE 'list-user-with-marker','user','list','list user keys','user list object'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'list'], {
        'list': '',
        'max-entries': 1,
        'marker': marker
    })
    assert ret == 200
    assert out['count'] == 1
    assert out['truncated'] == False
    assert len(out['keys']) == 1

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})

    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']
    assert out['tenant'] == ''
    assert out['max_buckets'] == 4
    assert out['caps'] == []
    assert out['op_mask'] == 'read, write, delete'
    assert out['default_placement'] == ''
    assert out['default_storage_class'] == ''
    assert out['placement_tags'] == []
    assert not out['bucket_quota']['enabled']
    assert not out['bucket_quota']['check_on_raw']
    assert out['bucket_quota']['max_size'] == -1
    assert out['bucket_quota']['max_size_kb'] == 0
    assert out['bucket_quota']['max_objects'] == -1
    assert not out['user_quota']['enabled']
    assert not out['user_quota']['check_on_raw']
    assert out['user_quota']['max_size'] == -1
    assert out['user_quota']['max_size_kb'] == 0
    assert out['user_quota']['max_objects'] == -1
    assert out['temp_url_keys'] == []
    assert out['type'] == 'rgw'
    assert out['mfa_ids'] == []
    # TESTCASE 'info-existing','user','info','existing user query with wrong uid but correct access key','returns correct info'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {
        'access-key': access_key,
        'uid': 'uid_not_exist'
    })

    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']
    assert out['tenant'] == ''
    assert out['max_buckets'] == 4
    assert out['caps'] == []
    assert out['op_mask'] == "read, write, delete"
    assert out['default_placement'] == ''
    assert out['default_storage_class'] == ''
    assert out['placement_tags'] == []
    assert not out['bucket_quota']['enabled']
    assert not out['bucket_quota']['check_on_raw']
    assert out['bucket_quota']['max_size'] == -1
    assert out['bucket_quota']['max_size_kb'] == 0
    assert out['bucket_quota']['max_objects'] == -1
    assert not out['user_quota']['enabled']
    assert not out['user_quota']['check_on_raw']
    assert out['user_quota']['max_size'] == -1
    assert out['user_quota']['max_size_kb'] == 0
    assert out['user_quota']['max_objects'] == -1
    assert out['temp_url_keys'] == []
    assert out['type'] == 'rgw'
    assert out['mfa_ids'] == []

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {
        'uid': user1,
        'suspended': True
    })
    assert ret == 200

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert ret == 200
    assert out['suspended']
    assert out['email'] == email

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {
        'uid': user1,
        'suspended': 'false'
    })
    assert not err

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert ret == 200
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['key', 'create'], {
        'uid': user1,
        'access-key': access_key2,
        'secret-key': secret_key2
    })

    assert ret == 200

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert ret == 200
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][
        'access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][
        'secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (ret, out) = rgwadmin_rest(admin_conn, ['key', 'rm'], {
        'uid': user1,
        'access-key': access_key2
    })

    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})

    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'create'], {
        'subuser': subuser1,
        'secret-key': swift_secret1,
        'key-type': 'swift'
    })

    assert ret == 200

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert ret == 200
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'create'], {
        'subuser': subuser2,
        'secret-key': swift_secret2,
        'key-type': 'swift'
    })

    assert ret == 200

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert ret == 200
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][
        'user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[
        'swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (ret, out) = rgwadmin_rest(admin_conn, ['key', 'rm'], {
        'subuser': subuser1,
        'key-type': 'swift'
    })

    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'rm'],
                               {'subuser': subuser1})

    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (ret, out) = rgwadmin_rest(admin_conn, ['subuser', 'rm'], {
        'subuser': subuser2,
        'key-type': 'swift',
        '{purge-keys': True
    })

    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','info','no session/buckets','succeeds, empty list'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid': user1})
    assert ret == 200
    assert len(out) == 0

    # connect to rgw
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=True if endpoint.cert else False,
        port=endpoint.port,
        host=endpoint.hostname,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
        'uid': user1,
        'stats': True
    })
    assert ret == 200
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {'uid': user1})
    assert ret == 200
    assert len(out) == 1
    assert out[0] == bucket_name

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
        'bucket': bucket_name,
        'stats': True
    })

    assert ret == 200
    assert out['owner'] == user1
    assert out['tenant'] == ''
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
        'uid': user1,
        'stats': True
    })
    assert ret == 200
    assert len(out) == 1
    assert out[0][
        'id'] == bucket_id  # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
        'bucket': bucket_name,
        'stats': True
    })
    assert ret == 200
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    # TESTCASE 'bucket-stats6', 'bucket', 'stats', 'non-existent bucket', 'fails, 'bucket not found error'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'],
                               {'bucket': 'doesnotexist'})
    assert ret == 404
    assert out['Code'] == 'NoSuchBucket'

    # reclaim it
    key.delete()

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'unlink'], {
        'uid': user1,
        'bucket': bucket_name
    })

    assert ret == 200

    # create a second user to link the bucket to
    (ret, out) = rgwadmin_rest(
        admin_conn, ['user', 'create'], {
            'uid': user2,
            'display-name': display_name2,
            'access-key': access_key2,
            'secret-key': secret_key2,
            'max-buckets': '1',
        })

    assert ret == 200

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied

    # delete the object
    key.delete()

    # link the bucket to another user
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {
        'uid': user2,
        'bucket': bucket_name,
        'bucket-id': bucket_id,
    })

    assert ret == 200

    # try creating an object with the first user which should cause an error
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('three')
    except boto.exception.S3ResponseError:
        denied = True

    assert denied

    # relink the bucket to the first user and delete the second user
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'link'], {
        'uid': user1,
        'bucket': bucket_name,
        'bucket-id': bucket_id,
    })
    assert ret == 200

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user2})
    assert ret == 200

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = 'four'
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)

    # now delete it
    (ret, out) = rgwadmin_rest(admin_conn, ['object', 'rm'], {
        'bucket': bucket_name,
        'object': object_name
    })
    assert ret == 200

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'info'], {
        'bucket': bucket_name,
        'stats': True
    })
    assert ret == 200
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # create a bucket for deletion stats
    useless_bucket = connection.create_bucket('useless-bucket')
    useless_key = useless_bucket.new_key('useless_key')
    useless_key.set_contents_from_string('useless string')

    # delete it
    useless_key.delete()
    useless_bucket.delete()

    # wait for the statistics to flush
    time.sleep(60)

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (20 * 60):  # wait up to 20 minutes
        (ret, out) = rgwadmin_rest(
            admin_conn, ['usage', 'show'],
            {'categories': 'delete_obj'
             })  # last operation we did is delete obj, wait for it to flush

        if get_user_successful_ops(out, user1) > 0:
            break
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'])
    assert ret == 200
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = get_user_summary(out, user1)
    total = user_summary['total']
    assert total['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid': user1})
    assert ret == 200
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    for entry in user_summary['categories']:
        assert entry['successful_ops'] > 0
    assert user_summary['user'] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = [
        'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'
    ]
    for cat in test_categories:
        (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {
            'uid': user1,
            'categories': cat
        })
        assert ret == 200
        assert len(out['summary']) > 0
        user_summary = out['summary'][0]
        assert user_summary['user'] == user1
        assert len(user_summary['categories']) == 1
        entry = user_summary['categories'][0]
        assert entry['category'] == cat
        assert entry['successful_ops'] > 0

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'trim'], {'uid': user1})
    assert ret == 200
    (ret, out) = rgwadmin_rest(admin_conn, ['usage', 'show'], {'uid': user1})
    assert ret == 200
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {
        'uid': user1,
        'suspended': True
    })
    assert ret == 200

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('five')
    except boto.exception.S3ResponseError as e:
        assert e.status == 403

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'modify'], {
        'uid': user1,
        'suspended': 'false'
    })
    assert ret == 200

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('six')

    # TESTCASE 'garbage-list', 'garbage', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = 'foo' * 10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)

    # now delete the head
    big_key.delete()

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user1})
    assert ret == 409

    # delete should fail because ``key`` still exists
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409

    key.delete()
    bucket.delete()

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')

    # should be private already but guarantee it
    key.set_acl('private')

    (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {
        'bucket': bucket.name,
        'object': key.key
    })
    assert ret == 200
    assert len(out['acl']['grant_map']) == 1

    # add another grantee by making the object public read
    key.set_acl('public-read')

    (ret, out) = rgwadmin_rest(admin_conn, ['policy', 'show'], {
        'bucket': bucket.name,
        'object': key.key
    })
    assert ret == 200
    assert len(out['acl']['grant_map']) == 2

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key_name = ['eight', 'nine', 'ten', 'eleven']
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])

    (ret, out) = rgwadmin_rest(admin_conn, ['bucket', 'rm'], {
        'bucket': bucket_name,
        'purge-objects': True
    })
    assert ret == 200

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps = 'usage=read'
    (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'add'], {
        'uid': user1,
        'user-caps': caps
    })
    assert ret == 200
    assert out[0]['perm'] == 'read'

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['caps', 'rm'], {
        'uid': user1,
        'user-caps': caps
    })
    assert ret == 200
    assert not out

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {'uid': user1})
    assert ret == 409

    # TESTCASE 'rm-user2', 'user', 'rm', user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('twelve')

    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'rm'], {
        'uid': user1,
        'purge-data': True
    })
    assert ret == 200

    # TESTCASE 'rm-user3','user','info','deleted user','fails'
    (ret, out) = rgwadmin_rest(admin_conn, ['user', 'info'], {'uid': user1})
    assert ret == 404

    # TESTCASE 'info' 'display info' 'succeeds'
    (ret, out) = rgwadmin_rest(admin_conn, ['info', ''])
    assert ret == 200
    info = out['info']
    backends = info['storage_backends']
    name = backends[0]['name']
    fsid = backends[0]['cluster_id']
    # name is always "rados" at time of writing, but zipper would allow
    # other backends, at some point
    assert len(name) > 0
    # fsid is a uuid, but I'm not going to try to parse it
    assert len(fsid) > 0
Exemple #27
0
args = parser.parse_args()

s3_access_key=os.environ['S3_ACCESS_KEY_ID']
s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY']

connection = boto.s3.connection.S3Connection(
    aws_access_key_id=s3_access_key,
    aws_secret_access_key=s3_secret_key,
    is_secure=False,
    port=args.port,
    host=args.host,
    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)

print 'create a bucket'
bucket = connection.create_bucket(args.bucket)



start = 0
while start  < args.num:
    threads = []
    num_threads = 0
    while num_threads < args.num_threads and start < args.num:
        end = min(args.num, start + args.slice)
        t = threading.Thread(target=create_objs, args=(bucket, start, end,))
        threads.append(t)
        t. start()
        start += args.slice
        num_threads += 1
    for t in threads:
Exemple #28
0
args = parser.parse_args()

s3_access_key=os.environ['S3_ACCESS_KEY_ID']
s3_secret_key=os.environ['S3_SECRET_ACCESS_KEY']

connection = boto.s3.connection.S3Connection(
    aws_access_key_id=s3_access_key,
    aws_secret_access_key=s3_secret_key,
    is_secure=False,
    port=8000,
    host='127.0.0.1',
    calling_format=boto.s3.connection.OrdinaryCallingFormat(),
)

print 'create a bucket'
bucket = connection.create_bucket('bucket10')

for i in range(args.num):
    print 'creating obj'+ `i`
    obj = boto.s3.key.Key(bucket)
    obj.key = 'obj' + `i`
    obj.set_contents_from_string('This is a test of S3')

print 'list objects'
i = 0
for k in bucket.list():
    print 'obj ' + `i`
    print k
    i = i + 1
Exemple #29
0
def task(ctx, config):
    """
    Test radosgw-admin functionality against a running rgw instance.
    """
    global log

    assert ctx.rgw.config, \
        "radosgw_admin task needs a config passed from the rgw task"
    config = ctx.rgw.config
    log.debug('config is: %r', config)

    clients_from_config = config.keys()

    # choose first client as default
    client = next(iter(clients_from_config))

    # once the client is chosen, pull the host name and  assigned port out of
    # the role_endpoints that were assigned by the rgw task
    endpoint = ctx.rgw.role_endpoints[client]

    cleanup(ctx, client)

    ##
    user1 = 'foo'
    user2 = 'fud'
    user3 = 'bar'
    user4 = 'bud'
    subuser1 = 'foo:foo1'
    subuser2 = 'foo:foo2'
    display_name1 = 'Foo'
    display_name2 = 'Fud'
    display_name3 = 'Bar'
    email = '*****@*****.**'
    access_key = '9te6NH5mcdcq0Tc5i8i1'
    secret_key = 'Ny4IOauQoL18Gp2zM7lC1vLmoawgqcYP/YGcWfXu'
    access_key2 = 'p5YnriCv1nAtykxBrupQ'
    secret_key2 = 'Q8Tk6Q/27hfbFSYdSkPtUqhqx1GgzvpXa4WARozh'
    access_key3 = 'NX5QOQKC6BH2IDN8HC7A'
    secret_key3 = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn'
    swift_secret1 = 'gpS2G9RREMrnbqlp29PP2D36kgPR1tm72n5fPYfL'
    swift_secret2 = 'ri2VJQcKSYATOY6uaDUX7pxgkW+W1YmC6OCxPHwy'

    bucket_name = 'myfoo'
    bucket_name2 = 'mybar'

    # connect to rgw
    connection = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        is_secure=False,
        port=endpoint.port,
        host=endpoint.hostname,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )
    connection.auth_region_name = 'us-east-1'

    connection2 = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key2,
        aws_secret_access_key=secret_key2,
        is_secure=False,
        port=endpoint.port,
        host=endpoint.hostname,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )
    connection2.auth_region_name = 'us-east-1'

    connection3 = boto.s3.connection.S3Connection(
        aws_access_key_id=access_key3,
        aws_secret_access_key=secret_key3,
        is_secure=False,
        port=endpoint.port,
        host=endpoint.hostname,
        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
    )
    connection3.auth_region_name = 'us-east-1'

    acc = usage_acc()
    rl = requestlog_queue(acc.generate_make_entry())
    connection.set_request_hook(rl)
    connection2.set_request_hook(rl)
    connection3.set_request_hook(rl)

    # legend (test cases can be easily grep-ed out)
    # TESTCASE 'testname','object','method','operation','assertion'

    # TESTCASE 'usage-show0' 'usage' 'show' 'all usage' 'succeeds'
    (err, summary0) = rgwadmin(ctx,
                               client, ['usage', 'show'],
                               check_status=True)

    # TESTCASE 'info-nosuch','user','info','non-existent user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'create-ok','user','create','w/all valid info','succeeds'
    (err, out) = rgwadmin(
        ctx,
        client, [
            'user', 'create', '--uid', user1, '--display-name', display_name1,
            '--email', email, '--access-key', access_key, '--secret',
            secret_key, '--max-buckets', '4'
        ],
        check_status=True)

    # TESTCASE 'duplicate email','user','create','existing user email','fails'
    (err, out) = rgwadmin(ctx, client, [
        'user',
        'create',
        '--uid',
        user2,
        '--display-name',
        display_name2,
        '--email',
        email,
    ])
    assert err

    # TESTCASE 'info-existing','user','info','existing user','returns correct info'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert out['user_id'] == user1
    assert out['email'] == email
    assert out['display_name'] == display_name1
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key
    assert not out['suspended']

    # TESTCASE 'suspend-ok','user','suspend','active user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'suspend', '--uid', user1],
                          check_status=True)

    # TESTCASE 'suspend-suspended','user','suspend','suspended user','succeeds w/advisory'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert out['suspended']

    # TESTCASE 're-enable','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'enable', '--uid', user1],
                          check_status=True)

    # TESTCASE 'info-re-enabled','user','info','re-enabled user','no longer suspended'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert not out['suspended']

    # TESTCASE 'add-keys','key','create','w/valid info','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'create',
                              '--uid',
                              user1,
                              '--access-key',
                              access_key2,
                              '--secret',
                              secret_key2,
                          ],
                          check_status=True)

    # TESTCASE 'info-new-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['keys']) == 2
    assert out['keys'][0]['access_key'] == access_key2 or out['keys'][1][
        'access_key'] == access_key2
    assert out['keys'][0]['secret_key'] == secret_key2 or out['keys'][1][
        'secret_key'] == secret_key2

    # TESTCASE 'rm-key','key','rm','newly added key','succeeds, key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'rm',
                              '--uid',
                              user1,
                              '--access-key',
                              access_key2,
                          ],
                          check_status=True)
    assert len(out['keys']) == 1
    assert out['keys'][0]['access_key'] == access_key
    assert out['keys'][0]['secret_key'] == secret_key

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    subuser_access = 'full'
    subuser_perm = 'full-control'

    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser', 'create', '--subuser', subuser1,
                              '--access', subuser_access
                          ],
                          check_status=True)

    # TESTCASE 'add-swift-key','key','create','swift key','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'modify',
                              '--subuser',
                              subuser1,
                              '--secret',
                              swift_secret1,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)

    # TESTCASE 'subuser-perm-mask', 'subuser', 'info', 'test subuser perm mask durability', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])

    assert out['subusers'][0]['permissions'] == subuser_perm

    # TESTCASE 'info-swift-key','user','info','after key addition','returns all keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['swift_keys']) == 1
    assert out['swift_keys'][0]['user'] == subuser1
    assert out['swift_keys'][0]['secret_key'] == swift_secret1

    # TESTCASE 'add-swift-subuser','key','create','swift sub-user key','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'create',
                              '--subuser',
                              subuser2,
                              '--secret',
                              swift_secret2,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)

    # TESTCASE 'info-swift-subuser','user','info','after key addition','returns all sub-users/keys'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'info', '--uid', user1],
                          check_status=True)
    assert len(out['swift_keys']) == 2
    assert out['swift_keys'][0]['user'] == subuser2 or out['swift_keys'][1][
        'user'] == subuser2
    assert out['swift_keys'][0]['secret_key'] == swift_secret2 or out[
        'swift_keys'][1]['secret_key'] == swift_secret2

    # TESTCASE 'rm-swift-key1','key','rm','subuser','succeeds, one key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'key',
                              'rm',
                              '--subuser',
                              subuser1,
                              '--key-type',
                              'swift',
                          ],
                          check_status=True)
    assert len(out['swift_keys']) == 1

    # TESTCASE 'rm-subuser','subuser','rm','subuser','success, subuser is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'rm',
                              '--subuser',
                              subuser1,
                          ],
                          check_status=True)
    assert len(out['subusers']) == 1

    # TESTCASE 'rm-subuser-with-keys','subuser','rm','subuser','succeeds, second subser and key is removed'
    (err, out) = rgwadmin(ctx,
                          client, [
                              'subuser',
                              'rm',
                              '--subuser',
                              subuser2,
                              '--key-type',
                              'swift',
                              '--purge-keys',
                          ],
                          check_status=True)
    assert len(out['swift_keys']) == 0
    assert len(out['subusers']) == 0

    # TESTCASE 'bucket-stats','bucket','stats','no session/buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--uid', user1],
                          check_status=True)
    assert len(out) == 0

    # TESTCASE 'bucket-stats2','bucket','stats','no buckets','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'list', '--uid', user1],
                          check_status=True)
    assert len(out) == 0

    # create a first bucket
    bucket = connection.create_bucket(bucket_name)

    rl.log_and_clear("create_bucket", bucket_name, user1)

    # TESTCASE 'bucket-list','bucket','list','one bucket','succeeds, expected list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'list', '--uid', user1],
                          check_status=True)
    assert len(out) == 1
    assert out[0] == bucket_name

    bucket_list = connection.get_all_buckets()
    assert len(bucket_list) == 1
    assert bucket_list[0].name == bucket_name

    rl.log_and_clear("list_buckets", '', user1)

    # TESTCASE 'bucket-list-all','bucket','list','all buckets','succeeds, expected list'
    (err, out) = rgwadmin(ctx, client, ['bucket', 'list'], check_status=True)
    assert len(out) >= 1
    assert bucket_name in out

    # TESTCASE 'max-bucket-limit,'bucket','create','4 buckets','5th bucket fails due to max buckets == 4'
    bucket2 = connection.create_bucket(bucket_name + '2')
    rl.log_and_clear("create_bucket", bucket_name + '2', user1)
    bucket3 = connection.create_bucket(bucket_name + '3')
    rl.log_and_clear("create_bucket", bucket_name + '3', user1)
    bucket4 = connection.create_bucket(bucket_name + '4')
    rl.log_and_clear("create_bucket", bucket_name + '4', user1)
    # the 5th should fail.
    failed = False
    try:
        connection.create_bucket(bucket_name + '5')
    except Exception:
        failed = True
    assert failed
    rl.log_and_clear("create_bucket", bucket_name + '5', user1)

    # delete the buckets
    bucket2.delete()
    rl.log_and_clear("delete_bucket", bucket_name + '2', user1)
    bucket3.delete()
    rl.log_and_clear("delete_bucket", bucket_name + '3', user1)
    bucket4.delete()
    rl.log_and_clear("delete_bucket", bucket_name + '4', user1)

    # TESTCASE 'bucket-stats3','bucket','stats','new empty bucket','succeeds, empty list'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['owner'] == user1
    bucket_id = out['id']

    # TESTCASE 'bucket-stats4','bucket','stats','new empty bucket','succeeds, expected bucket ID'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--uid', user1],
                          check_status=True)
    assert len(out) == 1
    assert out[0][
        'id'] == bucket_id  # does it return the same ID twice in a row?

    # use some space
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('one')
    rl.log_and_clear("put_obj", bucket_name, user1)

    # TESTCASE 'bucket-stats5','bucket','stats','after creating key','succeeds, lists one non-empty object'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 1
    assert out['usage']['rgw.main']['size_kb'] > 0

    #validate we have a positive user stats now
    (err, out) = rgwadmin(ctx,
                          client,
                          ['user', 'stats', '--uid', user1, '--sync-stats'],
                          check_status=True)
    assert out['stats']['size'] > 0

    # reclaim it
    key.delete()
    rl.log_and_clear("delete_obj", bucket_name, user1)

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'fails', 'access denied error'
    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'unlink', '--uid', user1, '--bucket', bucket_name],
        check_status=True)

    # create a second user to link the bucket to
    (err, out) = rgwadmin(ctx,
                          client, [
                              'user',
                              'create',
                              '--uid',
                              user2,
                              '--display-name',
                              display_name2,
                              '--access-key',
                              access_key2,
                              '--secret',
                              secret_key2,
                              '--max-buckets',
                              '1',
                          ],
                          check_status=True)

    # try creating an object with the first user before the bucket is relinked
    denied = False
    key = boto.s3.key.Key(bucket)

    try:
        key.set_contents_from_string('two')
    except boto.exception.S3ResponseError:
        denied = True

    assert not denied
    rl.log_and_clear("put_obj", bucket_name, user1)

    # delete the object
    key.delete()
    rl.log_and_clear("delete_obj", bucket_name, user1)

    # link the bucket to another user
    (err,
     out) = rgwadmin(ctx,
                     client,
                     ['metadata', 'get', 'bucket:{n}'.format(n=bucket_name)],
                     check_status=True)

    bucket_data = out['data']
    assert bucket_data['bucket']['name'] == bucket_name

    bucket_id = bucket_data['bucket']['bucket_id']

    # link the bucket to another user
    (err, out) = rgwadmin(ctx,
                          client, [
                              'bucket', 'link', '--uid', user2, '--bucket',
                              bucket_name, '--bucket-id', bucket_id
                          ],
                          check_status=True)

    # try to remove user, should fail (has a linked bucket)
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user2])
    assert err

    # TESTCASE 'bucket unlink', 'bucket', 'unlink', 'unlink bucket from user', 'succeeds, bucket unlinked'
    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'unlink', '--uid', user2, '--bucket', bucket_name],
        check_status=True)

    # relink the bucket to the first user and delete the second user
    (err, out) = rgwadmin(ctx,
                          client, [
                              'bucket', 'link', '--uid', user1, '--bucket',
                              bucket_name, '--bucket-id', bucket_id
                          ],
                          check_status=True)

    (err, out) = rgwadmin(ctx,
                          client, ['user', 'rm', '--uid', user2],
                          check_status=True)

    #TESTCASE 'bucket link', 'bucket', 'tenanted user', 'succeeds'
    tenant_name = "testx"
    # create a tenanted user to link the bucket to
    (err, out) = rgwadmin(ctx,
                          client, [
                              'user',
                              'create',
                              '--tenant',
                              tenant_name,
                              '--uid',
                              'tenanteduser',
                              '--display-name',
                              'tenanted-user',
                              '--access-key',
                              access_key2,
                              '--secret',
                              secret_key2,
                              '--max-buckets',
                              '1',
                          ],
                          check_status=True)

    # link the bucket to a tenanted user
    (err, out) = rgwadmin(ctx,
                          client, [
                              'bucket', 'link', '--bucket', '/' + bucket_name,
                              '--tenant', tenant_name, '--uid', 'tenanteduser'
                          ],
                          check_status=True)

    # check if the bucket name has tenant/ prefix
    (err,
     out) = rgwadmin(ctx,
                     client, [
                         'metadata', 'get',
                         'bucket:{n}'.format(n=tenant_name + '/' + bucket_name)
                     ],
                     check_status=True)

    bucket_data = out['data']
    assert bucket_data['bucket']['name'] == bucket_name
    assert bucket_data['bucket']['tenant'] == tenant_name

    # relink the bucket to the first user and delete the tenanted user
    (err, out) = rgwadmin(ctx,
                          client, [
                              'bucket', 'link', '--bucket',
                              tenant_name + '/' + bucket_name, '--uid', user1
                          ],
                          check_status=True)

    (err, out) = rgwadmin(
        ctx,
        client,
        ['user', 'rm', '--tenant', tenant_name, '--uid', 'tenanteduser'],
        check_status=True)

    # TESTCASE 'object-rm', 'object', 'rm', 'remove object', 'succeeds, object is removed'

    # upload an object
    object_name = 'four'
    key = boto.s3.key.Key(bucket, object_name)
    key.set_contents_from_string(object_name)
    rl.log_and_clear("put_obj", bucket_name, user1)

    # fetch it too (for usage stats presently)
    s = key.get_contents_as_string(encoding='ascii')
    rl.log_and_clear("get_obj", bucket_name, user1)
    assert s == object_name
    # list bucket too (for usage stats presently)
    keys = list(bucket.list())
    rl.log_and_clear("list_bucket", bucket_name, user1)
    assert len(keys) == 1
    assert keys[0].name == object_name

    # now delete it
    (err, out) = rgwadmin(
        ctx,
        client,
        ['object', 'rm', '--bucket', bucket_name, '--object', object_name],
        check_status=True)

    # TESTCASE 'bucket-stats6','bucket','stats','after deleting key','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx,
                          client, ['bucket', 'stats', '--bucket', bucket_name],
                          check_status=True)
    assert out['id'] == bucket_id
    assert out['usage']['rgw.main']['num_objects'] == 0

    # list log objects
    # TESTCASE 'log-list','log','list','after activity','succeeds, lists one no objects'
    (err, out) = rgwadmin(ctx, client, ['log', 'list'], check_status=True)
    assert len(out) > 0

    for obj in out:
        # TESTCASE 'log-show','log','show','after activity','returns expected info'
        if obj[:
               4] == 'meta' or obj[:
                                   4] == 'data' or obj[:
                                                       18] == 'obj_delete_at_hint':
            continue

        (err, rgwlog) = rgwadmin(ctx,
                                 client, ['log', 'show', '--object', obj],
                                 check_status=True)
        assert len(rgwlog) > 0

        # skip any entry for which there is no bucket name--e.g., list_buckets,
        # since that is valid but cannot pass the following checks
        entry_bucket_name = rgwlog['bucket']
        if entry_bucket_name.strip() != "":
            # exempt bucket_name2 from checking as it was only used for multi-region tests
            assert rgwlog['bucket'].find(
                bucket_name) == 0 or rgwlog['bucket'].find(bucket_name2) == 0
            assert rgwlog['bucket'] != bucket_name or rgwlog[
                'bucket_id'] == bucket_id
            assert rgwlog['bucket_owner'] == user1 or rgwlog[
                'bucket'] == bucket_name + '5' or rgwlog[
                    'bucket'] == bucket_name2
            for entry in rgwlog['log_entries']:
                log.debug('checking log entry: ', entry)
                assert entry['bucket'] == rgwlog['bucket']
                possible_buckets = [bucket_name + '5', bucket_name2]
                user = entry['user']
                assert user == user1 or user.endswith('system-user') or \
                    rgwlog['bucket'] in possible_buckets

        # TESTCASE 'log-rm','log','rm','delete log objects','succeeds'
        (err, out) = rgwadmin(ctx,
                              client, ['log', 'rm', '--object', obj],
                              check_status=True)

    # TODO: show log by bucket+date

    # TESTCASE 'user-suspend2','user','suspend','existing user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'suspend', '--uid', user1],
                          check_status=True)

    # TESTCASE 'user-suspend3','user','suspend','suspended user','cannot write objects'
    denied = False
    try:
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string('five')
    except boto.exception.S3ResponseError as e:
        denied = True
        assert e.status == 403

    assert denied
    rl.log_and_clear("put_obj", bucket_name, user1)

    # TESTCASE 'user-renable2','user','enable','suspended user','succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['user', 'enable', '--uid', user1],
                          check_status=True)

    # TESTCASE 'user-renable3','user','enable','reenabled user','can write objects'
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('six')
    rl.log_and_clear("put_obj", bucket_name, user1)

    # TESTCASE 'gc-list', 'gc', 'list', 'get list of objects ready for garbage collection'

    # create an object large enough to be split into multiple parts
    test_string = 'foo' * 10000000

    big_key = boto.s3.key.Key(bucket)
    big_key.set_contents_from_string(test_string)
    rl.log_and_clear("put_obj", bucket_name, user1)

    # now delete the head
    big_key.delete()
    rl.log_and_clear("delete_obj", bucket_name, user1)

    # wait a bit to give the garbage collector time to cycle
    time.sleep(15)

    (err, out) = rgwadmin(ctx, client, ['gc', 'list', '--include-all'])
    assert len(out) > 0

    # TESTCASE 'gc-process', 'gc', 'process', 'manually collect garbage'
    (err, out) = rgwadmin(ctx, client, ['gc', 'process'], check_status=True)

    #confirm
    (err, out) = rgwadmin(ctx, client, ['gc', 'list', '--include-all'])

    # don't assume rgw_gc_obj_min_wait has been overridden
    omit_tdir = hasattr(ctx.rgw, 'omit_tdir') and ctx.rgw.omit_tdir == True
    if omit_tdir == False:
        assert len(out) == 0

    # TESTCASE 'rm-user-buckets','user','rm','existing user','fails, still has buckets'
    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # delete should fail because ``key`` still exists
    try:
        bucket.delete()
    except boto.exception.S3ResponseError as e:
        assert e.status == 409
    rl.log_and_clear("delete_bucket", bucket_name, user1)

    key.delete()
    rl.log_and_clear("delete_obj", bucket_name, user1)
    bucket.delete()
    rl.log_and_clear("delete_bucket", bucket_name, user1)

    # TESTCASE 'policy', 'bucket', 'policy', 'get bucket policy', 'returns S3 policy'
    bucket = connection.create_bucket(bucket_name)
    rl.log_and_clear("create_bucket", bucket_name, user1)

    # create an object
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('seven')
    rl.log_and_clear("put_obj", bucket_name, user1)

    # should be private already but guarantee it
    key.set_acl('private')
    rl.log_and_clear("put_acls", bucket_name, user1)

    (err, out) = rgwadmin(
        ctx,
        client,
        ['policy', '--bucket', bucket.name, '--object',
         key.key.decode()],
        check_status=True,
        format='xml')

    acl = get_acl(key)
    rl.log_and_clear("get_acls", bucket_name, user1)

    assert acl == out.strip('\n')

    # add another grantee by making the object public read
    key.set_acl('public-read')
    rl.log_and_clear("put_acls", bucket_name, user1)

    (err, out) = rgwadmin(
        ctx,
        client,
        ['policy', '--bucket', bucket.name, '--object',
         key.key.decode()],
        check_status=True,
        format='xml')

    acl = get_acl(key)
    rl.log_and_clear("get_acls", bucket_name, user1)

    assert acl == out.strip('\n')

    # TESTCASE 'rm-bucket', 'bucket', 'rm', 'bucket with objects', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    rl.log_and_clear("create_bucket", bucket_name, user1)
    key_name = ['eight', 'nine', 'ten', 'eleven']
    for i in range(4):
        key = boto.s3.key.Key(bucket)
        key.set_contents_from_string(key_name[i])
    rl.log_and_clear("put_obj", bucket_name, user1)

    (err, out) = rgwadmin(
        ctx,
        client, ['bucket', 'rm', '--bucket', bucket_name, '--purge-objects'],
        check_status=True)

    # TESTCASE 'caps-add', 'caps', 'add', 'add user cap', 'succeeds'
    caps = 'user=read'
    (err, out) = rgwadmin(ctx, client,
                          ['caps', 'add', '--uid', user1, '--caps', caps])

    assert out['caps'][0]['perm'] == 'read'

    # TESTCASE 'caps-rm', 'caps', 'rm', 'remove existing cap from user', 'succeeds'
    (err, out) = rgwadmin(ctx, client,
                          ['caps', 'rm', '--uid', user1, '--caps', caps])

    assert not out['caps']

    # TESTCASE 'rm-user','user','rm','existing user','fails, still has buckets'
    bucket = connection.create_bucket(bucket_name)
    rl.log_and_clear("create_bucket", bucket_name, user1)
    key = boto.s3.key.Key(bucket)

    (err, out) = rgwadmin(ctx, client, ['user', 'rm', '--uid', user1])
    assert err

    # TESTCASE 'rm-user2', 'user', 'rm', 'user with data', 'succeeds'
    bucket = connection.create_bucket(bucket_name)
    rl.log_and_clear("create_bucket", bucket_name, user1)
    key = boto.s3.key.Key(bucket)
    key.set_contents_from_string('twelve')
    rl.log_and_clear("put_obj", bucket_name, user1)

    time.sleep(35)

    # need to wait for all usage data to get flushed, should take up to 30 seconds
    timestamp = time.time()
    while time.time() - timestamp <= (2 * 60):  # wait up to 20 minutes
        (err, out) = rgwadmin(ctx, client, [
            'usage', 'show', '--categories', 'delete_obj'
        ])  # one of the operations we did is delete_obj, should be present.
        if get_user_successful_ops(out, user1) > 0:
            break
        time.sleep(1)

    assert time.time() - timestamp <= (20 * 60)

    # TESTCASE 'usage-show' 'usage' 'show' 'all usage' 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['usage', 'show'], check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0

    r = acc.compare_results(out)
    if len(r) != 0:
        sys.stderr.write(("\n".join(r)) + "\n")
        assert (len(r) == 0)

    user_summary = get_user_summary(out, user1)

    total = user_summary['total']
    assert total['successful_ops'] > 0

    # TESTCASE 'usage-show2' 'usage' 'show' 'user usage' 'succeeds'
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'show', '--uid', user1],
                          check_status=True)
    assert len(out['entries']) > 0
    assert len(out['summary']) > 0
    user_summary = out['summary'][0]
    for entry in user_summary['categories']:
        assert entry['successful_ops'] > 0
    assert user_summary['user'] == user1

    # TESTCASE 'usage-show3' 'usage' 'show' 'user usage categories' 'succeeds'
    test_categories = [
        'create_bucket', 'put_obj', 'delete_obj', 'delete_bucket'
    ]
    for cat in test_categories:
        (err, out) = rgwadmin(
            ctx,
            client, ['usage', 'show', '--uid', user1, '--categories', cat],
            check_status=True)
        assert len(out['summary']) > 0
        user_summary = out['summary'][0]
        assert user_summary['user'] == user1
        assert len(user_summary['categories']) == 1
        entry = user_summary['categories'][0]
        assert entry['category'] == cat
        assert entry['successful_ops'] > 0

    # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'new user', 'succeeds'
    # create a new user user3
    (err,
     out) = rgwadmin(ctx,
                     client, [
                         'user', 'create', '--uid', user3, '--display-name',
                         display_name3, '--access-key', access_key3,
                         '--secret', secret_key3, '--max-buckets', '4'
                     ],
                     check_status=True)

    # create a bucket
    bucket = connection3.create_bucket(bucket_name + '6')

    rl.log_and_clear("create_bucket", bucket_name + '6', user3)

    # create object
    object_name1 = 'thirteen'
    key1 = boto.s3.key.Key(bucket, object_name1)
    key1.set_contents_from_string(object_name1)
    rl.log_and_clear("put_obj", bucket_name + '6', user3)

    # rename user3
    (err,
     out) = rgwadmin(ctx,
                     client,
                     ['user', 'rename', '--uid', user3, '--new-uid', user4],
                     check_status=True)
    assert out['user_id'] == user4
    assert out['keys'][0]['access_key'] == access_key3
    assert out['keys'][0]['secret_key'] == secret_key3

    time.sleep(5)

    # get bucket and object to test if user keys are preserved
    bucket = connection3.get_bucket(bucket_name + '6')
    s = key1.get_contents_as_string(encoding='ascii')
    rl.log_and_clear("get_obj", bucket_name + '6', user4)
    assert s == object_name1

    # TESTCASE 'user-rename', 'user', 'rename', 'existing user', 'another existing user', 'fails'
    # create a new user user2
    (err,
     out) = rgwadmin(ctx,
                     client, [
                         'user', 'create', '--uid', user2, '--display-name',
                         display_name2, '--access-key', access_key2,
                         '--secret', secret_key2, '--max-buckets', '4'
                     ],
                     check_status=True)

    # create a bucket
    bucket = connection2.create_bucket(bucket_name + '7')

    rl.log_and_clear("create_bucket", bucket_name + '7', user2)

    # create object
    object_name2 = 'fourteen'
    key2 = boto.s3.key.Key(bucket, object_name2)
    key2.set_contents_from_string(object_name2)
    rl.log_and_clear("put_obj", bucket_name + '7', user2)

    (err,
     out) = rgwadmin(ctx, client,
                     ['user', 'rename', '--uid', user4, '--new-uid', user2])
    assert err

    # test if user 2 and user4 can still access their bucket and objects after rename fails
    bucket = connection3.get_bucket(bucket_name + '6')
    s = key1.get_contents_as_string(encoding='ascii')
    rl.log_and_clear("get_obj", bucket_name + '6', user4)
    assert s == object_name1

    bucket = connection2.get_bucket(bucket_name + '7')
    s = key2.get_contents_as_string(encoding='ascii')
    rl.log_and_clear("get_obj", bucket_name + '7', user2)
    assert s == object_name2

    (err, out) = rgwadmin(ctx,
                          client,
                          ['user', 'rm', '--uid', user4, '--purge-data'],
                          check_status=True)

    (err, out) = rgwadmin(ctx,
                          client,
                          ['user', 'rm', '--uid', user2, '--purge-data'],
                          check_status=True)

    time.sleep(5)

    # should be all through with connection. (anything using connection
    #  should be BEFORE the usage stuff above.)
    rl.log_and_clear("(before-close)", '-', '-', ignore_this_entry)
    connection.close()
    connection = None

    # the usage flush interval is 30 seconds, wait that much an then some
    # to make sure everything has been flushed
    time.sleep(35)

    # TESTCASE 'usage-trim' 'usage' 'trim' 'user usage' 'succeeds, usage removed'
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'trim', '--uid', user1],
                          check_status=True)
    (err, out) = rgwadmin(ctx,
                          client, ['usage', 'show', '--uid', user1],
                          check_status=True)
    assert len(out['entries']) == 0
    assert len(out['summary']) == 0

    (err, out) = rgwadmin(ctx,
                          client,
                          ['user', 'rm', '--uid', user1, '--purge-data'],
                          check_status=True)

    # TESTCASE 'rm-user3','user','rm','deleted user','fails'
    (err, out) = rgwadmin(ctx, client, ['user', 'info', '--uid', user1])
    assert err

    # TESTCASE 'zone-info', 'zone', 'get', 'get zone info', 'succeeds, has default placement rule'
    (err, out) = rgwadmin(ctx, client,
                          ['zone', 'get', '--rgw-zone', 'default'])
    orig_placement_pools = len(out['placement_pools'])

    # removed this test, it is not correct to assume that zone has default placement, it really
    # depends on how we set it up before
    #
    # assert len(out) > 0
    # assert len(out['placement_pools']) == 1

    # default_rule = out['placement_pools'][0]
    # assert default_rule['key'] == 'default-placement'

    rule = {
        'key': 'new-placement',
        'val': {
            'data_pool': '.rgw.buckets.2',
            'index_pool': '.rgw.buckets.index.2'
        }
    }

    out['placement_pools'].append(rule)

    (err, out) = rgwadmin(ctx,
                          client, ['zone', 'set'],
                          stdin=StringIO(json.dumps(out)),
                          check_status=True)

    (err, out) = rgwadmin(ctx, client, ['zone', 'get'])
    assert len(out) > 0
    assert len(out['placement_pools']) == orig_placement_pools + 1

    zonecmd = [
        'zone', 'placement', 'rm', '--rgw-zone', 'default', '--placement-id',
        'new-placement'
    ]

    (err, out) = rgwadmin(ctx, client, zonecmd, check_status=True)

    # TESTCASE 'zonegroup-info', 'zonegroup', 'get', 'get zonegroup info', 'succeeds'
    (err, out) = rgwadmin(ctx, client, ['zonegroup', 'get'], check_status=True)