Exemple #1
0
 def test_headers_to_account_info_storage_policies(self):
     headers = {
         'x-account-storage-policy-zero-object-count': '13',
         'x-account-storage-policy-zero-container-count': '120',
         'x-account-storage-policy-zero-bytes-used': '1002',
         'x-account-storage-policy-one-object-count': '10',
         'x-account-storage-policy-one-container-count': '20',
     }
     spc = StoragePolicyCollection(
         [StoragePolicy(0, 'zero', True),
          StoragePolicy(1, 'one', False)])
     with PatchPolicies(spc):
         resp = headers_to_account_info(headers.items(), 200)
     self.assertEqual(
         resp['storage_policies'], {
             0: {
                 'object_count': 13,
                 'container_count': 120,
                 'bytes': 1002
             },
             1: {
                 'object_count': 10,
                 'container_count': 20,
                 'bytes': 0
             },
         })
Exemple #2
0
 def test_storage_policy_repr(self):
     test_policies = [
         StoragePolicy(0, 'aay', True),
         StoragePolicy(1, 'bee', False),
         StoragePolicy(2, 'cee', False),
         ECStoragePolicy(10,
                         'ten',
                         ec_type='jerasure_rs_vand',
                         ec_ndata=10,
                         ec_nparity=3)
     ]
     policies = StoragePolicyCollection(test_policies)
     for policy in policies:
         policy_repr = repr(policy)
         self.assert_(policy.__class__.__name__ in policy_repr)
         self.assert_('is_default=%s' % policy.is_default in policy_repr)
         self.assert_('is_deprecated=%s' %
                      policy.is_deprecated in policy_repr)
         self.assert_(policy.name in policy_repr)
         if policy.policy_type == EC_POLICY:
             self.assert_('ec_type=%s' % policy.ec_type in policy_repr)
             self.assert_('ec_ndata=%s' % policy.ec_ndata in policy_repr)
             self.assert_('ec_nparity=%s' %
                          policy.ec_nparity in policy_repr)
             self.assert_('ec_segment_size=%s' %
                          policy.ec_segment_size in policy_repr)
     collection_repr = repr(policies)
     collection_repr_lines = collection_repr.splitlines()
     self.assert_(policies.__class__.__name__ in collection_repr_lines[0])
     self.assertEqual(len(policies), len(collection_repr_lines[1:-1]))
     for policy, line in zip(policies, collection_repr_lines[1:-1]):
         self.assert_(repr(policy) in line)
     with patch_policies(policies):
         self.assertEqual(repr(POLICIES), collection_repr)
    def test_get_object_ring(self):
        test_policies = [
            StoragePolicy(0, 'aay', True),
            StoragePolicy(1, 'bee', False),
            StoragePolicy(2, 'cee', False)
        ]
        policies = StoragePolicyCollection(test_policies)

        class NamedFakeRing(FakeRing):
            def __init__(self, swift_dir, ring_name=None):
                self.ring_name = ring_name
                super(NamedFakeRing, self).__init__()

        with mock.patch('swift.common.storage_policy.Ring', new=NamedFakeRing):
            for policy in policies:
                self.assertFalse(policy.object_ring)
                ring = policies.get_object_ring(int(policy), '/path/not/used')
                self.assertEqual(ring.ring_name, policy.ring_name)
                self.assertTrue(policy.object_ring)
                self.assert_(isinstance(policy.object_ring, NamedFakeRing))

        def blow_up(*args, **kwargs):
            raise Exception('kaboom!')

        with mock.patch('swift.common.storage_policy.Ring', new=blow_up):
            for policy in policies:
                policy.load_ring('/path/not/used')
                expected = policies.get_object_ring(int(policy),
                                                    '/path/not/used')
                self.assertEqual(policy.object_ring, expected)

        # bad policy index
        self.assertRaises(PolicyError, policies.get_object_ring, 99,
                          '/path/not/used')
Exemple #4
0
 def test_validate_policies_names(self):
     # duplicate names
     test_policies = [StoragePolicy(0, 'zero', True),
                      StoragePolicy(1, 'zero', False),
                      StoragePolicy(2, 'two', False)]
     self.assertRaises(PolicyError, StoragePolicyCollection,
                       test_policies)
Exemple #5
0
def patch_policies(thing_or_policies=None, legacy_only=False,
                   with_ec_default=False, fake_ring_args=None):
    if isinstance(thing_or_policies, (
            Iterable, storage_policy.StoragePolicyCollection)):
        return PatchPolicies(thing_or_policies, fake_ring_args=fake_ring_args)

    if legacy_only:
        default_policies = [
            StoragePolicy(0, name='legacy', is_default=True),
        ]
        default_ring_args = [{}]
    elif with_ec_default:
        default_policies = [
            ECStoragePolicy(0, name='ec', is_default=True,
                            ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10,
                            ec_nparity=4, ec_segment_size=4096),
            StoragePolicy(1, name='unu'),
        ]
        default_ring_args = [{'replicas': 14}, {}]
    else:
        default_policies = [
            StoragePolicy(0, name='nulo', is_default=True),
            StoragePolicy(1, name='unu'),
        ]
        default_ring_args = [{}, {}]

    fake_ring_args = fake_ring_args or default_ring_args
    decorator = PatchPolicies(default_policies, fake_ring_args=fake_ring_args)

    if not thing_or_policies:
        return decorator
    else:
        # it's a thing, we return the wrapped thing instead of the decorator
        return decorator(thing_or_policies)
Exemple #6
0
    def test_get_ring_names(self):
        self.recon_instance.server_type = 'not-object'
        self.assertEqual(self.recon_instance._get_ring_names(), ['not-object'])

        self.recon_instance.server_type = 'object'

        with patch_policies([StoragePolicy(0, 'zero', is_default=True)]):
            self.assertEqual(self.recon_instance._get_ring_names(), ['object'])

        with patch_policies([
                StoragePolicy(0, 'zero', is_default=True),
                StoragePolicy(1, 'one')
        ]):
            self.assertEqual(self.recon_instance._get_ring_names(),
                             ['object', 'object-1'])
            self.assertEqual(self.recon_instance._get_ring_names('0'),
                             ['object'])
            self.assertEqual(self.recon_instance._get_ring_names('zero'),
                             ['object'])
            self.assertEqual(self.recon_instance._get_ring_names('1'),
                             ['object-1'])
            self.assertEqual(self.recon_instance._get_ring_names('one'),
                             ['object-1'])

            self.assertEqual(self.recon_instance._get_ring_names('3'), [])
            self.assertEqual(self.recon_instance._get_ring_names('wrong'), [])
Exemple #7
0
 def test_singleton_passthrough(self):
     test_policies = [StoragePolicy(0, 'aay', True),
                      StoragePolicy(1, 'bee', False),
                      StoragePolicy(2, 'cee', False)]
     with patch_policies(test_policies):
         for policy in POLICIES:
             self.assertEqual(POLICIES[int(policy)], policy)
Exemple #8
0
    def test_storage_policy_get_options(self):
        policy = StoragePolicy(1, 'gold', True, False)
        self.assertEqual({'name': 'gold',
                          'default': True,
                          'deprecated': False},
                         policy.get_options())

        policy = StoragePolicy(1, 'gold', False, True)
        self.assertEqual({'name': 'gold',
                          'default': False,
                          'deprecated': True},
                         policy.get_options())
    def test_storage_policy_ordering(self):
        test_policies = StoragePolicyCollection([
            StoragePolicy(0, 'zero', is_default=True),
            StoragePolicy(503, 'error'),
            StoragePolicy(204, 'empty'),
            StoragePolicy(404, 'missing'),
        ])
        self.assertEqual([0, 204, 404, 503],
                         [int(p) for p in sorted(list(test_policies))])

        p503 = test_policies[503]
        self.assertTrue(501 < p503 < 507)
Exemple #10
0
 def test_validate_policy_params(self):
     StoragePolicy(0, 'name')  # sanity
     # bogus indexes
     self.assertRaises(PolicyError, StoragePolicy, 'x', 'name')
     self.assertRaises(PolicyError, StoragePolicy, -1, 'name')
     # non-zero Policy-0
     self.assertRaisesWithMessage(PolicyError, 'reserved', StoragePolicy, 1,
                                  'policy-0')
     # deprecate default
     self.assertRaisesWithMessage(PolicyError,
                                  'Deprecated policy can not be default',
                                  StoragePolicy,
                                  1,
                                  'Policy-1',
                                  is_default=True,
                                  is_deprecated=True)
     # weird names
     names = (
         '',
         'name_foo',
         'name\nfoo',
         'name foo',
         u'name \u062a',
         'name \xd8\xaa',
     )
     for name in names:
         self.assertRaisesWithMessage(PolicyError, 'Invalid name',
                                      StoragePolicy, 1, name)
Exemple #11
0
    def test_names_are_normalized(self):
        test_policies = [StoragePolicy(0, 'zero', True),
                         StoragePolicy(1, 'ZERO', False)]
        self.assertRaises(PolicyError, StoragePolicyCollection,
                          test_policies)

        policies = StoragePolicyCollection([StoragePolicy(0, 'zEro', True),
                                            StoragePolicy(1, 'One', False)])

        pol0 = policies[0]
        pol1 = policies[1]

        for name in ('zero', 'ZERO', 'zErO', 'ZeRo'):
            self.assertEqual(pol0, policies.get_by_name(name))
            self.assertEqual(policies.get_by_name(name).name, 'zEro')
        for name in ('one', 'ONE', 'oNe', 'OnE'):
            self.assertEqual(pol1, policies.get_by_name(name))
            self.assertEqual(policies.get_by_name(name).name, 'One')
Exemple #12
0
 def test_policies_type_attribute(self):
     test_policies = [
         StoragePolicy(0, 'zero', is_default=True),
         StoragePolicy(1, 'one'),
         StoragePolicy(2, 'two'),
         StoragePolicy(3, 'three', is_deprecated=True),
         ECStoragePolicy(10,
                         'ten',
                         ec_type='jerasure_rs_vand',
                         ec_ndata=10,
                         ec_nparity=3),
     ]
     policies = StoragePolicyCollection(test_policies)
     self.assertEquals(policies.get_by_index(0).policy_type, REPL_POLICY)
     self.assertEquals(policies.get_by_index(1).policy_type, REPL_POLICY)
     self.assertEquals(policies.get_by_index(2).policy_type, REPL_POLICY)
     self.assertEquals(policies.get_by_index(3).policy_type, REPL_POLICY)
     self.assertEquals(policies.get_by_index(10).policy_type, EC_POLICY)
Exemple #13
0
 def test_quorum_size_replication(self):
     expected_sizes = {1: 1,
                       2: 2,
                       3: 2,
                       4: 3,
                       5: 3}
     for n, expected in expected_sizes.items():
         policy = StoragePolicy(0, 'zero',
                                object_ring=FakeRing(replicas=n))
         self.assertEqual(policy.quorum, expected)
Exemple #14
0
 def test_storage_policy_repr(self):
     test_policies = [
         StoragePolicy(0, 'aay', True),
         StoragePolicy(1, 'bee', False),
         StoragePolicy(2, 'cee', False)
     ]
     policies = StoragePolicyCollection(test_policies)
     for policy in policies:
         policy_repr = repr(policy)
         self.assert_(policy.__class__.__name__ in policy_repr)
         self.assert_('is_default=%s' % policy.is_default in policy_repr)
         self.assert_('is_deprecated=%s' %
                      policy.is_deprecated in policy_repr)
         self.assert_(policy.name in policy_repr)
     collection_repr = repr(policies)
     collection_repr_lines = collection_repr.splitlines()
     self.assert_(policies.__class__.__name__ in collection_repr_lines[0])
     self.assertEqual(len(policies), len(collection_repr_lines[1:-1]))
     for policy, line in zip(policies, collection_repr_lines[1:-1]):
         self.assert_(repr(policy) in line)
     with patch_policies(policies):
         self.assertEqual(repr(POLICIES), collection_repr)
Exemple #15
0
    def setUp(self):
        skip_if_no_xattrs()
        self.logger = FakeLogger()
        self.testdir = tempfile.mkdtemp()
        self.devices = os.path.join(self.testdir, 'node')
        shutil.rmtree(self.testdir, ignore_errors=1)
        os.mkdir(self.testdir)
        os.mkdir(self.devices)

        self.rb = ring.RingBuilder(8, 6.0, 1)

        for i in range(6):
            ip = "127.0.0.%s" % i
            self.rb.add_dev({
                'id': i,
                'region': 0,
                'zone': 0,
                'weight': 1,
                'ip': ip,
                'port': 10000,
                'device': 'sda1'
            })
        self.rb.rebalance(seed=1)

        self.existing_device = 'sda1'
        os.mkdir(os.path.join(self.devices, self.existing_device))
        self.objects = os.path.join(self.devices, self.existing_device,
                                    'objects')
        os.mkdir(self.objects)
        self._hash = utils.hash_path('a/c/o')
        digest = binascii.unhexlify(self._hash)
        part = struct.unpack_from('>I', digest)[0] >> 24
        self.next_part = struct.unpack_from('>I', digest)[0] >> 23
        self.objdir = os.path.join(self.objects, str(part), self._hash[-3:],
                                   self._hash)
        os.makedirs(self.objdir)
        self.object_fname = "1278553064.00000.data"
        self.objname = os.path.join(self.objdir, self.object_fname)
        with open(self.objname, "wb") as dummy:
            dummy.write(b"Hello World!")
            write_metadata(dummy, {'name': '/a/c/o', 'Content-Length': '12'})

        test_policies = [StoragePolicy(0, 'platin', True)]
        storage_policy._POLICIES = StoragePolicyCollection(test_policies)

        self.expected_dir = os.path.join(self.objects, str(self.next_part),
                                         self._hash[-3:], self._hash)
        self.expected_file = os.path.join(self.expected_dir, self.object_fname)
Exemple #16
0
    def _setup_object(self, condition=None):
        attempts = []
        for _ in range(50):
            account = 'a'
            container = 'c'
            obj = 'o-' + str(uuid.uuid4())
            self._hash = utils.hash_path(account, container, obj)
            digest = binascii.unhexlify(self._hash)
            self.part = struct.unpack_from('>I', digest)[0] >> 24
            self.next_part = struct.unpack_from('>I', digest)[0] >> 23
            path = os.path.join(os.path.sep, account, container, obj)
            # There's 1/512 chance that both old and new parts will be 0;
            # that's not a terribly interesting case, as there's nothing to do
            attempts.append((self.part, self.next_part, 2**PART_POWER))
            if (self.part != self.next_part
                    and (condition(self.part) if condition else True)):
                break
        else:
            self.fail(
                'Failed to setup object satisfying test preconditions %s' %
                attempts)

        shutil.rmtree(self.objects, ignore_errors=True)
        os.mkdir(self.objects)
        self.objdir = os.path.join(self.objects, str(self.part),
                                   self._hash[-3:], self._hash)
        os.makedirs(self.objdir)
        self.object_fname = utils.Timestamp.now().internal + ".data"

        self.objname = os.path.join(self.objdir, self.object_fname)
        with open(self.objname, "wb") as dummy:
            dummy.write(b"Hello World!")
            write_metadata(dummy, {'name': path, 'Content-Length': '12'})

        self.policy = StoragePolicy(0, 'platinum', True)
        storage_policy._POLICIES = StoragePolicyCollection([self.policy])

        self.part_dir = os.path.join(self.objects, str(self.part))
        self.suffix_dir = os.path.join(self.part_dir, self._hash[-3:])
        self.next_part_dir = os.path.join(self.objects, str(self.next_part))
        self.next_suffix_dir = os.path.join(self.next_part_dir,
                                            self._hash[-3:])
        self.expected_dir = os.path.join(self.next_suffix_dir, self._hash)
        self.expected_file = os.path.join(self.expected_dir, self.object_fname)
Exemple #17
0
    def test_storage_policy_get_options(self):
        policy = StoragePolicy(1, 'gold', True, False)
        self.assertEqual({
            'name': 'gold',
            'default': True,
            'deprecated': False
        }, policy.get_options())

        policy = StoragePolicy(1, 'gold', False, True)
        self.assertEqual({
            'name': 'gold',
            'default': False,
            'deprecated': True
        }, policy.get_options())
    def test_load_from_config(self, tempdir):
        conf_path = os.path.join(tempdir, 'interal_client.conf')
        conf_body = """
        [DEFAULT]
        swift_dir = %s

        [pipeline:main]
        pipeline = catch_errors cache proxy-server

        [app:proxy-server]
        use = egg:swift#proxy
        auto_create_account_prefix = -

        [filter:cache]
        use = egg:swift#memcache

        [filter:catch_errors]
        use = egg:swift#catch_errors
        """ % tempdir
        with open(conf_path, 'w') as f:
            f.write(dedent(conf_body))
        account_ring_path = os.path.join(tempdir, 'account.ring.gz')
        write_fake_ring(account_ring_path)
        container_ring_path = os.path.join(tempdir, 'container.ring.gz')
        write_fake_ring(container_ring_path)
        object_ring_path = os.path.join(tempdir, 'object.ring.gz')
        write_fake_ring(object_ring_path)
        with patch_policies([StoragePolicy(0, 'legacy', True)]):
            client = internal_client.InternalClient(conf_path, 'test', 1)
        self.assertEqual(client.account_ring, client.app.app.app.account_ring)
        self.assertEqual(client.account_ring.serialized_path,
                         account_ring_path)
        self.assertEqual(client.container_ring,
                         client.app.app.app.container_ring)
        self.assertEqual(client.container_ring.serialized_path,
                         container_ring_path)
        object_ring = client.app.app.app.get_object_ring(0)
        self.assertEqual(client.get_object_ring(0),
                         object_ring)
        self.assertEqual(object_ring.serialized_path,
                         object_ring_path)
        self.assertEquals(client.auto_create_account_prefix, '-')
Exemple #19
0
    def test_deprecate_policies(self):
        # deprecation specified
        test_policies = [StoragePolicy(0, 'zero', True),
                         StoragePolicy(1, 'one', False),
                         StoragePolicy(2, 'two', False, is_deprecated=True)]
        policies = StoragePolicyCollection(test_policies)
        self.assertEqual(policies.default, test_policies[0])
        self.assertEqual(policies.default.name, 'zero')
        self.assertEqual(len(policies), 3)

        # multiple policies requires default
        test_policies = [StoragePolicy(0, 'zero', False),
                         StoragePolicy(1, 'one', False, is_deprecated=True),
                         StoragePolicy(2, 'two', False)]
        self.assertRaisesWithMessage(
            PolicyError, 'Unable to find default policy',
            StoragePolicyCollection, test_policies)
Exemple #20
0

class FakeAppReadline(object):

    def __call__(self, env, start_response):
        start_response('200 OK', [('Content-Type', 'text/plain'),
                                  ('Content-Length', '8')])
        env['wsgi.input'].readline()
        return ["FAKE APP"]


def start_response(*args):
    pass


@patch_policies([StoragePolicy(0, 'zero', False)])
class TestProxyLogging(unittest.TestCase):

    def setUp(self):
        pass

    def _log_parts(self, app, should_be_empty=False):
        info_calls = app.access_logger.log_dict['info']
        if should_be_empty:
            self.assertEquals([], info_calls)
        else:
            self.assertEquals(1, len(info_calls))
            return info_calls[0][0][0].split(' ')

    def assertTiming(self, exp_metric, app, exp_timing=None):
        timing_calls = app.access_logger.log_dict['timing']
Exemple #21
0
                       [(k, v) for k, v in response.headers.items()])
        return iter(response.body)


class FakeCache(FakeMemcache):
    def __init__(self, stub=None, **pre_cached):
        super(FakeCache, self).__init__()
        if pre_cached:
            self.store.update(pre_cached)
        self.stub = stub

    def get(self, key):
        return self.stub or self.store.get(key)


@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
class TestFuncs(unittest.TestCase):
    def setUp(self):
        self.app = proxy_server.Application(None,
                                            FakeMemcache(),
                                            account_ring=FakeRing(),
                                            container_ring=FakeRing())

    def test_get_info(self):
        app = FakeApp()
        # Do a non cached call to account
        env = {}
        info_a = get_info(app, env, 'a')
        # Check that you got proper info
        self.assertEqual(info_a['status'], 200)
        self.assertEqual(info_a['bytes'], 6666)
Exemple #22
0
def setup_servers(the_object_server=object_server, extra_conf=None):
    """
    Setup proxy, account, container and object servers using a set of fake
    rings and policies.

    :param the_object_server: The object server module to use (optional,
                              defaults to swift.obj.server)
    :param extra_conf: A dict of config options that will update the basic
                       config passed to all server instances.
    :returns: A dict containing the following entries:
                  orig_POLICIES: the value of storage_policy.POLICIES prior to
                                 it being patched with fake policies
                  orig_SysLogHandler: the value of utils.SysLogHandler prior to
                                      it being patched
                  testdir: root directory used for test files
                  test_POLICIES: a StoragePolicyCollection of fake policies
                  test_servers: a tuple of test server instances
                  test_sockets: a tuple of sockets used by test servers
                  test_coros: a tuple of greenthreads in which test servers are
                              running
    """
    context = {
        "orig_POLICIES": storage_policy._POLICIES,
        "orig_SysLogHandler": utils.SysLogHandler}

    utils.HASH_PATH_SUFFIX = b'endcap'
    utils.SysLogHandler = mock.MagicMock()
    # Since we're starting up a lot here, we're going to test more than
    # just chunked puts; we're also going to test parts of
    # proxy_server.Application we couldn't get to easily otherwise.
    context["testdir"] = _testdir = \
        os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked')
    mkdirs(_testdir)
    rmtree(_testdir)
    for drive in ('sda1', 'sdb1', 'sdc1', 'sdd1', 'sde1',
                  'sdf1', 'sdg1', 'sdh1', 'sdi1', 'sdj1',
                  'sdk1', 'sdl1'):
        mkdirs(os.path.join(_testdir, drive, 'tmp'))
    conf = {'devices': _testdir, 'swift_dir': _testdir,
            'mount_check': 'false', 'allowed_headers':
            'content-encoding, x-object-manifest, content-disposition, foo',
            'allow_versions': 't'}
    if extra_conf:
        conf.update(extra_conf)
    prolis = listen_zero()
    acc1lis = listen_zero()
    acc2lis = listen_zero()
    con1lis = listen_zero()
    con2lis = listen_zero()
    obj1lis = listen_zero()
    obj2lis = listen_zero()
    obj3lis = listen_zero()
    obj4lis = listen_zero()
    obj5lis = listen_zero()
    obj6lis = listen_zero()
    objsocks = [obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis]
    context["test_sockets"] = \
        (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, obj3lis,
         obj4lis, obj5lis, obj6lis)
    account_ring_path = os.path.join(_testdir, 'account.ring.gz')
    account_devs = [
        {'port': acc1lis.getsockname()[1]},
        {'port': acc2lis.getsockname()[1]},
    ]
    write_fake_ring(account_ring_path, *account_devs)
    container_ring_path = os.path.join(_testdir, 'container.ring.gz')
    container_devs = [
        {'port': con1lis.getsockname()[1]},
        {'port': con2lis.getsockname()[1]},
    ]
    write_fake_ring(container_ring_path, *container_devs)
    storage_policy._POLICIES = storage_policy.StoragePolicyCollection([
        StoragePolicy(0, 'zero', True),
        StoragePolicy(1, 'one', False),
        StoragePolicy(2, 'two', False),
        ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE,
                        ec_ndata=2, ec_nparity=1, ec_segment_size=4096),
        ECStoragePolicy(4, 'ec-dup', ec_type=DEFAULT_TEST_EC_TYPE,
                        ec_ndata=2, ec_nparity=1, ec_segment_size=4096,
                        ec_duplication_factor=2)])
    obj_rings = {
        0: ('sda1', 'sdb1'),
        1: ('sdc1', 'sdd1'),
        2: ('sde1', 'sdf1'),
        # sdg1, sdh1, sdi1 taken by policy 3 (see below)
    }
    for policy_index, devices in obj_rings.items():
        policy = storage_policy.POLICIES[policy_index]
        obj_ring_path = os.path.join(_testdir, policy.ring_name + '.ring.gz')
        obj_devs = [
            {'port': objsock.getsockname()[1], 'device': dev}
            for objsock, dev in zip(objsocks, devices)]
        write_fake_ring(obj_ring_path, *obj_devs)

    # write_fake_ring can't handle a 3-element ring, and the EC policy needs
    # at least 6 devs to work with (ec_k=2, ec_m=1, duplication_factor=2),
    # so we do it manually
    devs = [{'id': 0, 'zone': 0, 'device': 'sdg1', 'ip': '127.0.0.1',
             'port': obj1lis.getsockname()[1]},
            {'id': 1, 'zone': 0, 'device': 'sdh1', 'ip': '127.0.0.1',
             'port': obj2lis.getsockname()[1]},
            {'id': 2, 'zone': 0, 'device': 'sdi1', 'ip': '127.0.0.1',
             'port': obj3lis.getsockname()[1]},
            {'id': 3, 'zone': 0, 'device': 'sdj1', 'ip': '127.0.0.1',
             'port': obj4lis.getsockname()[1]},
            {'id': 4, 'zone': 0, 'device': 'sdk1', 'ip': '127.0.0.1',
             'port': obj5lis.getsockname()[1]},
            {'id': 5, 'zone': 0, 'device': 'sdl1', 'ip': '127.0.0.1',
             'port': obj6lis.getsockname()[1]}]
    pol3_replica2part2dev_id = [[0, 1, 2, 0],
                                [1, 2, 0, 1],
                                [2, 0, 1, 2]]
    pol4_replica2part2dev_id = [[0, 1, 2, 3],
                                [1, 2, 3, 4],
                                [2, 3, 4, 5],
                                [3, 4, 5, 0],
                                [4, 5, 0, 1],
                                [5, 0, 1, 2]]
    obj3_ring_path = os.path.join(
        _testdir, storage_policy.POLICIES[3].ring_name + '.ring.gz')
    part_shift = 30
    with closing(GzipFile(obj3_ring_path, 'wb')) as fh:
        pickle.dump(RingData(pol3_replica2part2dev_id, devs, part_shift), fh)

    obj4_ring_path = os.path.join(
        _testdir, storage_policy.POLICIES[4].ring_name + '.ring.gz')
    part_shift = 30
    with closing(GzipFile(obj4_ring_path, 'wb')) as fh:
        pickle.dump(RingData(pol4_replica2part2dev_id, devs, part_shift), fh)

    prosrv = proxy_server.Application(conf, logger=debug_logger('proxy'))
    for policy in storage_policy.POLICIES:
        # make sure all the rings are loaded
        prosrv.get_object_ring(policy.idx)
    # don't lose this one!
    context["test_POLICIES"] = storage_policy._POLICIES
    acc1srv = account_server.AccountController(
        conf, logger=debug_logger('acct1'))
    acc2srv = account_server.AccountController(
        conf, logger=debug_logger('acct2'))
    con1srv = container_server.ContainerController(
        conf, logger=debug_logger('cont1'))
    con2srv = container_server.ContainerController(
        conf, logger=debug_logger('cont2'))
    obj1srv = the_object_server.ObjectController(
        conf, logger=debug_logger('obj1'))
    obj2srv = the_object_server.ObjectController(
        conf, logger=debug_logger('obj2'))
    obj3srv = the_object_server.ObjectController(
        conf, logger=debug_logger('obj3'))
    obj4srv = the_object_server.ObjectController(
        conf, logger=debug_logger('obj4'))
    obj5srv = the_object_server.ObjectController(
        conf, logger=debug_logger('obj5'))
    obj6srv = the_object_server.ObjectController(
        conf, logger=debug_logger('obj6'))
    context["test_servers"] = \
        (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv, obj3srv,
         obj4srv, obj5srv, obj6srv)
    nl = NullLogger()
    logging_prosv = proxy_logging.ProxyLoggingMiddleware(
        listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger)
    prospa = spawn(wsgi.server, prolis, logging_prosv, nl)
    acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
    acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
    con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
    con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
    obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
    obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
    obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl)
    obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl)
    obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl)
    obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl)
    context["test_coros"] = \
        (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa,
         obj4spa, obj5spa, obj6spa)
    # Create account
    ts = normalize_timestamp(time.time())
    partition, nodes = prosrv.account_ring.get_nodes('a')
    for node in nodes:
        conn = swift.proxy.controllers.obj.http_connect(node['ip'],
                                                        node['port'],
                                                        node['device'],
                                                        partition, 'PUT', '/a',
                                                        {'X-Timestamp': ts,
                                                         'x-trans-id': 'test'})
        resp = conn.getresponse()
        assert(resp.status == 201)
    # Create another account
    # used for account-to-account tests
    ts = normalize_timestamp(time.time())
    partition, nodes = prosrv.account_ring.get_nodes('a1')
    for node in nodes:
        conn = swift.proxy.controllers.obj.http_connect(node['ip'],
                                                        node['port'],
                                                        node['device'],
                                                        partition, 'PUT',
                                                        '/a1',
                                                        {'X-Timestamp': ts,
                                                         'x-trans-id': 'test'})
        resp = conn.getresponse()
        assert(resp.status == 201)
    # Create containers, 1 per test policy
    sock = connect_tcp(('localhost', prolis.getsockname()[1]))
    fd = sock.makefile()
    fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n'
             'Connection: close\r\nX-Auth-Token: t\r\n'
             'Content-Length: 0\r\n\r\n')
    fd.flush()
    headers = readuntil2crlfs(fd)
    exp = 'HTTP/1.1 201'
    assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % (
        exp, headers[:len(exp)])
    # Create container in other account
    # used for account-to-account tests
    sock = connect_tcp(('localhost', prolis.getsockname()[1]))
    fd = sock.makefile()
    fd.write('PUT /v1/a1/c1 HTTP/1.1\r\nHost: localhost\r\n'
             'Connection: close\r\nX-Auth-Token: t\r\n'
             'Content-Length: 0\r\n\r\n')
    fd.flush()
    headers = readuntil2crlfs(fd)
    exp = 'HTTP/1.1 201'
    assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % (
        exp, headers[:len(exp)])

    sock = connect_tcp(('localhost', prolis.getsockname()[1]))
    fd = sock.makefile()
    fd.write(
        'PUT /v1/a/c1 HTTP/1.1\r\nHost: localhost\r\n'
        'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: one\r\n'
        'Content-Length: 0\r\n\r\n')
    fd.flush()
    headers = readuntil2crlfs(fd)
    exp = 'HTTP/1.1 201'
    assert headers[:len(exp)] == exp, \
        "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)])

    sock = connect_tcp(('localhost', prolis.getsockname()[1]))
    fd = sock.makefile()
    fd.write(
        'PUT /v1/a/c2 HTTP/1.1\r\nHost: localhost\r\n'
        'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: two\r\n'
        'Content-Length: 0\r\n\r\n')
    fd.flush()
    headers = readuntil2crlfs(fd)
    exp = 'HTTP/1.1 201'
    assert headers[:len(exp)] == exp, \
        "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)])
    return context
Exemple #23
0
class TestAccountBroker(unittest.TestCase):
    """Tests for AccountBroker"""
    def test_creation(self):
        # Test AccountBroker.__init__
        broker = AccountBroker(':memory:', account='a')
        self.assertEqual(broker.db_file, ':memory:')
        try:
            with broker.get() as conn:
                pass
        except DatabaseConnectionError as e:
            self.assertTrue(hasattr(e, 'path'))
            self.assertEquals(e.path, ':memory:')
            self.assertTrue(hasattr(e, 'msg'))
            self.assertEquals(e.msg, "DB doesn't exist")
        except Exception as e:
            self.fail("Unexpected exception raised: %r" % e)
        else:
            self.fail("Expected a DatabaseConnectionError exception")
        broker.initialize(Timestamp('1').internal)
        with broker.get() as conn:
            curs = conn.cursor()
            curs.execute('SELECT 1')
            self.assertEqual(curs.fetchall()[0][0], 1)

    def test_exception(self):
        # Test AccountBroker throwing a conn away after exception
        first_conn = None
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(Timestamp('1').internal)
        with broker.get() as conn:
            first_conn = conn
        try:
            with broker.get() as conn:
                self.assertEqual(first_conn, conn)
                raise Exception('OMG')
        except Exception:
            pass
        self.assert_(broker.conn is None)

    def test_empty(self):
        # Test AccountBroker.empty
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(Timestamp('1').internal)
        self.assert_(broker.empty())
        broker.put_container('o',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        self.assert_(not broker.empty())
        sleep(.00001)
        broker.put_container('o', 0,
                             Timestamp(time()).internal, 0, 0,
                             POLICIES.default.idx)
        self.assert_(broker.empty())

    def test_is_status_deleted(self):
        # Test AccountBroker.is_status_deleted
        broker1 = AccountBroker(':memory:', account='a')
        broker1.initialize(Timestamp(time()).internal)
        self.assert_(not broker1.is_status_deleted())
        broker1.delete_db(Timestamp(time()).internal)
        self.assert_(broker1.is_status_deleted())
        broker2 = AccountBroker(':memory:', account='a')
        broker2.initialize(Timestamp(time()).internal)
        # Set delete_timestamp greater than put_timestamp
        broker2.merge_timestamps(time(),
                                 Timestamp(time()).internal,
                                 Timestamp(time() + 999).internal)
        self.assert_(broker2.is_status_deleted())

    def test_reclaim(self):
        broker = AccountBroker(':memory:', account='test_account')
        broker.initialize(Timestamp('1').internal)
        broker.put_container('c',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 0").fetchone()[0], 1)
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 1").fetchone()[0], 0)
        broker.reclaim(Timestamp(time() - 999).internal, time())
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 0").fetchone()[0], 1)
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 1").fetchone()[0], 0)
        sleep(.00001)
        broker.put_container('c', 0,
                             Timestamp(time()).internal, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 0").fetchone()[0], 0)
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 1").fetchone()[0], 1)
        broker.reclaim(Timestamp(time() - 999).internal, time())
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 0").fetchone()[0], 0)
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 1").fetchone()[0], 1)
        sleep(.00001)
        broker.reclaim(Timestamp(time()).internal, time())
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 0").fetchone()[0], 0)
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 1").fetchone()[0], 0)
        # Test reclaim after deletion. Create 3 test containers
        broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx)
        broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx)
        broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx)
        broker.reclaim(Timestamp(time()).internal, time())
        # self.assertEqual(len(res), 2)
        # self.assert_(isinstance(res, tuple))
        # containers, account_name = res
        # self.assert_(containers is None)
        # self.assert_(account_name is None)
        # Now delete the account
        broker.delete_db(Timestamp(time()).internal)
        broker.reclaim(Timestamp(time()).internal, time())
        # self.assertEqual(len(res), 2)
        # self.assert_(isinstance(res, tuple))
        # containers, account_name = res
        # self.assertEqual(account_name, 'test_account')
        # self.assertEqual(len(containers), 3)
        # self.assert_('x' in containers)
        # self.assert_('y' in containers)
        # self.assert_('z' in containers)
        # self.assert_('a' not in containers)

    def test_delete_db_status(self):
        ts = (Timestamp(t).internal for t in itertools.count(int(time())))
        start = ts.next()
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(start)
        info = broker.get_info()
        self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
        self.assert_(Timestamp(info['created_at']) >= start)
        self.assertEqual(info['delete_timestamp'], '0')
        if self.__class__ == TestAccountBrokerBeforeMetadata:
            self.assertEqual(info['status_changed_at'], '0')
        else:
            self.assertEqual(info['status_changed_at'],
                             Timestamp(start).internal)

        # delete it
        delete_timestamp = ts.next()
        broker.delete_db(delete_timestamp)
        info = broker.get_info()
        self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
        self.assert_(Timestamp(info['created_at']) >= start)
        self.assertEqual(info['delete_timestamp'], delete_timestamp)
        self.assertEqual(info['status_changed_at'], delete_timestamp)

    def test_delete_container(self):
        # Test AccountBroker.delete_container
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(Timestamp('1').internal)
        broker.put_container('o',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 0").fetchone()[0], 1)
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 1").fetchone()[0], 0)
        sleep(.00001)
        broker.put_container('o', 0,
                             Timestamp(time()).internal, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 0").fetchone()[0], 0)
            self.assertEqual(
                conn.execute("SELECT count(*) FROM container "
                             "WHERE deleted = 1").fetchone()[0], 1)

    def test_put_container(self):
        # Test AccountBroker.put_container
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(Timestamp('1').internal)

        # Create initial container
        timestamp = Timestamp(time()).internal
        broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT name FROM container").fetchone()[0],
                '"{<container \'&\' name>}"')
            self.assertEqual(
                conn.execute("SELECT put_timestamp FROM container").fetchone()
                [0], timestamp)
            self.assertEqual(
                conn.execute("SELECT deleted FROM container").fetchone()[0], 0)

        # Reput same event
        broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT name FROM container").fetchone()[0],
                '"{<container \'&\' name>}"')
            self.assertEqual(
                conn.execute("SELECT put_timestamp FROM container").fetchone()
                [0], timestamp)
            self.assertEqual(
                conn.execute("SELECT deleted FROM container").fetchone()[0], 0)

        # Put new event
        sleep(.00001)
        timestamp = Timestamp(time()).internal
        broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT name FROM container").fetchone()[0],
                '"{<container \'&\' name>}"')
            self.assertEqual(
                conn.execute("SELECT put_timestamp FROM container").fetchone()
                [0], timestamp)
            self.assertEqual(
                conn.execute("SELECT deleted FROM container").fetchone()[0], 0)

        # Put old event
        otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
        broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT name FROM container").fetchone()[0],
                '"{<container \'&\' name>}"')
            self.assertEqual(
                conn.execute("SELECT put_timestamp FROM container").fetchone()
                [0], timestamp)
            self.assertEqual(
                conn.execute("SELECT deleted FROM container").fetchone()[0], 0)

        # Put old delete event
        dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
        broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT name FROM container").fetchone()[0],
                '"{<container \'&\' name>}"')
            self.assertEqual(
                conn.execute("SELECT put_timestamp FROM container").fetchone()
                [0], timestamp)
            self.assertEqual(
                conn.execute(
                    "SELECT delete_timestamp FROM container").fetchone()[0],
                dtimestamp)
            self.assertEqual(
                conn.execute("SELECT deleted FROM container").fetchone()[0], 0)

        # Put new delete event
        sleep(.00001)
        timestamp = Timestamp(time()).internal
        broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT name FROM container").fetchone()[0],
                '"{<container \'&\' name>}"')
            self.assertEqual(
                conn.execute(
                    "SELECT delete_timestamp FROM container").fetchone()[0],
                timestamp)
            self.assertEqual(
                conn.execute("SELECT deleted FROM container").fetchone()[0], 1)

        # Put new event
        sleep(.00001)
        timestamp = Timestamp(time()).internal
        broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
                             POLICIES.default.idx)
        with broker.get() as conn:
            self.assertEqual(
                conn.execute("SELECT name FROM container").fetchone()[0],
                '"{<container \'&\' name>}"')
            self.assertEqual(
                conn.execute("SELECT put_timestamp FROM container").fetchone()
                [0], timestamp)
            self.assertEqual(
                conn.execute("SELECT deleted FROM container").fetchone()[0], 0)

    def test_get_info(self):
        # Test AccountBroker.get_info
        broker = AccountBroker(':memory:', account='test1')
        broker.initialize(Timestamp('1').internal)

        info = broker.get_info()
        self.assertEqual(info['account'], 'test1')
        self.assertEqual(info['hash'], '00000000000000000000000000000000')
        self.assertEqual(info['put_timestamp'], Timestamp(1).internal)
        self.assertEqual(info['delete_timestamp'], '0')
        if self.__class__ == TestAccountBrokerBeforeMetadata:
            self.assertEqual(info['status_changed_at'], '0')
        else:
            self.assertEqual(info['status_changed_at'], Timestamp(1).internal)

        info = broker.get_info()
        self.assertEqual(info['container_count'], 0)

        broker.put_container('c1',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        info = broker.get_info()
        self.assertEqual(info['container_count'], 1)

        sleep(.00001)
        broker.put_container('c2',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        info = broker.get_info()
        self.assertEqual(info['container_count'], 2)

        sleep(.00001)
        broker.put_container('c2',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        info = broker.get_info()
        self.assertEqual(info['container_count'], 2)

        sleep(.00001)
        broker.put_container('c1', 0,
                             Timestamp(time()).internal, 0, 0,
                             POLICIES.default.idx)
        info = broker.get_info()
        self.assertEqual(info['container_count'], 1)

        sleep(.00001)
        broker.put_container('c2', 0,
                             Timestamp(time()).internal, 0, 0,
                             POLICIES.default.idx)
        info = broker.get_info()
        self.assertEqual(info['container_count'], 0)

    def test_list_containers_iter(self):
        # Test AccountBroker.list_containers_iter
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(Timestamp('1').internal)
        for cont1 in xrange(4):
            for cont2 in xrange(125):
                broker.put_container('%d-%04d' % (cont1, cont2),
                                     Timestamp(time()).internal, 0, 0, 0,
                                     POLICIES.default.idx)
        for cont in xrange(125):
            broker.put_container('2-0051-%04d' % cont,
                                 Timestamp(time()).internal, 0, 0, 0,
                                 POLICIES.default.idx)

        for cont in xrange(125):
            broker.put_container('3-%04d-0049' % cont,
                                 Timestamp(time()).internal, 0, 0, 0,
                                 POLICIES.default.idx)

        listing = broker.list_containers_iter(100, '', None, None, '')
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0099')

        listing = broker.list_containers_iter(100, '', '0-0050', None, '')
        self.assertEqual(len(listing), 50)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0049')

        listing = broker.list_containers_iter(100, '0-0099', None, None, '')
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '1-0074')

        listing = broker.list_containers_iter(55, '1-0074', None, None, '')
        self.assertEqual(len(listing), 55)
        self.assertEqual(listing[0][0], '1-0075')
        self.assertEqual(listing[-1][0], '2-0004')

        listing = broker.list_containers_iter(10, '', None, '0-01', '')
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = broker.list_containers_iter(10, '', None, '0-01', '-')
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = broker.list_containers_iter(10, '', None, '0-', '-')
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0009')

        listing = broker.list_containers_iter(10, '', None, '', '-')
        self.assertEqual(len(listing), 4)
        self.assertEqual([row[0] for row in listing], ['0-', '1-', '2-', '3-'])

        listing = broker.list_containers_iter(10, '2-', None, None, '-')
        self.assertEqual(len(listing), 1)
        self.assertEqual([row[0] for row in listing], ['3-'])

        listing = broker.list_containers_iter(10, '', None, '2', '-')
        self.assertEqual(len(listing), 1)
        self.assertEqual([row[0] for row in listing], ['2-'])

        listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-')
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '2-0051')
        self.assertEqual(listing[1][0], '2-0051-')
        self.assertEqual(listing[2][0], '2-0052')
        self.assertEqual(listing[-1][0], '2-0059')

        listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-')
        self.assertEqual(len(listing), 10)
        self.assertEqual([row[0] for row in listing], [
            '3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048',
            '3-0048-', '3-0049', '3-0049-', '3-0050'
        ])

        broker.put_container('3-0049-',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        listing = broker.list_containers_iter(10, '3-0048', None, None, None)
        self.assertEqual(len(listing), 10)
        self.assertEqual([row[0] for row in listing], [
            '3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050',
            '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049'
        ])

        listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-')
        self.assertEqual(len(listing), 10)
        self.assertEqual([row[0] for row in listing], [
            '3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051',
            '3-0051-', '3-0052', '3-0052-', '3-0053'
        ])

        listing = broker.list_containers_iter(10, None, None, '3-0049-', '-')
        self.assertEqual(len(listing), 2)
        self.assertEqual([row[0] for row in listing],
                         ['3-0049-', '3-0049-0049'])

    def test_double_check_trailing_delimiter(self):
        # Test AccountBroker.list_containers_iter for an
        # account that has an odd container with a trailing delimiter
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(Timestamp('1').internal)
        broker.put_container('a',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('a-',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('a-a',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('a-a-a',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('a-a-b',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('a-b',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('b',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('b-a',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('b-b',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        broker.put_container('c',
                             Timestamp(time()).internal, 0, 0, 0,
                             POLICIES.default.idx)
        listing = broker.list_containers_iter(15, None, None, None, None)
        self.assertEqual(len(listing), 10)
        self.assertEqual([row[0] for row in listing], [
            'a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', 'b-a', 'b-b', 'c'
        ])
        listing = broker.list_containers_iter(15, None, None, '', '-')
        self.assertEqual(len(listing), 5)
        self.assertEqual([row[0] for row in listing],
                         ['a', 'a-', 'b', 'b-', 'c'])
        listing = broker.list_containers_iter(15, None, None, 'a-', '-')
        self.assertEqual(len(listing), 4)
        self.assertEqual([row[0] for row in listing],
                         ['a-', 'a-a', 'a-a-', 'a-b'])
        listing = broker.list_containers_iter(15, None, None, 'b-', '-')
        self.assertEqual(len(listing), 2)
        self.assertEqual([row[0] for row in listing], ['b-a', 'b-b'])

    def test_chexor(self):
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(Timestamp('1').internal)
        broker.put_container('a',
                             Timestamp(1).internal,
                             Timestamp(0).internal, 0, 0, POLICIES.default.idx)
        broker.put_container('b',
                             Timestamp(2).internal,
                             Timestamp(0).internal, 0, 0, POLICIES.default.idx)
        hasha = hashlib.md5(
            '%s-%s' %
            ('a', "%s-%s-%s-%s" %
             (Timestamp(1).internal, Timestamp(0).internal, 0, 0))).digest()
        hashb = hashlib.md5(
            '%s-%s' %
            ('b', "%s-%s-%s-%s" %
             (Timestamp(2).internal, Timestamp(0).internal, 0, 0))).digest()
        hashc = \
            ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
        self.assertEqual(broker.get_info()['hash'], hashc)
        broker.put_container('b',
                             Timestamp(3).internal,
                             Timestamp(0).internal, 0, 0, POLICIES.default.idx)
        hashb = hashlib.md5(
            '%s-%s' %
            ('b', "%s-%s-%s-%s" %
             (Timestamp(3).internal, Timestamp(0).internal, 0, 0))).digest()
        hashc = \
            ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
        self.assertEqual(broker.get_info()['hash'], hashc)

    def test_merge_items(self):
        broker1 = AccountBroker(':memory:', account='a')
        broker1.initialize(Timestamp('1').internal)
        broker2 = AccountBroker(':memory:', account='a')
        broker2.initialize(Timestamp('1').internal)
        broker1.put_container('a',
                              Timestamp(1).internal, 0, 0, 0,
                              POLICIES.default.idx)
        broker1.put_container('b',
                              Timestamp(2).internal, 0, 0, 0,
                              POLICIES.default.idx)
        id = broker1.get_info()['id']
        broker2.merge_items(
            broker1.get_items_since(broker2.get_sync(id), 1000), id)
        items = broker2.get_items_since(-1, 1000)
        self.assertEqual(len(items), 2)
        self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
        broker1.put_container('c',
                              Timestamp(3).internal, 0, 0, 0,
                              POLICIES.default.idx)
        broker2.merge_items(
            broker1.get_items_since(broker2.get_sync(id), 1000), id)
        items = broker2.get_items_since(-1, 1000)
        self.assertEqual(len(items), 3)
        self.assertEqual(['a', 'b', 'c'],
                         sorted([rec['name'] for rec in items]))

    def test_load_old_pending_puts(self):
        # pending puts from pre-storage-policy account brokers won't contain
        # the storage policy index
        tempdir = mkdtemp()
        broker_path = os.path.join(tempdir, 'test-load-old.db')
        try:
            broker = AccountBroker(broker_path, account='real')
            broker.initialize(Timestamp(1).internal)
            with open(broker_path + '.pending', 'a+b') as pending:
                pending.write(':')
                pending.write(
                    pickle.dumps(
                        # name, put_timestamp, delete_timestamp, object_count,
                        # bytes_used, deleted
                        ('oldcon', Timestamp(200).internal,
                         Timestamp(0).internal, 896, 9216695, 0
                         )).encode('base64'))

            broker._commit_puts()
            with broker.get() as conn:
                results = list(
                    conn.execute('''
                    SELECT name, storage_policy_index FROM container
                '''))
            self.assertEqual(len(results), 1)
            self.assertEqual(dict(results[0]), {
                'name': 'oldcon',
                'storage_policy_index': 0
            })
        finally:
            rmtree(tempdir)

    @patch_policies([
        StoragePolicy(0, 'zero', False),
        StoragePolicy(1, 'one', True),
        StoragePolicy(2, 'two', False),
        StoragePolicy(3, 'three', False)
    ])
    def test_get_policy_stats(self):
        ts = (Timestamp(t).internal for t in itertools.count(int(time())))
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(ts.next())
        # check empty policy_stats
        self.assertTrue(broker.empty())
        policy_stats = broker.get_policy_stats()
        self.assertEqual(policy_stats, {})

        # add some empty containers
        for policy in POLICIES:
            container_name = 'c-%s' % policy.name
            put_timestamp = ts.next()
            broker.put_container(container_name, put_timestamp, 0, 0, 0,
                                 policy.idx)

            policy_stats = broker.get_policy_stats()
            stats = policy_stats[policy.idx]
            self.assertEqual(stats['object_count'], 0)
            self.assertEqual(stats['bytes_used'], 0)

        # update the containers object & byte count
        for policy in POLICIES:
            container_name = 'c-%s' % policy.name
            put_timestamp = ts.next()
            count = policy.idx * 100  # good as any integer
            broker.put_container(container_name, put_timestamp, 0, count,
                                 count, policy.idx)

            policy_stats = broker.get_policy_stats()
            stats = policy_stats[policy.idx]
            self.assertEqual(stats['object_count'], count)
            self.assertEqual(stats['bytes_used'], count)

        # check all the policy_stats at once
        for policy_index, stats in policy_stats.items():
            policy = POLICIES[policy_index]
            count = policy.idx * 100  # coupled with policy for test
            self.assertEqual(stats['object_count'], count)
            self.assertEqual(stats['bytes_used'], count)

        # now delete the containers one by one
        for policy in POLICIES:
            container_name = 'c-%s' % policy.name
            delete_timestamp = ts.next()
            broker.put_container(container_name, 0, delete_timestamp, 0, 0,
                                 policy.idx)

            policy_stats = broker.get_policy_stats()
            stats = policy_stats[policy.idx]
            self.assertEqual(stats['object_count'], 0)
            self.assertEqual(stats['bytes_used'], 0)

    @patch_policies(
        [StoragePolicy(0, 'zero', False),
         StoragePolicy(1, 'one', True)])
    def test_policy_stats_tracking(self):
        ts = (Timestamp(t).internal for t in itertools.count(int(time())))
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(ts.next())

        # policy 0
        broker.put_container('con1', ts.next(), 0, 12, 2798641, 0)
        broker.put_container('con1', ts.next(), 0, 13, 8156441, 0)
        # policy 1
        broker.put_container('con2', ts.next(), 0, 7, 5751991, 1)
        broker.put_container('con2', ts.next(), 0, 8, 6085379, 1)

        stats = broker.get_policy_stats()
        self.assertEqual(len(stats), 2)
        self.assertEqual(stats[0]['object_count'], 13)
        self.assertEqual(stats[0]['bytes_used'], 8156441)
        self.assertEqual(stats[1]['object_count'], 8)
        self.assertEqual(stats[1]['bytes_used'], 6085379)

        # Break encapsulation here to make sure that there's only 2 rows in
        # the stats table. It's possible that there could be 4 rows (one per
        # put_container) but that they came out in the right order so that
        # get_policy_stats() collapsed them down to the right number. To prove
        # that's not so, we have to go peek at the broker's internals.
        with broker.get() as conn:
            nrows = conn.execute(
                "SELECT COUNT(*) FROM policy_stat").fetchall()[0][0]
        self.assertEqual(nrows, 2)
Exemple #24
0
    'device': 'sda1',
    'ip': '',
    'port': ''
}, {
    'device': 'sda1',
    'ip': '',
    'port': ''
}, {
    'device': 'sda1',
    'ip': '',
    'port': ''
}]


@unit.patch_policies([
    StoragePolicy(0, 'zero', False, object_ring=unit.FakeRing()),
    StoragePolicy(1, 'one', True, object_ring=unit.FakeRing(replicas=4))
])
class TestReaper(unittest.TestCase):
    def setUp(self):
        self.to_delete = []
        self.myexp = ClientException("",
                                     http_host=None,
                                     http_port=None,
                                     http_device=None,
                                     http_status=404,
                                     http_reason=None)

    def tearDown(self):
        for todel in self.to_delete:
            shutil.rmtree(todel)
Exemple #25
0
    def test_validate_policies_defaults(self):
        # 0 explicit default
        test_policies = [
            StoragePolicy(0, 'zero', True),
            StoragePolicy(1, 'one', False),
            StoragePolicy(2, 'two', False)
        ]
        policies = StoragePolicyCollection(test_policies)
        self.assertEquals(policies.default, test_policies[0])
        self.assertEquals(policies.default.name, 'zero')

        # non-zero explicit default
        test_policies = [
            StoragePolicy(0, 'zero', False),
            StoragePolicy(1, 'one', False),
            StoragePolicy(2, 'two', True)
        ]
        policies = StoragePolicyCollection(test_policies)
        self.assertEquals(policies.default, test_policies[2])
        self.assertEquals(policies.default.name, 'two')

        # multiple defaults
        test_policies = [
            StoragePolicy(0, 'zero', False),
            StoragePolicy(1, 'one', True),
            StoragePolicy(2, 'two', True)
        ]
        self.assertRaisesWithMessage(PolicyError, 'Duplicate default',
                                     StoragePolicyCollection, test_policies)

        # nothing specified
        test_policies = []
        policies = StoragePolicyCollection(test_policies)
        self.assertEquals(policies.default, policies[0])
        self.assertEquals(policies.default.name, 'Policy-0')

        # no default specified with only policy index 0
        test_policies = [StoragePolicy(0, 'zero')]
        policies = StoragePolicyCollection(test_policies)
        self.assertEqual(policies.default, policies[0])

        # no default specified with multiple policies
        test_policies = [
            StoragePolicy(0, 'zero', False),
            StoragePolicy(1, 'one', False),
            StoragePolicy(2, 'two', False)
        ]
        self.assertRaisesWithMessage(PolicyError,
                                     'Unable to find default policy',
                                     StoragePolicyCollection, test_policies)
Exemple #26
0
    """
    This patch is just a hook over handle_request to ensure that when
    get_controller is called the ObjectController class is patched to
    return a (possibly stubbed) ObjectController class.
    """

    object_controller = proxy_server.ObjectController

    def handle_request(self, req):
        with mock.patch('swift.proxy.server.ObjectController',
                        new=self.object_controller):
            return super(PatchedObjControllerApp, self).handle_request(req)


@patch_policies(
    [StoragePolicy(0, 'zero', True, object_ring=FakeRing(max_more_nodes=9))])
class TestObjControllerWriteAffinity(unittest.TestCase):
    def setUp(self):
        self.app = proxy_server.Application(None,
                                            FakeMemcache(),
                                            account_ring=FakeRing(),
                                            container_ring=FakeRing(),
                                            logger=debug_logger())
        self.app.request_node_count = lambda ring: 10000000
        self.app.sort_nodes = lambda l: l  # stop shuffling the primary nodes

    def test_iter_nodes_local_first_noops_when_no_affinity(self):
        controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
        self.app.write_affinity_is_local_fn = None
        object_ring = self.app.get_object_ring(None)
        all_nodes = object_ring.get_part_nodes(1)
Exemple #27
0
             {'device': 'sda1',
              'ip': '',
              'port': ''}]

cont_nodes = [{'device': 'sda1',
               'ip': '',
               'port': ''},
              {'device': 'sda1',
               'ip': '',
               'port': ''},
              {'device': 'sda1',
               'ip': '',
               'port': ''}]


@unit.patch_policies([StoragePolicy(0, 'zero', False,
                                    object_ring=unit.FakeRing()),
                      StoragePolicy(1, 'one', True,
                                    object_ring=unit.FakeRing())])
class TestReaper(unittest.TestCase):

    def setUp(self):
        self.to_delete = []
        self.myexp = ClientException("", http_host=None,
                                     http_port=None,
                                     http_device=None,
                                     http_status=404,
                                     http_reason=None
                                     )

    def tearDown(self):
        for todel in self.to_delete:
Exemple #28
0
class TestStoragePolicies(unittest.TestCase):
    def _conf(self, conf_str):
        conf_str = "\n".join(line.strip() for line in conf_str.split("\n"))
        conf = ConfigParser()
        conf.readfp(StringIO.StringIO(conf_str))
        return conf

    @patch_policies([
        StoragePolicy(0, 'zero', True),
        StoragePolicy(1, 'one', False),
        StoragePolicy(2, 'two', False),
        StoragePolicy(3, 'three', False, is_deprecated=True)
    ])
    def test_swift_info(self):
        # the deprecated 'three' should not exist in expect
        expect = [{
            'default': True,
            'name': 'zero'
        }, {
            'name': 'two'
        }, {
            'name': 'one'
        }]
        swift_info = POLICIES.get_policy_info()
        self.assertEquals(sorted(expect, key=lambda k: k['name']),
                          sorted(swift_info, key=lambda k: k['name']))

    @patch_policies
    def test_get_policy_string(self):
        self.assertEquals(get_policy_string('something', 0), 'something')
        self.assertEquals(get_policy_string('something', None), 'something')
        self.assertEquals(get_policy_string('something', 1),
                          'something' + '-1')
        self.assertRaises(PolicyError, get_policy_string, 'something', 99)

    def test_defaults(self):
        self.assertTrue(len(POLICIES) > 0)

        # test class functions
        default_policy = POLICIES.default
        self.assert_(default_policy.is_default)
        zero_policy = POLICIES.get_by_index(0)
        self.assert_(zero_policy.idx == 0)
        zero_policy_by_name = POLICIES.get_by_name(zero_policy.name)
        self.assert_(zero_policy_by_name.idx == 0)

    def test_storage_policy_repr(self):
        test_policies = [
            StoragePolicy(0, 'aay', True),
            StoragePolicy(1, 'bee', False),
            StoragePolicy(2, 'cee', False)
        ]
        policies = StoragePolicyCollection(test_policies)
        for policy in policies:
            policy_repr = repr(policy)
            self.assert_(policy.__class__.__name__ in policy_repr)
            self.assert_('is_default=%s' % policy.is_default in policy_repr)
            self.assert_('is_deprecated=%s' %
                         policy.is_deprecated in policy_repr)
            self.assert_(policy.name in policy_repr)
        collection_repr = repr(policies)
        collection_repr_lines = collection_repr.splitlines()
        self.assert_(policies.__class__.__name__ in collection_repr_lines[0])
        self.assertEqual(len(policies), len(collection_repr_lines[1:-1]))
        for policy, line in zip(policies, collection_repr_lines[1:-1]):
            self.assert_(repr(policy) in line)
        with patch_policies(policies):
            self.assertEqual(repr(POLICIES), collection_repr)

    def test_validate_policies_defaults(self):
        # 0 explicit default
        test_policies = [
            StoragePolicy(0, 'zero', True),
            StoragePolicy(1, 'one', False),
            StoragePolicy(2, 'two', False)
        ]
        policies = StoragePolicyCollection(test_policies)
        self.assertEquals(policies.default, test_policies[0])
        self.assertEquals(policies.default.name, 'zero')

        # non-zero explicit default
        test_policies = [
            StoragePolicy(0, 'zero', False),
            StoragePolicy(1, 'one', False),
            StoragePolicy(2, 'two', True)
        ]
        policies = StoragePolicyCollection(test_policies)
        self.assertEquals(policies.default, test_policies[2])
        self.assertEquals(policies.default.name, 'two')

        # multiple defaults
        test_policies = [
            StoragePolicy(0, 'zero', False),
            StoragePolicy(1, 'one', True),
            StoragePolicy(2, 'two', True)
        ]
        self.assertRaisesWithMessage(PolicyError, 'Duplicate default',
                                     StoragePolicyCollection, test_policies)

        # nothing specified
        test_policies = []
        policies = StoragePolicyCollection(test_policies)
        self.assertEquals(policies.default, policies[0])
        self.assertEquals(policies.default.name, 'Policy-0')

        # no default specified with only policy index 0
        test_policies = [StoragePolicy(0, 'zero')]
        policies = StoragePolicyCollection(test_policies)
        self.assertEqual(policies.default, policies[0])

        # no default specified with multiple policies
        test_policies = [
            StoragePolicy(0, 'zero', False),
            StoragePolicy(1, 'one', False),
            StoragePolicy(2, 'two', False)
        ]
        self.assertRaisesWithMessage(PolicyError,
                                     'Unable to find default policy',
                                     StoragePolicyCollection, test_policies)

    def test_deprecate_policies(self):
        # deprecation specified
        test_policies = [
            StoragePolicy(0, 'zero', True),
            StoragePolicy(1, 'one', False),
            StoragePolicy(2, 'two', False, is_deprecated=True)
        ]
        policies = StoragePolicyCollection(test_policies)
        self.assertEquals(policies.default, test_policies[0])
        self.assertEquals(policies.default.name, 'zero')
        self.assertEquals(len(policies), 3)

        # multiple policies requires default
        test_policies = [
            StoragePolicy(0, 'zero', False),
            StoragePolicy(1, 'one', False, is_deprecated=True),
            StoragePolicy(2, 'two', False)
        ]
        self.assertRaisesWithMessage(PolicyError,
                                     'Unable to find default policy',
                                     StoragePolicyCollection, test_policies)

    def test_validate_policies_indexes(self):
        # duplicate indexes
        test_policies = [
            StoragePolicy(0, 'zero', True),
            StoragePolicy(1, 'one', False),
            StoragePolicy(1, 'two', False)
        ]
        self.assertRaises(PolicyError, StoragePolicyCollection, test_policies)

    def test_validate_policy_params(self):
        StoragePolicy(0, 'name')  # sanity
        # bogus indexes
        self.assertRaises(PolicyError, StoragePolicy, 'x', 'name')
        self.assertRaises(PolicyError, StoragePolicy, -1, 'name')
        # non-zero Policy-0
        self.assertRaisesWithMessage(PolicyError, 'reserved', StoragePolicy, 1,
                                     'policy-0')
        # deprecate default
        self.assertRaisesWithMessage(PolicyError,
                                     'Deprecated policy can not be default',
                                     StoragePolicy,
                                     1,
                                     'Policy-1',
                                     is_default=True,
                                     is_deprecated=True)
        # weird names
        names = (
            '',
            'name_foo',
            'name\nfoo',
            'name foo',
            u'name \u062a',
            'name \xd8\xaa',
        )
        for name in names:
            self.assertRaisesWithMessage(PolicyError, 'Invalid name',
                                         StoragePolicy, 1, name)

    def test_validate_policies_names(self):
        # duplicate names
        test_policies = [
            StoragePolicy(0, 'zero', True),
            StoragePolicy(1, 'zero', False),
            StoragePolicy(2, 'two', False)
        ]
        self.assertRaises(PolicyError, StoragePolicyCollection, test_policies)

    def test_names_are_normalized(self):
        test_policies = [
            StoragePolicy(0, 'zero', True),
            StoragePolicy(1, 'ZERO', False)
        ]
        self.assertRaises(PolicyError, StoragePolicyCollection, test_policies)

        policies = StoragePolicyCollection(
            [StoragePolicy(0, 'zEro', True),
             StoragePolicy(1, 'One', False)])

        pol0 = policies[0]
        pol1 = policies[1]

        for name in ('zero', 'ZERO', 'zErO', 'ZeRo'):
            self.assertEqual(pol0, policies.get_by_name(name))
            self.assertEqual(policies.get_by_name(name).name, 'zEro')
        for name in ('one', 'ONE', 'oNe', 'OnE'):
            self.assertEqual(pol1, policies.get_by_name(name))
            self.assertEqual(policies.get_by_name(name).name, 'One')

    def assertRaisesWithMessage(self, exc_class, message, f, *args, **kwargs):
        try:
            f(*args, **kwargs)
        except exc_class as err:
            err_msg = str(err)
            self.assert_(
                message in err_msg, 'Error message %r did not '
                'have expected substring %r' % (err_msg, message))
        else:
            self.fail('%r did not raise %s' % (message, exc_class.__name__))

    def test_deprecated_default(self):
        bad_conf = self._conf("""
        [storage-policy:1]
        name = one
        deprecated = yes
        default = yes
        """)

        self.assertRaisesWithMessage(PolicyError,
                                     "Deprecated policy can not be default",
                                     parse_storage_policies, bad_conf)

    def test_multiple_policies_with_no_policy_index_zero(self):
        bad_conf = self._conf("""
        [storage-policy:1]
        name = one
        default = yes
        """)

        # Policy-0 will not be implicitly added if other policies are defined
        self.assertRaisesWithMessage(
            PolicyError, "must specify a storage policy section "
            "for policy index 0", parse_storage_policies, bad_conf)

    def test_no_default(self):
        orig_conf = self._conf("""
        [storage-policy:0]
        name = zero
        [storage-policy:1]
        name = one
        default = yes
        """)

        policies = parse_storage_policies(orig_conf)
        self.assertEqual(policies.default, policies[1])
        self.assert_(policies[0].name, 'Policy-0')

        bad_conf = self._conf("""
        [storage-policy:0]
        name = zero
        [storage-policy:1]
        name = one
        deprecated = yes
        """)

        # multiple polices and no explicit default
        self.assertRaisesWithMessage(PolicyError, "Unable to find default",
                                     parse_storage_policies, bad_conf)

        good_conf = self._conf("""
        [storage-policy:0]
        name = Policy-0
        default = yes
        [storage-policy:1]
        name = one
        deprecated = yes
        """)

        policies = parse_storage_policies(good_conf)
        self.assertEqual(policies.default, policies[0])
        self.assert_(policies[1].is_deprecated, True)

    def test_parse_storage_policies(self):
        # ValueError when deprecating policy 0
        bad_conf = self._conf("""
        [storage-policy:0]
        name = zero
        deprecated = yes

        [storage-policy:1]
        name = one
        deprecated = yes
        """)

        self.assertRaisesWithMessage(
            PolicyError, "Unable to find policy that's not deprecated",
            parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:]
        name = zero
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid index',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:-1]
        name = zero
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid index',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:x]
        name = zero
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid index',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:x-1]
        name = zero
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid index',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:x]
        name = zero
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid index',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:x:1]
        name = zero
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid index',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:1]
        name = zero
        boo = berries
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid option',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:0]
        name =
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid name',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:3]
        name = Policy-0
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid name',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:1]
        name = policY-0
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid name',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:0]
        name = one
        [storage-policy:1]
        name = ONE
        """)

        self.assertRaisesWithMessage(PolicyError, 'Duplicate name',
                                     parse_storage_policies, bad_conf)

        bad_conf = self._conf("""
        [storage-policy:0]
        name = good_stuff
        """)

        self.assertRaisesWithMessage(PolicyError, 'Invalid name',
                                     parse_storage_policies, bad_conf)

        # Additional section added to ensure parser ignores other sections
        conf = self._conf("""
        [some-other-section]
        foo = bar
        [storage-policy:0]
        name = zero
        [storage-policy:5]
        name = one
        default = yes
        [storage-policy:6]
        name = duplicate-sections-are-ignored
        [storage-policy:6]
        name = apple
        """)
        policies = parse_storage_policies(conf)

        self.assertEquals(True, policies.get_by_index(5).is_default)
        self.assertEquals(False, policies.get_by_index(0).is_default)
        self.assertEquals(False, policies.get_by_index(6).is_default)

        self.assertEquals("object", policies.get_by_name("zero").ring_name)
        self.assertEquals("object-5", policies.get_by_name("one").ring_name)
        self.assertEquals("object-6", policies.get_by_name("apple").ring_name)

        self.assertEqual(0, int(policies.get_by_name('zero')))
        self.assertEqual(5, int(policies.get_by_name('one')))
        self.assertEqual(6, int(policies.get_by_name('apple')))

        self.assertEquals("zero", policies.get_by_index(0).name)
        self.assertEquals("zero", policies.get_by_index("0").name)
        self.assertEquals("one", policies.get_by_index(5).name)
        self.assertEquals("apple", policies.get_by_index(6).name)
        self.assertEquals("zero", policies.get_by_index(None).name)
        self.assertEquals("zero", policies.get_by_index('').name)

    def test_reload_invalid_storage_policies(self):
        conf = self._conf("""
        [storage-policy:0]
        name = zero
        [storage-policy:00]
        name = double-zero
        """)
        with NamedTemporaryFile() as f:
            conf.write(f)
            f.flush()
            with mock.patch('swift.common.storage_policy.SWIFT_CONF_FILE',
                            new=f.name):
                try:
                    reload_storage_policies()
                except SystemExit as e:
                    err_msg = str(e)
                else:
                    self.fail('SystemExit not raised')
        parts = [
            'Invalid Storage Policy Configuration',
            'Duplicate index',
        ]
        for expected in parts:
            self.assert_(expected in err_msg,
                         '%s was not in %s' % (expected, err_msg))

    def test_storage_policy_ordering(self):
        test_policies = StoragePolicyCollection([
            StoragePolicy(0, 'zero', is_default=True),
            StoragePolicy(503, 'error'),
            StoragePolicy(204, 'empty'),
            StoragePolicy(404, 'missing'),
        ])
        self.assertEqual([0, 204, 404, 503],
                         [int(p) for p in sorted(list(test_policies))])

        p503 = test_policies[503]
        self.assertTrue(501 < p503 < 507)

    def test_get_object_ring(self):
        test_policies = [
            StoragePolicy(0, 'aay', True),
            StoragePolicy(1, 'bee', False),
            StoragePolicy(2, 'cee', False)
        ]
        policies = StoragePolicyCollection(test_policies)

        class NamedFakeRing(FakeRing):
            def __init__(self, swift_dir, ring_name=None):
                self.ring_name = ring_name
                super(NamedFakeRing, self).__init__()

        with mock.patch('swift.common.storage_policy.Ring', new=NamedFakeRing):
            for policy in policies:
                self.assertFalse(policy.object_ring)
                ring = policies.get_object_ring(int(policy), '/path/not/used')
                self.assertEqual(ring.ring_name, policy.ring_name)
                self.assertTrue(policy.object_ring)
                self.assert_(isinstance(policy.object_ring, NamedFakeRing))

        def blow_up(*args, **kwargs):
            raise Exception('kaboom!')

        with mock.patch('swift.common.storage_policy.Ring', new=blow_up):
            for policy in policies:
                policy.load_ring('/path/not/used')
                expected = policies.get_object_ring(int(policy),
                                                    '/path/not/used')
                self.assertEqual(policy.object_ring, expected)

        # bad policy index
        self.assertRaises(PolicyError, policies.get_object_ring, 99,
                          '/path/not/used')

    def test_singleton_passthrough(self):
        test_policies = [
            StoragePolicy(0, 'aay', True),
            StoragePolicy(1, 'bee', False),
            StoragePolicy(2, 'cee', False)
        ]
        with patch_policies(test_policies):
            for policy in POLICIES:
                self.assertEqual(POLICIES[int(policy)], policy)

    def test_storage_policy_get_options(self):
        policy = StoragePolicy(1, 'gold', True, False)
        self.assertEqual({
            'name': 'gold',
            'default': True,
            'deprecated': False
        }, policy.get_options())

        policy = StoragePolicy(1, 'gold', False, True)
        self.assertEqual({
            'name': 'gold',
            'default': False,
            'deprecated': True
        }, policy.get_options())
Exemple #29
0
import os
import time
import string
from shutil import rmtree
from hashlib import md5
from tempfile import mkdtemp
from test.unit import FakeLogger, patch_policies
from swift.obj import auditor
from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash, \
    get_data_dir, DiskFileManager, AuditLocation
from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
    storage_directory
from swift.common.storage_policy import StoragePolicy


_mocked_policies = [StoragePolicy(0, 'zero', False),
                    StoragePolicy(1, 'one', True)]


@patch_policies(_mocked_policies)
class TestAuditor(unittest.TestCase):

    def setUp(self):
        self.testdir = os.path.join(mkdtemp(), 'tmp_test_object_auditor')
        self.devices = os.path.join(self.testdir, 'node')
        self.rcache = os.path.join(self.testdir, 'object.recon')
        self.logger = FakeLogger()
        rmtree(self.testdir, ignore_errors=1)
        mkdirs(os.path.join(self.devices, 'sda'))
        os.mkdir(os.path.join(self.devices, 'sdb'))
Exemple #30
0
import string
from shutil import rmtree
from hashlib import md5
from tempfile import mkdtemp
from test.unit import FakeLogger, patch_policies, make_timestamp_iter, \
    DEFAULT_TEST_EC_TYPE
from swift.obj import auditor
from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash, \
    get_data_dir, DiskFileManager, ECDiskFileManager, AuditLocation, \
    clear_auditor_status, get_auditor_status
from swift.common.utils import mkdirs, normalize_timestamp, Timestamp
from swift.common.storage_policy import ECStoragePolicy, StoragePolicy, \
    POLICIES

_mocked_policies = [
    StoragePolicy(0, 'zero', False),
    StoragePolicy(1, 'one', True),
    ECStoragePolicy(2,
                    'two',
                    ec_type=DEFAULT_TEST_EC_TYPE,
                    ec_ndata=2,
                    ec_nparity=1,
                    ec_segment_size=4096),
]


def works_only_once(callable_thing, exception):
    called = [False]

    def only_once(*a, **kw):
        if called[0]:
            func = account_func
        resp = func(ipaddr,
                    port,
                    device,
                    partition,
                    method,
                    path,
                    headers=headers,
                    query_string=query_string)
        return resp

    return http_connect


@patch_policies(
    [StoragePolicy(0, 'zero', True, object_ring=FakeRing(replicas=1))])
class TestObjectSysmeta(unittest.TestCase):
    '''Tests object sysmeta is correctly handled by combination
    of proxy server and object server.
    '''
    def _assertStatus(self, resp, expected):
        self.assertEqual(resp.status_int, expected,
                         'Expected %d, got %s' % (expected, resp.status))

    def _assertInHeaders(self, resp, expected):
        for key, val in expected.items():
            self.assertTrue(key in resp.headers,
                            'Header %s missing from %s' % (key, resp.headers))
            self.assertEqual(
                val, resp.headers[key], 'Expected header %s:%s, got %s:%s' %
                (key, val, key, resp.headers[key]))