def test_headers_to_account_info_storage_policies(self): headers = { 'x-account-storage-policy-zero-object-count': '13', 'x-account-storage-policy-zero-container-count': '120', 'x-account-storage-policy-zero-bytes-used': '1002', 'x-account-storage-policy-one-object-count': '10', 'x-account-storage-policy-one-container-count': '20', } spc = StoragePolicyCollection( [StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False)]) with PatchPolicies(spc): resp = headers_to_account_info(headers.items(), 200) self.assertEqual( resp['storage_policies'], { 0: { 'object_count': 13, 'container_count': 120, 'bytes': 1002 }, 1: { 'object_count': 10, 'container_count': 20, 'bytes': 0 }, })
def test_storage_policy_repr(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False), ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', ec_ndata=10, ec_nparity=3) ] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assert_(policy.__class__.__name__ in policy_repr) self.assert_('is_default=%s' % policy.is_default in policy_repr) self.assert_('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assert_(policy.name in policy_repr) if policy.policy_type == EC_POLICY: self.assert_('ec_type=%s' % policy.ec_type in policy_repr) self.assert_('ec_ndata=%s' % policy.ec_ndata in policy_repr) self.assert_('ec_nparity=%s' % policy.ec_nparity in policy_repr) self.assert_('ec_segment_size=%s' % policy.ec_segment_size in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assert_(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr)
def test_get_object_ring(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False) ] policies = StoragePolicyCollection(test_policies) class NamedFakeRing(FakeRing): def __init__(self, swift_dir, ring_name=None): self.ring_name = ring_name super(NamedFakeRing, self).__init__() with mock.patch('swift.common.storage_policy.Ring', new=NamedFakeRing): for policy in policies: self.assertFalse(policy.object_ring) ring = policies.get_object_ring(int(policy), '/path/not/used') self.assertEqual(ring.ring_name, policy.ring_name) self.assertTrue(policy.object_ring) self.assert_(isinstance(policy.object_ring, NamedFakeRing)) def blow_up(*args, **kwargs): raise Exception('kaboom!') with mock.patch('swift.common.storage_policy.Ring', new=blow_up): for policy in policies: policy.load_ring('/path/not/used') expected = policies.get_object_ring(int(policy), '/path/not/used') self.assertEqual(policy.object_ring, expected) # bad policy index self.assertRaises(PolicyError, policies.get_object_ring, 99, '/path/not/used')
def test_validate_policies_names(self): # duplicate names test_policies = [StoragePolicy(0, 'zero', True), StoragePolicy(1, 'zero', False), StoragePolicy(2, 'two', False)] self.assertRaises(PolicyError, StoragePolicyCollection, test_policies)
def patch_policies(thing_or_policies=None, legacy_only=False, with_ec_default=False, fake_ring_args=None): if isinstance(thing_or_policies, ( Iterable, storage_policy.StoragePolicyCollection)): return PatchPolicies(thing_or_policies, fake_ring_args=fake_ring_args) if legacy_only: default_policies = [ StoragePolicy(0, name='legacy', is_default=True), ] default_ring_args = [{}] elif with_ec_default: default_policies = [ ECStoragePolicy(0, name='ec', is_default=True, ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4, ec_segment_size=4096), StoragePolicy(1, name='unu'), ] default_ring_args = [{'replicas': 14}, {}] else: default_policies = [ StoragePolicy(0, name='nulo', is_default=True), StoragePolicy(1, name='unu'), ] default_ring_args = [{}, {}] fake_ring_args = fake_ring_args or default_ring_args decorator = PatchPolicies(default_policies, fake_ring_args=fake_ring_args) if not thing_or_policies: return decorator else: # it's a thing, we return the wrapped thing instead of the decorator return decorator(thing_or_policies)
def test_get_ring_names(self): self.recon_instance.server_type = 'not-object' self.assertEqual(self.recon_instance._get_ring_names(), ['not-object']) self.recon_instance.server_type = 'object' with patch_policies([StoragePolicy(0, 'zero', is_default=True)]): self.assertEqual(self.recon_instance._get_ring_names(), ['object']) with patch_policies([ StoragePolicy(0, 'zero', is_default=True), StoragePolicy(1, 'one') ]): self.assertEqual(self.recon_instance._get_ring_names(), ['object', 'object-1']) self.assertEqual(self.recon_instance._get_ring_names('0'), ['object']) self.assertEqual(self.recon_instance._get_ring_names('zero'), ['object']) self.assertEqual(self.recon_instance._get_ring_names('1'), ['object-1']) self.assertEqual(self.recon_instance._get_ring_names('one'), ['object-1']) self.assertEqual(self.recon_instance._get_ring_names('3'), []) self.assertEqual(self.recon_instance._get_ring_names('wrong'), [])
def test_singleton_passthrough(self): test_policies = [StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False)] with patch_policies(test_policies): for policy in POLICIES: self.assertEqual(POLICIES[int(policy)], policy)
def test_storage_policy_get_options(self): policy = StoragePolicy(1, 'gold', True, False) self.assertEqual({'name': 'gold', 'default': True, 'deprecated': False}, policy.get_options()) policy = StoragePolicy(1, 'gold', False, True) self.assertEqual({'name': 'gold', 'default': False, 'deprecated': True}, policy.get_options())
def test_storage_policy_ordering(self): test_policies = StoragePolicyCollection([ StoragePolicy(0, 'zero', is_default=True), StoragePolicy(503, 'error'), StoragePolicy(204, 'empty'), StoragePolicy(404, 'missing'), ]) self.assertEqual([0, 204, 404, 503], [int(p) for p in sorted(list(test_policies))]) p503 = test_policies[503] self.assertTrue(501 < p503 < 507)
def test_validate_policy_params(self): StoragePolicy(0, 'name') # sanity # bogus indexes self.assertRaises(PolicyError, StoragePolicy, 'x', 'name') self.assertRaises(PolicyError, StoragePolicy, -1, 'name') # non-zero Policy-0 self.assertRaisesWithMessage(PolicyError, 'reserved', StoragePolicy, 1, 'policy-0') # deprecate default self.assertRaisesWithMessage(PolicyError, 'Deprecated policy can not be default', StoragePolicy, 1, 'Policy-1', is_default=True, is_deprecated=True) # weird names names = ( '', 'name_foo', 'name\nfoo', 'name foo', u'name \u062a', 'name \xd8\xaa', ) for name in names: self.assertRaisesWithMessage(PolicyError, 'Invalid name', StoragePolicy, 1, name)
def test_names_are_normalized(self): test_policies = [StoragePolicy(0, 'zero', True), StoragePolicy(1, 'ZERO', False)] self.assertRaises(PolicyError, StoragePolicyCollection, test_policies) policies = StoragePolicyCollection([StoragePolicy(0, 'zEro', True), StoragePolicy(1, 'One', False)]) pol0 = policies[0] pol1 = policies[1] for name in ('zero', 'ZERO', 'zErO', 'ZeRo'): self.assertEqual(pol0, policies.get_by_name(name)) self.assertEqual(policies.get_by_name(name).name, 'zEro') for name in ('one', 'ONE', 'oNe', 'OnE'): self.assertEqual(pol1, policies.get_by_name(name)) self.assertEqual(policies.get_by_name(name).name, 'One')
def test_policies_type_attribute(self): test_policies = [ StoragePolicy(0, 'zero', is_default=True), StoragePolicy(1, 'one'), StoragePolicy(2, 'two'), StoragePolicy(3, 'three', is_deprecated=True), ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', ec_ndata=10, ec_nparity=3), ] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.get_by_index(0).policy_type, REPL_POLICY) self.assertEquals(policies.get_by_index(1).policy_type, REPL_POLICY) self.assertEquals(policies.get_by_index(2).policy_type, REPL_POLICY) self.assertEquals(policies.get_by_index(3).policy_type, REPL_POLICY) self.assertEquals(policies.get_by_index(10).policy_type, EC_POLICY)
def test_quorum_size_replication(self): expected_sizes = {1: 1, 2: 2, 3: 2, 4: 3, 5: 3} for n, expected in expected_sizes.items(): policy = StoragePolicy(0, 'zero', object_ring=FakeRing(replicas=n)) self.assertEqual(policy.quorum, expected)
def test_storage_policy_repr(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False) ] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assert_(policy.__class__.__name__ in policy_repr) self.assert_('is_default=%s' % policy.is_default in policy_repr) self.assert_('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assert_(policy.name in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assert_(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr)
def setUp(self): skip_if_no_xattrs() self.logger = FakeLogger() self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') shutil.rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) os.mkdir(self.devices) self.rb = ring.RingBuilder(8, 6.0, 1) for i in range(6): ip = "127.0.0.%s" % i self.rb.add_dev({ 'id': i, 'region': 0, 'zone': 0, 'weight': 1, 'ip': ip, 'port': 10000, 'device': 'sda1' }) self.rb.rebalance(seed=1) self.existing_device = 'sda1' os.mkdir(os.path.join(self.devices, self.existing_device)) self.objects = os.path.join(self.devices, self.existing_device, 'objects') os.mkdir(self.objects) self._hash = utils.hash_path('a/c/o') digest = binascii.unhexlify(self._hash) part = struct.unpack_from('>I', digest)[0] >> 24 self.next_part = struct.unpack_from('>I', digest)[0] >> 23 self.objdir = os.path.join(self.objects, str(part), self._hash[-3:], self._hash) os.makedirs(self.objdir) self.object_fname = "1278553064.00000.data" self.objname = os.path.join(self.objdir, self.object_fname) with open(self.objname, "wb") as dummy: dummy.write(b"Hello World!") write_metadata(dummy, {'name': '/a/c/o', 'Content-Length': '12'}) test_policies = [StoragePolicy(0, 'platin', True)] storage_policy._POLICIES = StoragePolicyCollection(test_policies) self.expected_dir = os.path.join(self.objects, str(self.next_part), self._hash[-3:], self._hash) self.expected_file = os.path.join(self.expected_dir, self.object_fname)
def _setup_object(self, condition=None): attempts = [] for _ in range(50): account = 'a' container = 'c' obj = 'o-' + str(uuid.uuid4()) self._hash = utils.hash_path(account, container, obj) digest = binascii.unhexlify(self._hash) self.part = struct.unpack_from('>I', digest)[0] >> 24 self.next_part = struct.unpack_from('>I', digest)[0] >> 23 path = os.path.join(os.path.sep, account, container, obj) # There's 1/512 chance that both old and new parts will be 0; # that's not a terribly interesting case, as there's nothing to do attempts.append((self.part, self.next_part, 2**PART_POWER)) if (self.part != self.next_part and (condition(self.part) if condition else True)): break else: self.fail( 'Failed to setup object satisfying test preconditions %s' % attempts) shutil.rmtree(self.objects, ignore_errors=True) os.mkdir(self.objects) self.objdir = os.path.join(self.objects, str(self.part), self._hash[-3:], self._hash) os.makedirs(self.objdir) self.object_fname = utils.Timestamp.now().internal + ".data" self.objname = os.path.join(self.objdir, self.object_fname) with open(self.objname, "wb") as dummy: dummy.write(b"Hello World!") write_metadata(dummy, {'name': path, 'Content-Length': '12'}) self.policy = StoragePolicy(0, 'platinum', True) storage_policy._POLICIES = StoragePolicyCollection([self.policy]) self.part_dir = os.path.join(self.objects, str(self.part)) self.suffix_dir = os.path.join(self.part_dir, self._hash[-3:]) self.next_part_dir = os.path.join(self.objects, str(self.next_part)) self.next_suffix_dir = os.path.join(self.next_part_dir, self._hash[-3:]) self.expected_dir = os.path.join(self.next_suffix_dir, self._hash) self.expected_file = os.path.join(self.expected_dir, self.object_fname)
def test_storage_policy_get_options(self): policy = StoragePolicy(1, 'gold', True, False) self.assertEqual({ 'name': 'gold', 'default': True, 'deprecated': False }, policy.get_options()) policy = StoragePolicy(1, 'gold', False, True) self.assertEqual({ 'name': 'gold', 'default': False, 'deprecated': True }, policy.get_options())
def test_load_from_config(self, tempdir): conf_path = os.path.join(tempdir, 'interal_client.conf') conf_body = """ [DEFAULT] swift_dir = %s [pipeline:main] pipeline = catch_errors cache proxy-server [app:proxy-server] use = egg:swift#proxy auto_create_account_prefix = - [filter:cache] use = egg:swift#memcache [filter:catch_errors] use = egg:swift#catch_errors """ % tempdir with open(conf_path, 'w') as f: f.write(dedent(conf_body)) account_ring_path = os.path.join(tempdir, 'account.ring.gz') write_fake_ring(account_ring_path) container_ring_path = os.path.join(tempdir, 'container.ring.gz') write_fake_ring(container_ring_path) object_ring_path = os.path.join(tempdir, 'object.ring.gz') write_fake_ring(object_ring_path) with patch_policies([StoragePolicy(0, 'legacy', True)]): client = internal_client.InternalClient(conf_path, 'test', 1) self.assertEqual(client.account_ring, client.app.app.app.account_ring) self.assertEqual(client.account_ring.serialized_path, account_ring_path) self.assertEqual(client.container_ring, client.app.app.app.container_ring) self.assertEqual(client.container_ring.serialized_path, container_ring_path) object_ring = client.app.app.app.get_object_ring(0) self.assertEqual(client.get_object_ring(0), object_ring) self.assertEqual(object_ring.serialized_path, object_ring_path) self.assertEquals(client.auto_create_account_prefix, '-')
def test_deprecate_policies(self): # deprecation specified test_policies = [StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False, is_deprecated=True)] policies = StoragePolicyCollection(test_policies) self.assertEqual(policies.default, test_policies[0]) self.assertEqual(policies.default.name, 'zero') self.assertEqual(len(policies), 3) # multiple policies requires default test_policies = [StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', False, is_deprecated=True), StoragePolicy(2, 'two', False)] self.assertRaisesWithMessage( PolicyError, 'Unable to find default policy', StoragePolicyCollection, test_policies)
class FakeAppReadline(object): def __call__(self, env, start_response): start_response('200 OK', [('Content-Type', 'text/plain'), ('Content-Length', '8')]) env['wsgi.input'].readline() return ["FAKE APP"] def start_response(*args): pass @patch_policies([StoragePolicy(0, 'zero', False)]) class TestProxyLogging(unittest.TestCase): def setUp(self): pass def _log_parts(self, app, should_be_empty=False): info_calls = app.access_logger.log_dict['info'] if should_be_empty: self.assertEquals([], info_calls) else: self.assertEquals(1, len(info_calls)) return info_calls[0][0][0].split(' ') def assertTiming(self, exp_metric, app, exp_timing=None): timing_calls = app.access_logger.log_dict['timing']
[(k, v) for k, v in response.headers.items()]) return iter(response.body) class FakeCache(FakeMemcache): def __init__(self, stub=None, **pre_cached): super(FakeCache, self).__init__() if pre_cached: self.store.update(pre_cached) self.stub = stub def get(self, key): return self.stub or self.store.get(key) @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) class TestFuncs(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing()) def test_get_info(self): app = FakeApp() # Do a non cached call to account env = {} info_a = get_info(app, env, 'a') # Check that you got proper info self.assertEqual(info_a['status'], 200) self.assertEqual(info_a['bytes'], 6666)
def setup_servers(the_object_server=object_server, extra_conf=None): """ Setup proxy, account, container and object servers using a set of fake rings and policies. :param the_object_server: The object server module to use (optional, defaults to swift.obj.server) :param extra_conf: A dict of config options that will update the basic config passed to all server instances. :returns: A dict containing the following entries: orig_POLICIES: the value of storage_policy.POLICIES prior to it being patched with fake policies orig_SysLogHandler: the value of utils.SysLogHandler prior to it being patched testdir: root directory used for test files test_POLICIES: a StoragePolicyCollection of fake policies test_servers: a tuple of test server instances test_sockets: a tuple of sockets used by test servers test_coros: a tuple of greenthreads in which test servers are running """ context = { "orig_POLICIES": storage_policy._POLICIES, "orig_SysLogHandler": utils.SysLogHandler} utils.HASH_PATH_SUFFIX = b'endcap' utils.SysLogHandler = mock.MagicMock() # Since we're starting up a lot here, we're going to test more than # just chunked puts; we're also going to test parts of # proxy_server.Application we couldn't get to easily otherwise. context["testdir"] = _testdir = \ os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked') mkdirs(_testdir) rmtree(_testdir) for drive in ('sda1', 'sdb1', 'sdc1', 'sdd1', 'sde1', 'sdf1', 'sdg1', 'sdh1', 'sdi1', 'sdj1', 'sdk1', 'sdl1'): mkdirs(os.path.join(_testdir, drive, 'tmp')) conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', 'allow_versions': 't'} if extra_conf: conf.update(extra_conf) prolis = listen_zero() acc1lis = listen_zero() acc2lis = listen_zero() con1lis = listen_zero() con2lis = listen_zero() obj1lis = listen_zero() obj2lis = listen_zero() obj3lis = listen_zero() obj4lis = listen_zero() obj5lis = listen_zero() obj6lis = listen_zero() objsocks = [obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis] context["test_sockets"] = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') account_devs = [ {'port': acc1lis.getsockname()[1]}, {'port': acc2lis.getsockname()[1]}, ] write_fake_ring(account_ring_path, *account_devs) container_ring_path = os.path.join(_testdir, 'container.ring.gz') container_devs = [ {'port': con1lis.getsockname()[1]}, {'port': con2lis.getsockname()[1]}, ] write_fake_ring(container_ring_path, *container_devs) storage_policy._POLICIES = storage_policy.StoragePolicyCollection([ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False), ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096), ECStoragePolicy(4, 'ec-dup', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096, ec_duplication_factor=2)]) obj_rings = { 0: ('sda1', 'sdb1'), 1: ('sdc1', 'sdd1'), 2: ('sde1', 'sdf1'), # sdg1, sdh1, sdi1 taken by policy 3 (see below) } for policy_index, devices in obj_rings.items(): policy = storage_policy.POLICIES[policy_index] obj_ring_path = os.path.join(_testdir, policy.ring_name + '.ring.gz') obj_devs = [ {'port': objsock.getsockname()[1], 'device': dev} for objsock, dev in zip(objsocks, devices)] write_fake_ring(obj_ring_path, *obj_devs) # write_fake_ring can't handle a 3-element ring, and the EC policy needs # at least 6 devs to work with (ec_k=2, ec_m=1, duplication_factor=2), # so we do it manually devs = [{'id': 0, 'zone': 0, 'device': 'sdg1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1]}, {'id': 1, 'zone': 0, 'device': 'sdh1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}, {'id': 2, 'zone': 0, 'device': 'sdi1', 'ip': '127.0.0.1', 'port': obj3lis.getsockname()[1]}, {'id': 3, 'zone': 0, 'device': 'sdj1', 'ip': '127.0.0.1', 'port': obj4lis.getsockname()[1]}, {'id': 4, 'zone': 0, 'device': 'sdk1', 'ip': '127.0.0.1', 'port': obj5lis.getsockname()[1]}, {'id': 5, 'zone': 0, 'device': 'sdl1', 'ip': '127.0.0.1', 'port': obj6lis.getsockname()[1]}] pol3_replica2part2dev_id = [[0, 1, 2, 0], [1, 2, 0, 1], [2, 0, 1, 2]] pol4_replica2part2dev_id = [[0, 1, 2, 3], [1, 2, 3, 4], [2, 3, 4, 5], [3, 4, 5, 0], [4, 5, 0, 1], [5, 0, 1, 2]] obj3_ring_path = os.path.join( _testdir, storage_policy.POLICIES[3].ring_name + '.ring.gz') part_shift = 30 with closing(GzipFile(obj3_ring_path, 'wb')) as fh: pickle.dump(RingData(pol3_replica2part2dev_id, devs, part_shift), fh) obj4_ring_path = os.path.join( _testdir, storage_policy.POLICIES[4].ring_name + '.ring.gz') part_shift = 30 with closing(GzipFile(obj4_ring_path, 'wb')) as fh: pickle.dump(RingData(pol4_replica2part2dev_id, devs, part_shift), fh) prosrv = proxy_server.Application(conf, logger=debug_logger('proxy')) for policy in storage_policy.POLICIES: # make sure all the rings are loaded prosrv.get_object_ring(policy.idx) # don't lose this one! context["test_POLICIES"] = storage_policy._POLICIES acc1srv = account_server.AccountController( conf, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( conf, logger=debug_logger('acct2')) con1srv = container_server.ContainerController( conf, logger=debug_logger('cont1')) con2srv = container_server.ContainerController( conf, logger=debug_logger('cont2')) obj1srv = the_object_server.ObjectController( conf, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( conf, logger=debug_logger('obj2')) obj3srv = the_object_server.ObjectController( conf, logger=debug_logger('obj3')) obj4srv = the_object_server.ObjectController( conf, logger=debug_logger('obj4')) obj5srv = the_object_server.ObjectController( conf, logger=debug_logger('obj5')) obj6srv = the_object_server.ObjectController( conf, logger=debug_logger('obj6')) context["test_servers"] = \ (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv, obj3srv, obj4srv, obj5srv, obj6srv) nl = NullLogger() logging_prosv = proxy_logging.ProxyLoggingMiddleware( listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger) prospa = spawn(wsgi.server, prolis, logging_prosv, nl) acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl) acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl) con1spa = spawn(wsgi.server, con1lis, con1srv, nl) con2spa = spawn(wsgi.server, con2lis, con2srv, nl) obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl) obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl) obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl) obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl) context["test_coros"] = \ (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa, obj4spa, obj5spa, obj6spa) # Create account ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create another account # used for account-to-account tests ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a1') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a1', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create containers, 1 per test policy sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) # Create container in other account # used for account-to-account tests sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a1/c1 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write( 'PUT /v1/a/c1 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: one\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, \ "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write( 'PUT /v1/a/c2 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: two\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, \ "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) return context
class TestAccountBroker(unittest.TestCase): """Tests for AccountBroker""" def test_creation(self): # Test AccountBroker.__init__ broker = AccountBroker(':memory:', account='a') self.assertEqual(broker.db_file, ':memory:') try: with broker.get() as conn: pass except DatabaseConnectionError as e: self.assertTrue(hasattr(e, 'path')) self.assertEquals(e.path, ':memory:') self.assertTrue(hasattr(e, 'msg')) self.assertEquals(e.msg, "DB doesn't exist") except Exception as e: self.fail("Unexpected exception raised: %r" % e) else: self.fail("Expected a DatabaseConnectionError exception") broker.initialize(Timestamp('1').internal) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1) def test_exception(self): # Test AccountBroker throwing a conn away after exception first_conn = None broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) with broker.get() as conn: first_conn = conn try: with broker.get() as conn: self.assertEqual(first_conn, conn) raise Exception('OMG') except Exception: pass self.assert_(broker.conn is None) def test_empty(self): # Test AccountBroker.empty broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) self.assert_(broker.empty()) broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) self.assert_(not broker.empty()) sleep(.00001) broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) self.assert_(broker.empty()) def test_is_status_deleted(self): # Test AccountBroker.is_status_deleted broker1 = AccountBroker(':memory:', account='a') broker1.initialize(Timestamp(time()).internal) self.assert_(not broker1.is_status_deleted()) broker1.delete_db(Timestamp(time()).internal) self.assert_(broker1.is_status_deleted()) broker2 = AccountBroker(':memory:', account='a') broker2.initialize(Timestamp(time()).internal) # Set delete_timestamp greater than put_timestamp broker2.merge_timestamps(time(), Timestamp(time()).internal, Timestamp(time() + 999).internal) self.assert_(broker2.is_status_deleted()) def test_reclaim(self): broker = AccountBroker(':memory:', account='test_account') broker.initialize(Timestamp('1').internal) broker.put_container('c', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('c', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) broker.reclaim(Timestamp(time()).internal, time()) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) # Test reclaim after deletion. Create 3 test containers broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx) broker.reclaim(Timestamp(time()).internal, time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assert_(containers is None) # self.assert_(account_name is None) # Now delete the account broker.delete_db(Timestamp(time()).internal) broker.reclaim(Timestamp(time()).internal, time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assertEqual(account_name, 'test_account') # self.assertEqual(len(containers), 3) # self.assert_('x' in containers) # self.assert_('y' in containers) # self.assert_('z' in containers) # self.assert_('a' not in containers) def test_delete_db_status(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) start = ts.next() broker = AccountBroker(':memory:', account='a') broker.initialize(start) info = broker.get_info() self.assertEqual(info['put_timestamp'], Timestamp(start).internal) self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], Timestamp(start).internal) # delete it delete_timestamp = ts.next() broker.delete_db(delete_timestamp) info = broker.get_info() self.assertEqual(info['put_timestamp'], Timestamp(start).internal) self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp) def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual( conn.execute("SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) def test_put_container(self): # Test AccountBroker.put_container broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) # Create initial container timestamp = Timestamp(time()).internal broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual( conn.execute("SELECT put_timestamp FROM container").fetchone() [0], timestamp) self.assertEqual( conn.execute("SELECT deleted FROM container").fetchone()[0], 0) # Reput same event broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual( conn.execute("SELECT put_timestamp FROM container").fetchone() [0], timestamp) self.assertEqual( conn.execute("SELECT deleted FROM container").fetchone()[0], 0) # Put new event sleep(.00001) timestamp = Timestamp(time()).internal broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual( conn.execute("SELECT put_timestamp FROM container").fetchone() [0], timestamp) self.assertEqual( conn.execute("SELECT deleted FROM container").fetchone()[0], 0) # Put old event otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual( conn.execute("SELECT put_timestamp FROM container").fetchone() [0], timestamp) self.assertEqual( conn.execute("SELECT deleted FROM container").fetchone()[0], 0) # Put old delete event dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual( conn.execute("SELECT put_timestamp FROM container").fetchone() [0], timestamp) self.assertEqual( conn.execute( "SELECT delete_timestamp FROM container").fetchone()[0], dtimestamp) self.assertEqual( conn.execute("SELECT deleted FROM container").fetchone()[0], 0) # Put new delete event sleep(.00001) timestamp = Timestamp(time()).internal broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual( conn.execute( "SELECT delete_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual( conn.execute("SELECT deleted FROM container").fetchone()[0], 1) # Put new event sleep(.00001) timestamp = Timestamp(time()).internal broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual( conn.execute("SELECT name FROM container").fetchone()[0], '"{<container \'&\' name>}"') self.assertEqual( conn.execute("SELECT put_timestamp FROM container").fetchone() [0], timestamp) self.assertEqual( conn.execute("SELECT deleted FROM container").fetchone()[0], 0) def test_get_info(self): # Test AccountBroker.get_info broker = AccountBroker(':memory:', account='test1') broker.initialize(Timestamp('1').internal) info = broker.get_info() self.assertEqual(info['account'], 'test1') self.assertEqual(info['hash'], '00000000000000000000000000000000') self.assertEqual(info['put_timestamp'], Timestamp(1).internal) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], Timestamp(1).internal) info = broker.get_info() self.assertEqual(info['container_count'], 0) broker.put_container('c1', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c1', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 0) def test_list_containers_iter(self): # Test AccountBroker.list_containers_iter broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) for cont1 in xrange(4): for cont2 in xrange(125): broker.put_container('%d-%04d' % (cont1, cont2), Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) for cont in xrange(125): broker.put_container('2-0051-%04d' % cont, Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) for cont in xrange(125): broker.put_container('3-%04d-0049' % cont, Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(100, '', None, None, '') self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0099') listing = broker.list_containers_iter(100, '', '0-0050', None, '') self.assertEqual(len(listing), 50) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0049') listing = broker.list_containers_iter(100, '0-0099', None, None, '') self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '1-0074') listing = broker.list_containers_iter(55, '1-0074', None, None, '') self.assertEqual(len(listing), 55) self.assertEqual(listing[0][0], '1-0075') self.assertEqual(listing[-1][0], '2-0004') listing = broker.list_containers_iter(10, '', None, '0-01', '') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = broker.list_containers_iter(10, '', None, '0-01', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = broker.list_containers_iter(10, '', None, '0-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0009') listing = broker.list_containers_iter(10, '', None, '', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['0-', '1-', '2-', '3-']) listing = broker.list_containers_iter(10, '2-', None, None, '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['3-']) listing = broker.list_containers_iter(10, '', None, '2', '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['2-']) listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '2-0051') self.assertEqual(listing[1][0], '2-0051-') self.assertEqual(listing[2][0], '2-0052') self.assertEqual(listing[-1][0], '2-0059') listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-') self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], [ '3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050' ]) broker.put_container('3-0049-', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(10, '3-0048', None, None, None) self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], [ '3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049' ]) listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-') self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], [ '3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051', '3-0051-', '3-0052', '3-0052-', '3-0053' ]) listing = broker.list_containers_iter(10, None, None, '3-0049-', '-') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['3-0049-', '3-0049-0049']) def test_double_check_trailing_delimiter(self): # Test AccountBroker.list_containers_iter for an # account that has an odd container with a trailing delimiter broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) broker.put_container('a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('a-', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('a-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('a-a-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('a-a-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('a-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('b-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('b-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker.put_container('c', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(15, None, None, None, None) self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], [ 'a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', 'b-a', 'b-b', 'c' ]) listing = broker.list_containers_iter(15, None, None, '', '-') self.assertEqual(len(listing), 5) self.assertEqual([row[0] for row in listing], ['a', 'a-', 'b', 'b-', 'c']) listing = broker.list_containers_iter(15, None, None, 'a-', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['a-', 'a-a', 'a-a-', 'a-b']) listing = broker.list_containers_iter(15, None, None, 'b-', '-') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['b-a', 'b-b']) def test_chexor(self): broker = AccountBroker(':memory:', account='a') broker.initialize(Timestamp('1').internal) broker.put_container('a', Timestamp(1).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) broker.put_container('b', Timestamp(2).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) hasha = hashlib.md5( '%s-%s' % ('a', "%s-%s-%s-%s" % (Timestamp(1).internal, Timestamp(0).internal, 0, 0))).digest() hashb = hashlib.md5( '%s-%s' % ('b', "%s-%s-%s-%s" % (Timestamp(2).internal, Timestamp(0).internal, 0, 0))).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) broker.put_container('b', Timestamp(3).internal, Timestamp(0).internal, 0, 0, POLICIES.default.idx) hashb = hashlib.md5( '%s-%s' % ('b', "%s-%s-%s-%s" % (Timestamp(3).internal, Timestamp(0).internal, 0, 0))).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) def test_merge_items(self): broker1 = AccountBroker(':memory:', account='a') broker1.initialize(Timestamp('1').internal) broker2 = AccountBroker(':memory:', account='a') broker2.initialize(Timestamp('1').internal) broker1.put_container('a', Timestamp(1).internal, 0, 0, 0, POLICIES.default.idx) broker1.put_container('b', Timestamp(2).internal, 0, 0, 0, POLICIES.default.idx) id = broker1.get_info()['id'] broker2.merge_items( broker1.get_items_since(broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 2) self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items])) broker1.put_container('c', Timestamp(3).internal, 0, 0, 0, POLICIES.default.idx) broker2.merge_items( broker1.get_items_since(broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 3) self.assertEqual(['a', 'b', 'c'], sorted([rec['name'] for rec in items])) def test_load_old_pending_puts(self): # pending puts from pre-storage-policy account brokers won't contain # the storage policy index tempdir = mkdtemp() broker_path = os.path.join(tempdir, 'test-load-old.db') try: broker = AccountBroker(broker_path, account='real') broker.initialize(Timestamp(1).internal) with open(broker_path + '.pending', 'a+b') as pending: pending.write(':') pending.write( pickle.dumps( # name, put_timestamp, delete_timestamp, object_count, # bytes_used, deleted ('oldcon', Timestamp(200).internal, Timestamp(0).internal, 896, 9216695, 0 )).encode('base64')) broker._commit_puts() with broker.get() as conn: results = list( conn.execute(''' SELECT name, storage_policy_index FROM container ''')) self.assertEqual(len(results), 1) self.assertEqual(dict(results[0]), { 'name': 'oldcon', 'storage_policy_index': 0 }) finally: rmtree(tempdir) @patch_policies([ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', True), StoragePolicy(2, 'two', False), StoragePolicy(3, 'three', False) ]) def test_get_policy_stats(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') broker.initialize(ts.next()) # check empty policy_stats self.assertTrue(broker.empty()) policy_stats = broker.get_policy_stats() self.assertEqual(policy_stats, {}) # add some empty containers for policy in POLICIES: container_name = 'c-%s' % policy.name put_timestamp = ts.next() broker.put_container(container_name, put_timestamp, 0, 0, 0, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], 0) self.assertEqual(stats['bytes_used'], 0) # update the containers object & byte count for policy in POLICIES: container_name = 'c-%s' % policy.name put_timestamp = ts.next() count = policy.idx * 100 # good as any integer broker.put_container(container_name, put_timestamp, 0, count, count, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], count) self.assertEqual(stats['bytes_used'], count) # check all the policy_stats at once for policy_index, stats in policy_stats.items(): policy = POLICIES[policy_index] count = policy.idx * 100 # coupled with policy for test self.assertEqual(stats['object_count'], count) self.assertEqual(stats['bytes_used'], count) # now delete the containers one by one for policy in POLICIES: container_name = 'c-%s' % policy.name delete_timestamp = ts.next() broker.put_container(container_name, 0, delete_timestamp, 0, 0, policy.idx) policy_stats = broker.get_policy_stats() stats = policy_stats[policy.idx] self.assertEqual(stats['object_count'], 0) self.assertEqual(stats['bytes_used'], 0) @patch_policies( [StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', True)]) def test_policy_stats_tracking(self): ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') broker.initialize(ts.next()) # policy 0 broker.put_container('con1', ts.next(), 0, 12, 2798641, 0) broker.put_container('con1', ts.next(), 0, 13, 8156441, 0) # policy 1 broker.put_container('con2', ts.next(), 0, 7, 5751991, 1) broker.put_container('con2', ts.next(), 0, 8, 6085379, 1) stats = broker.get_policy_stats() self.assertEqual(len(stats), 2) self.assertEqual(stats[0]['object_count'], 13) self.assertEqual(stats[0]['bytes_used'], 8156441) self.assertEqual(stats[1]['object_count'], 8) self.assertEqual(stats[1]['bytes_used'], 6085379) # Break encapsulation here to make sure that there's only 2 rows in # the stats table. It's possible that there could be 4 rows (one per # put_container) but that they came out in the right order so that # get_policy_stats() collapsed them down to the right number. To prove # that's not so, we have to go peek at the broker's internals. with broker.get() as conn: nrows = conn.execute( "SELECT COUNT(*) FROM policy_stat").fetchall()[0][0] self.assertEqual(nrows, 2)
'device': 'sda1', 'ip': '', 'port': '' }, { 'device': 'sda1', 'ip': '', 'port': '' }, { 'device': 'sda1', 'ip': '', 'port': '' }] @unit.patch_policies([ StoragePolicy(0, 'zero', False, object_ring=unit.FakeRing()), StoragePolicy(1, 'one', True, object_ring=unit.FakeRing(replicas=4)) ]) class TestReaper(unittest.TestCase): def setUp(self): self.to_delete = [] self.myexp = ClientException("", http_host=None, http_port=None, http_device=None, http_status=404, http_reason=None) def tearDown(self): for todel in self.to_delete: shutil.rmtree(todel)
def test_validate_policies_defaults(self): # 0 explicit default test_policies = [ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False) ] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.default, test_policies[0]) self.assertEquals(policies.default.name, 'zero') # non-zero explicit default test_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', True) ] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.default, test_policies[2]) self.assertEquals(policies.default.name, 'two') # multiple defaults test_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', True), StoragePolicy(2, 'two', True) ] self.assertRaisesWithMessage(PolicyError, 'Duplicate default', StoragePolicyCollection, test_policies) # nothing specified test_policies = [] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.default, policies[0]) self.assertEquals(policies.default.name, 'Policy-0') # no default specified with only policy index 0 test_policies = [StoragePolicy(0, 'zero')] policies = StoragePolicyCollection(test_policies) self.assertEqual(policies.default, policies[0]) # no default specified with multiple policies test_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False) ] self.assertRaisesWithMessage(PolicyError, 'Unable to find default policy', StoragePolicyCollection, test_policies)
""" This patch is just a hook over handle_request to ensure that when get_controller is called the ObjectController class is patched to return a (possibly stubbed) ObjectController class. """ object_controller = proxy_server.ObjectController def handle_request(self, req): with mock.patch('swift.proxy.server.ObjectController', new=self.object_controller): return super(PatchedObjControllerApp, self).handle_request(req) @patch_policies( [StoragePolicy(0, 'zero', True, object_ring=FakeRing(max_more_nodes=9))]) class TestObjControllerWriteAffinity(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), logger=debug_logger()) self.app.request_node_count = lambda ring: 10000000 self.app.sort_nodes = lambda l: l # stop shuffling the primary nodes def test_iter_nodes_local_first_noops_when_no_affinity(self): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') self.app.write_affinity_is_local_fn = None object_ring = self.app.get_object_ring(None) all_nodes = object_ring.get_part_nodes(1)
{'device': 'sda1', 'ip': '', 'port': ''}] cont_nodes = [{'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}] @unit.patch_policies([StoragePolicy(0, 'zero', False, object_ring=unit.FakeRing()), StoragePolicy(1, 'one', True, object_ring=unit.FakeRing())]) class TestReaper(unittest.TestCase): def setUp(self): self.to_delete = [] self.myexp = ClientException("", http_host=None, http_port=None, http_device=None, http_status=404, http_reason=None ) def tearDown(self): for todel in self.to_delete:
class TestStoragePolicies(unittest.TestCase): def _conf(self, conf_str): conf_str = "\n".join(line.strip() for line in conf_str.split("\n")) conf = ConfigParser() conf.readfp(StringIO.StringIO(conf_str)) return conf @patch_policies([ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False), StoragePolicy(3, 'three', False, is_deprecated=True) ]) def test_swift_info(self): # the deprecated 'three' should not exist in expect expect = [{ 'default': True, 'name': 'zero' }, { 'name': 'two' }, { 'name': 'one' }] swift_info = POLICIES.get_policy_info() self.assertEquals(sorted(expect, key=lambda k: k['name']), sorted(swift_info, key=lambda k: k['name'])) @patch_policies def test_get_policy_string(self): self.assertEquals(get_policy_string('something', 0), 'something') self.assertEquals(get_policy_string('something', None), 'something') self.assertEquals(get_policy_string('something', 1), 'something' + '-1') self.assertRaises(PolicyError, get_policy_string, 'something', 99) def test_defaults(self): self.assertTrue(len(POLICIES) > 0) # test class functions default_policy = POLICIES.default self.assert_(default_policy.is_default) zero_policy = POLICIES.get_by_index(0) self.assert_(zero_policy.idx == 0) zero_policy_by_name = POLICIES.get_by_name(zero_policy.name) self.assert_(zero_policy_by_name.idx == 0) def test_storage_policy_repr(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False) ] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assert_(policy.__class__.__name__ in policy_repr) self.assert_('is_default=%s' % policy.is_default in policy_repr) self.assert_('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assert_(policy.name in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assert_(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr) def test_validate_policies_defaults(self): # 0 explicit default test_policies = [ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False) ] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.default, test_policies[0]) self.assertEquals(policies.default.name, 'zero') # non-zero explicit default test_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', True) ] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.default, test_policies[2]) self.assertEquals(policies.default.name, 'two') # multiple defaults test_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', True), StoragePolicy(2, 'two', True) ] self.assertRaisesWithMessage(PolicyError, 'Duplicate default', StoragePolicyCollection, test_policies) # nothing specified test_policies = [] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.default, policies[0]) self.assertEquals(policies.default.name, 'Policy-0') # no default specified with only policy index 0 test_policies = [StoragePolicy(0, 'zero')] policies = StoragePolicyCollection(test_policies) self.assertEqual(policies.default, policies[0]) # no default specified with multiple policies test_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False) ] self.assertRaisesWithMessage(PolicyError, 'Unable to find default policy', StoragePolicyCollection, test_policies) def test_deprecate_policies(self): # deprecation specified test_policies = [ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False, is_deprecated=True) ] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.default, test_policies[0]) self.assertEquals(policies.default.name, 'zero') self.assertEquals(len(policies), 3) # multiple policies requires default test_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', False, is_deprecated=True), StoragePolicy(2, 'two', False) ] self.assertRaisesWithMessage(PolicyError, 'Unable to find default policy', StoragePolicyCollection, test_policies) def test_validate_policies_indexes(self): # duplicate indexes test_policies = [ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(1, 'two', False) ] self.assertRaises(PolicyError, StoragePolicyCollection, test_policies) def test_validate_policy_params(self): StoragePolicy(0, 'name') # sanity # bogus indexes self.assertRaises(PolicyError, StoragePolicy, 'x', 'name') self.assertRaises(PolicyError, StoragePolicy, -1, 'name') # non-zero Policy-0 self.assertRaisesWithMessage(PolicyError, 'reserved', StoragePolicy, 1, 'policy-0') # deprecate default self.assertRaisesWithMessage(PolicyError, 'Deprecated policy can not be default', StoragePolicy, 1, 'Policy-1', is_default=True, is_deprecated=True) # weird names names = ( '', 'name_foo', 'name\nfoo', 'name foo', u'name \u062a', 'name \xd8\xaa', ) for name in names: self.assertRaisesWithMessage(PolicyError, 'Invalid name', StoragePolicy, 1, name) def test_validate_policies_names(self): # duplicate names test_policies = [ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'zero', False), StoragePolicy(2, 'two', False) ] self.assertRaises(PolicyError, StoragePolicyCollection, test_policies) def test_names_are_normalized(self): test_policies = [ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'ZERO', False) ] self.assertRaises(PolicyError, StoragePolicyCollection, test_policies) policies = StoragePolicyCollection( [StoragePolicy(0, 'zEro', True), StoragePolicy(1, 'One', False)]) pol0 = policies[0] pol1 = policies[1] for name in ('zero', 'ZERO', 'zErO', 'ZeRo'): self.assertEqual(pol0, policies.get_by_name(name)) self.assertEqual(policies.get_by_name(name).name, 'zEro') for name in ('one', 'ONE', 'oNe', 'OnE'): self.assertEqual(pol1, policies.get_by_name(name)) self.assertEqual(policies.get_by_name(name).name, 'One') def assertRaisesWithMessage(self, exc_class, message, f, *args, **kwargs): try: f(*args, **kwargs) except exc_class as err: err_msg = str(err) self.assert_( message in err_msg, 'Error message %r did not ' 'have expected substring %r' % (err_msg, message)) else: self.fail('%r did not raise %s' % (message, exc_class.__name__)) def test_deprecated_default(self): bad_conf = self._conf(""" [storage-policy:1] name = one deprecated = yes default = yes """) self.assertRaisesWithMessage(PolicyError, "Deprecated policy can not be default", parse_storage_policies, bad_conf) def test_multiple_policies_with_no_policy_index_zero(self): bad_conf = self._conf(""" [storage-policy:1] name = one default = yes """) # Policy-0 will not be implicitly added if other policies are defined self.assertRaisesWithMessage( PolicyError, "must specify a storage policy section " "for policy index 0", parse_storage_policies, bad_conf) def test_no_default(self): orig_conf = self._conf(""" [storage-policy:0] name = zero [storage-policy:1] name = one default = yes """) policies = parse_storage_policies(orig_conf) self.assertEqual(policies.default, policies[1]) self.assert_(policies[0].name, 'Policy-0') bad_conf = self._conf(""" [storage-policy:0] name = zero [storage-policy:1] name = one deprecated = yes """) # multiple polices and no explicit default self.assertRaisesWithMessage(PolicyError, "Unable to find default", parse_storage_policies, bad_conf) good_conf = self._conf(""" [storage-policy:0] name = Policy-0 default = yes [storage-policy:1] name = one deprecated = yes """) policies = parse_storage_policies(good_conf) self.assertEqual(policies.default, policies[0]) self.assert_(policies[1].is_deprecated, True) def test_parse_storage_policies(self): # ValueError when deprecating policy 0 bad_conf = self._conf(""" [storage-policy:0] name = zero deprecated = yes [storage-policy:1] name = one deprecated = yes """) self.assertRaisesWithMessage( PolicyError, "Unable to find policy that's not deprecated", parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:] name = zero """) self.assertRaisesWithMessage(PolicyError, 'Invalid index', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:-1] name = zero """) self.assertRaisesWithMessage(PolicyError, 'Invalid index', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:x] name = zero """) self.assertRaisesWithMessage(PolicyError, 'Invalid index', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:x-1] name = zero """) self.assertRaisesWithMessage(PolicyError, 'Invalid index', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:x] name = zero """) self.assertRaisesWithMessage(PolicyError, 'Invalid index', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:x:1] name = zero """) self.assertRaisesWithMessage(PolicyError, 'Invalid index', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:1] name = zero boo = berries """) self.assertRaisesWithMessage(PolicyError, 'Invalid option', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:0] name = """) self.assertRaisesWithMessage(PolicyError, 'Invalid name', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:3] name = Policy-0 """) self.assertRaisesWithMessage(PolicyError, 'Invalid name', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:1] name = policY-0 """) self.assertRaisesWithMessage(PolicyError, 'Invalid name', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:0] name = one [storage-policy:1] name = ONE """) self.assertRaisesWithMessage(PolicyError, 'Duplicate name', parse_storage_policies, bad_conf) bad_conf = self._conf(""" [storage-policy:0] name = good_stuff """) self.assertRaisesWithMessage(PolicyError, 'Invalid name', parse_storage_policies, bad_conf) # Additional section added to ensure parser ignores other sections conf = self._conf(""" [some-other-section] foo = bar [storage-policy:0] name = zero [storage-policy:5] name = one default = yes [storage-policy:6] name = duplicate-sections-are-ignored [storage-policy:6] name = apple """) policies = parse_storage_policies(conf) self.assertEquals(True, policies.get_by_index(5).is_default) self.assertEquals(False, policies.get_by_index(0).is_default) self.assertEquals(False, policies.get_by_index(6).is_default) self.assertEquals("object", policies.get_by_name("zero").ring_name) self.assertEquals("object-5", policies.get_by_name("one").ring_name) self.assertEquals("object-6", policies.get_by_name("apple").ring_name) self.assertEqual(0, int(policies.get_by_name('zero'))) self.assertEqual(5, int(policies.get_by_name('one'))) self.assertEqual(6, int(policies.get_by_name('apple'))) self.assertEquals("zero", policies.get_by_index(0).name) self.assertEquals("zero", policies.get_by_index("0").name) self.assertEquals("one", policies.get_by_index(5).name) self.assertEquals("apple", policies.get_by_index(6).name) self.assertEquals("zero", policies.get_by_index(None).name) self.assertEquals("zero", policies.get_by_index('').name) def test_reload_invalid_storage_policies(self): conf = self._conf(""" [storage-policy:0] name = zero [storage-policy:00] name = double-zero """) with NamedTemporaryFile() as f: conf.write(f) f.flush() with mock.patch('swift.common.storage_policy.SWIFT_CONF_FILE', new=f.name): try: reload_storage_policies() except SystemExit as e: err_msg = str(e) else: self.fail('SystemExit not raised') parts = [ 'Invalid Storage Policy Configuration', 'Duplicate index', ] for expected in parts: self.assert_(expected in err_msg, '%s was not in %s' % (expected, err_msg)) def test_storage_policy_ordering(self): test_policies = StoragePolicyCollection([ StoragePolicy(0, 'zero', is_default=True), StoragePolicy(503, 'error'), StoragePolicy(204, 'empty'), StoragePolicy(404, 'missing'), ]) self.assertEqual([0, 204, 404, 503], [int(p) for p in sorted(list(test_policies))]) p503 = test_policies[503] self.assertTrue(501 < p503 < 507) def test_get_object_ring(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False) ] policies = StoragePolicyCollection(test_policies) class NamedFakeRing(FakeRing): def __init__(self, swift_dir, ring_name=None): self.ring_name = ring_name super(NamedFakeRing, self).__init__() with mock.patch('swift.common.storage_policy.Ring', new=NamedFakeRing): for policy in policies: self.assertFalse(policy.object_ring) ring = policies.get_object_ring(int(policy), '/path/not/used') self.assertEqual(ring.ring_name, policy.ring_name) self.assertTrue(policy.object_ring) self.assert_(isinstance(policy.object_ring, NamedFakeRing)) def blow_up(*args, **kwargs): raise Exception('kaboom!') with mock.patch('swift.common.storage_policy.Ring', new=blow_up): for policy in policies: policy.load_ring('/path/not/used') expected = policies.get_object_ring(int(policy), '/path/not/used') self.assertEqual(policy.object_ring, expected) # bad policy index self.assertRaises(PolicyError, policies.get_object_ring, 99, '/path/not/used') def test_singleton_passthrough(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False) ] with patch_policies(test_policies): for policy in POLICIES: self.assertEqual(POLICIES[int(policy)], policy) def test_storage_policy_get_options(self): policy = StoragePolicy(1, 'gold', True, False) self.assertEqual({ 'name': 'gold', 'default': True, 'deprecated': False }, policy.get_options()) policy = StoragePolicy(1, 'gold', False, True) self.assertEqual({ 'name': 'gold', 'default': False, 'deprecated': True }, policy.get_options())
import os import time import string from shutil import rmtree from hashlib import md5 from tempfile import mkdtemp from test.unit import FakeLogger, patch_policies from swift.obj import auditor from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash, \ get_data_dir, DiskFileManager, AuditLocation from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ storage_directory from swift.common.storage_policy import StoragePolicy _mocked_policies = [StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', True)] @patch_policies(_mocked_policies) class TestAuditor(unittest.TestCase): def setUp(self): self.testdir = os.path.join(mkdtemp(), 'tmp_test_object_auditor') self.devices = os.path.join(self.testdir, 'node') self.rcache = os.path.join(self.testdir, 'object.recon') self.logger = FakeLogger() rmtree(self.testdir, ignore_errors=1) mkdirs(os.path.join(self.devices, 'sda')) os.mkdir(os.path.join(self.devices, 'sdb'))
import string from shutil import rmtree from hashlib import md5 from tempfile import mkdtemp from test.unit import FakeLogger, patch_policies, make_timestamp_iter, \ DEFAULT_TEST_EC_TYPE from swift.obj import auditor from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash, \ get_data_dir, DiskFileManager, ECDiskFileManager, AuditLocation, \ clear_auditor_status, get_auditor_status from swift.common.utils import mkdirs, normalize_timestamp, Timestamp from swift.common.storage_policy import ECStoragePolicy, StoragePolicy, \ POLICIES _mocked_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', True), ECStoragePolicy(2, 'two', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096), ] def works_only_once(callable_thing, exception): called = [False] def only_once(*a, **kw): if called[0]:
func = account_func resp = func(ipaddr, port, device, partition, method, path, headers=headers, query_string=query_string) return resp return http_connect @patch_policies( [StoragePolicy(0, 'zero', True, object_ring=FakeRing(replicas=1))]) class TestObjectSysmeta(unittest.TestCase): '''Tests object sysmeta is correctly handled by combination of proxy server and object server. ''' def _assertStatus(self, resp, expected): self.assertEqual(resp.status_int, expected, 'Expected %d, got %s' % (expected, resp.status)) def _assertInHeaders(self, resp, expected): for key, val in expected.items(): self.assertTrue(key in resp.headers, 'Header %s missing from %s' % (key, resp.headers)) self.assertEqual( val, resp.headers[key], 'Expected header %s:%s, got %s:%s' % (key, val, key, resp.headers[key]))