def test_get_ring_names(self): self.recon_instance.server_type = 'not-object' self.assertEqual(self.recon_instance._get_ring_names(), ['not-object']) self.recon_instance.server_type = 'object' with patch_policies([StoragePolicy(0, 'zero', is_default=True)]): self.assertEqual(self.recon_instance._get_ring_names(), ['object']) with patch_policies([ StoragePolicy(0, 'zero', is_default=True), StoragePolicy(1, 'one') ]): self.assertEqual(self.recon_instance._get_ring_names(), ['object', 'object-1']) self.assertEqual(self.recon_instance._get_ring_names('0'), ['object']) self.assertEqual(self.recon_instance._get_ring_names('zero'), ['object']) self.assertEqual(self.recon_instance._get_ring_names('1'), ['object-1']) self.assertEqual(self.recon_instance._get_ring_names('one'), ['object-1']) self.assertEqual(self.recon_instance._get_ring_names('3'), []) self.assertEqual(self.recon_instance._get_ring_names('wrong'), [])
def test_get_ring_names(self): self.recon_instance.server_type = 'not-object' self.assertEqual(self.recon_instance._get_ring_names(), ['not-object']) self.recon_instance.server_type = 'object' with patch_policies([StoragePolicy(0, 'zero', is_default=True)]): self.assertEqual(self.recon_instance._get_ring_names(), ['object']) with patch_policies([StoragePolicy(0, 'zero', is_default=True), StoragePolicy(1, 'one')]): self.assertEqual(self.recon_instance._get_ring_names(), ['object', 'object-1']) self.assertEqual(self.recon_instance._get_ring_names('0'), ['object']) self.assertEqual(self.recon_instance._get_ring_names('zero'), ['object']) self.assertEqual(self.recon_instance._get_ring_names('1'), ['object-1']) self.assertEqual(self.recon_instance._get_ring_names('one'), ['object-1']) self.assertEqual(self.recon_instance._get_ring_names('3'), []) self.assertEqual(self.recon_instance._get_ring_names('wrong'), [])
def test_storage_policy_repr(self): test_policies = [StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False), ECStoragePolicy(10, 'ten', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=3)] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assertTrue(policy.__class__.__name__ in policy_repr) self.assertTrue('is_default=%s' % policy.is_default in policy_repr) self.assertTrue('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assertTrue(policy.name in policy_repr) if policy.policy_type == EC_POLICY: self.assertTrue('ec_type=%s' % policy.ec_type in policy_repr) self.assertTrue('ec_ndata=%s' % policy.ec_ndata in policy_repr) self.assertTrue('ec_nparity=%s' % policy.ec_nparity in policy_repr) self.assertTrue('ec_segment_size=%s' % policy.ec_segment_size in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assertTrue( policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assertTrue(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr)
def test_singleton_passthrough(self): test_policies = [StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False)] with patch_policies(test_policies): for policy in POLICIES: self.assertEqual(POLICIES[int(policy)], policy)
def test_storage_policy_repr(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False), ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', ec_ndata=10, ec_nparity=3) ] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assert_(policy.__class__.__name__ in policy_repr) self.assert_('is_default=%s' % policy.is_default in policy_repr) self.assert_('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assert_(policy.name in policy_repr) if policy.policy_type == EC_POLICY: self.assert_('ec_type=%s' % policy.ec_type in policy_repr) self.assert_('ec_ndata=%s' % policy.ec_ndata in policy_repr) self.assert_('ec_nparity=%s' % policy.ec_nparity in policy_repr) self.assert_('ec_segment_size=%s' % policy.ec_segment_size in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assert_(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr)
def test_storage_policy_repr(self): test_policies = [ StoragePolicy(0, "aay", True), StoragePolicy(1, "bee", False), StoragePolicy(2, "cee", False), ECStoragePolicy(10, "ten", ec_type="jerasure_rs_vand", ec_ndata=10, ec_nparity=3), ] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assertTrue(policy.__class__.__name__ in policy_repr) self.assertTrue("is_default=%s" % policy.is_default in policy_repr) self.assertTrue("is_deprecated=%s" % policy.is_deprecated in policy_repr) self.assertTrue(policy.name in policy_repr) if policy.policy_type == EC_POLICY: self.assertTrue("ec_type=%s" % policy.ec_type in policy_repr) self.assertTrue("ec_ndata=%s" % policy.ec_ndata in policy_repr) self.assertTrue("ec_nparity=%s" % policy.ec_nparity in policy_repr) self.assertTrue("ec_segment_size=%s" % policy.ec_segment_size in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assertTrue(policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assertTrue(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr)
def test_load_from_config(self, tempdir): conf_path = os.path.join(tempdir, 'interal_client.conf') conf_body = """ [DEFAULT] swift_dir = %s [pipeline:main] pipeline = catch_errors cache proxy-server [app:proxy-server] use = egg:swift#proxy auto_create_account_prefix = - [filter:cache] use = egg:swift#memcache [filter:catch_errors] use = egg:swift#catch_errors """ % tempdir with open(conf_path, 'w') as f: f.write(dedent(conf_body)) account_ring_path = os.path.join(tempdir, 'account.ring.gz') write_fake_ring(account_ring_path) container_ring_path = os.path.join(tempdir, 'container.ring.gz') write_fake_ring(container_ring_path) object_ring_path = os.path.join(tempdir, 'object.ring.gz') write_fake_ring(object_ring_path) with patch_policies([StoragePolicy(0, 'legacy', True)]): client = internal_client.InternalClient(conf_path, 'test', 1) self.assertEqual(client.account_ring, client.app.app.app.account_ring) self.assertEqual(client.account_ring.serialized_path, account_ring_path) self.assertEqual(client.container_ring, client.app.app.app.container_ring) self.assertEqual(client.container_ring.serialized_path, container_ring_path) object_ring = client.app.app.app.get_object_ring(0) self.assertEqual(client.get_object_ring(0), object_ring) self.assertEqual(object_ring.serialized_path, object_ring_path) self.assertEquals(client.auto_create_account_prefix, '-')
def test_storage_policy_repr(self): test_policies = [StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False)] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assert_(policy.__class__.__name__ in policy_repr) self.assert_('is_default=%s' % policy.is_default in policy_repr) self.assert_('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assert_(policy.name in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assert_(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr)
def test_storage_policy_repr(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False) ] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assert_(policy.__class__.__name__ in policy_repr) self.assert_('is_default=%s' % policy.is_default in policy_repr) self.assert_('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assert_(policy.name in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assert_(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr)
def test_bind_ports_cache(self): test_policies = [StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False)] my_ips = ['1.2.3.4', '2.3.4.5'] other_ips = ['3.4.5.6', '4.5.6.7'] bind_ip = my_ips[1] devs_by_ring_name1 = { 'object': [ # 'aay' {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0], 'port': 6006}, {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0], 'port': 6007}, {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6008}, None, {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6009}], 'object-1': [ # 'bee' {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6006}, # dupe {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0], 'port': 6010}, {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6011}, {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6012}], 'object-2': [ # 'cee' {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0], 'port': 6010}, # on our IP and a not-us IP {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0], 'port': 6013}, None, {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6014}, {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6015}], } devs_by_ring_name2 = { 'object': [ # 'aay' {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0], 'port': 6016}, {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6019}], 'object-1': [ # 'bee' {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6016}, # dupe {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6022}], 'object-2': [ # 'cee' {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0], 'port': 6020}, {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6025}], } ring_files = [ring_name + '.ring.gz' for ring_name in sorted(devs_by_ring_name1)] def _fake_load(gz_path, stub_objs, metadata_only=False): return RingData( devs=stub_objs[os.path.basename(gz_path)[:-8]], replica2part2dev_id=[], part_shift=24) with mock.patch( 'swift.common.storage_policy.RingData.load' ) as mock_ld, \ patch_policies(test_policies), \ mock.patch('swift.common.storage_policy.whataremyips') \ as mock_whataremyips, \ temptree(ring_files) as tempdir: mock_whataremyips.return_value = my_ips cache = BindPortsCache(tempdir, bind_ip) self.assertEqual([ mock.call(bind_ip), ], mock_whataremyips.mock_calls) mock_whataremyips.reset_mock() mock_ld.side_effect = partial(_fake_load, stub_objs=devs_by_ring_name1) self.assertEqual(set([ 6006, 6008, 6011, 6010, 6014, ]), cache.all_bind_ports_for_node()) self.assertEqual([ mock.call(os.path.join(tempdir, ring_files[0]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[1]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[2]), metadata_only=True), ], mock_ld.mock_calls) mock_ld.reset_mock() mock_ld.side_effect = partial(_fake_load, stub_objs=devs_by_ring_name2) self.assertEqual(set([ 6006, 6008, 6011, 6010, 6014, ]), cache.all_bind_ports_for_node()) self.assertEqual([], mock_ld.mock_calls) # but when all the file mtimes are made different, it'll # reload for gz_file in [os.path.join(tempdir, n) for n in ring_files]: os.utime(gz_file, (88, 88)) self.assertEqual(set([ 6016, 6020, ]), cache.all_bind_ports_for_node()) self.assertEqual([ mock.call(os.path.join(tempdir, ring_files[0]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[1]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[2]), metadata_only=True), ], mock_ld.mock_calls) mock_ld.reset_mock() # Don't do something stupid like crash if a ring file is missing. os.unlink(os.path.join(tempdir, 'object-2.ring.gz')) self.assertEqual(set([ 6016, 6020, ]), cache.all_bind_ports_for_node()) self.assertEqual([], mock_ld.mock_calls) # whataremyips() is only called in the constructor self.assertEqual([], mock_whataremyips.mock_calls)
def test_bind_ports_cache(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False) ] my_ips = ['1.2.3.4', '2.3.4.5'] other_ips = ['3.4.5.6', '4.5.6.7'] bind_ip = my_ips[1] devs_by_ring_name1 = { 'object': [ # 'aay' { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0], 'port': 6006 }, { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0], 'port': 6007 }, { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6008 }, None, { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6009 } ], 'object-1': [ # 'bee' { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6006 }, # dupe { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0], 'port': 6010 }, { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6011 }, { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6012 } ], 'object-2': [ # 'cee' { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0], 'port': 6010 }, # on our IP and a not-us IP { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0], 'port': 6013 }, None, { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6014 }, { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6015 } ], } devs_by_ring_name2 = { 'object': [ # 'aay' { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0], 'port': 6016 }, { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6019 } ], 'object-1': [ # 'bee' { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1], 'port': 6016 }, # dupe { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6022 } ], 'object-2': [ # 'cee' { 'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0], 'port': 6020 }, { 'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1], 'port': 6025 } ], } ring_files = [ ring_name + '.ring.gz' for ring_name in sorted(devs_by_ring_name1) ] def _fake_load(gz_path, stub_objs, metadata_only=False): return RingData(devs=stub_objs[os.path.basename(gz_path)[:-8]], replica2part2dev_id=[], part_shift=24) with mock.patch( 'swift.common.storage_policy.RingData.load' ) as mock_ld, \ patch_policies(test_policies), \ mock.patch('swift.common.storage_policy.whataremyips') \ as mock_whataremyips, \ temptree(ring_files) as tempdir: mock_whataremyips.return_value = my_ips cache = BindPortsCache(tempdir, bind_ip) self.assertEqual([ mock.call(bind_ip), ], mock_whataremyips.mock_calls) mock_whataremyips.reset_mock() mock_ld.side_effect = partial(_fake_load, stub_objs=devs_by_ring_name1) self.assertEqual(set([ 6006, 6008, 6011, 6010, 6014, ]), cache.all_bind_ports_for_node()) self.assertEqual([ mock.call(os.path.join(tempdir, ring_files[0]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[1]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[2]), metadata_only=True), ], mock_ld.mock_calls) mock_ld.reset_mock() mock_ld.side_effect = partial(_fake_load, stub_objs=devs_by_ring_name2) self.assertEqual(set([ 6006, 6008, 6011, 6010, 6014, ]), cache.all_bind_ports_for_node()) self.assertEqual([], mock_ld.mock_calls) # but when all the file mtimes are made different, it'll # reload for gz_file in [os.path.join(tempdir, n) for n in ring_files]: os.utime(gz_file, (88, 88)) self.assertEqual(set([ 6016, 6020, ]), cache.all_bind_ports_for_node()) self.assertEqual([ mock.call(os.path.join(tempdir, ring_files[0]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[1]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[2]), metadata_only=True), ], mock_ld.mock_calls) mock_ld.reset_mock() # Don't do something stupid like crash if a ring file is missing. os.unlink(os.path.join(tempdir, 'object-2.ring.gz')) self.assertEqual(set([ 6016, 6020, ]), cache.all_bind_ports_for_node()) self.assertEqual([], mock_ld.mock_calls) # whataremyips() is only called in the constructor self.assertEqual([], mock_whataremyips.mock_calls)
def test_bind_ports_cache(self): test_policies = [StoragePolicy(0, "aay", True), StoragePolicy(1, "bee", False), StoragePolicy(2, "cee", False)] my_ips = ["1.2.3.4", "2.3.4.5"] other_ips = ["3.4.5.6", "4.5.6.7"] bind_ip = my_ips[1] devs_by_ring_name1 = { "object": [ # 'aay' {"id": 0, "zone": 0, "region": 1, "ip": my_ips[0], "port": 6006}, {"id": 0, "zone": 0, "region": 1, "ip": other_ips[0], "port": 6007}, {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6008}, None, {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6009}, ], "object-1": [ # 'bee' {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6006}, # dupe {"id": 0, "zone": 0, "region": 1, "ip": other_ips[0], "port": 6010}, {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6011}, {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6012}, ], "object-2": [ # 'cee' {"id": 0, "zone": 0, "region": 1, "ip": my_ips[0], "port": 6010}, # on our IP and a not-us IP {"id": 0, "zone": 0, "region": 1, "ip": other_ips[0], "port": 6013}, None, {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6014}, {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6015}, ], } devs_by_ring_name2 = { "object": [ # 'aay' {"id": 0, "zone": 0, "region": 1, "ip": my_ips[0], "port": 6016}, {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6019}, ], "object-1": [ # 'bee' {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6016}, # dupe {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6022}, ], "object-2": [ # 'cee' {"id": 0, "zone": 0, "region": 1, "ip": my_ips[0], "port": 6020}, {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6025}, ], } ring_files = [ring_name + ".ring.gz" for ring_name in sorted(devs_by_ring_name1)] def _fake_load(gz_path, stub_objs, metadata_only=False): return RingData(devs=stub_objs[os.path.basename(gz_path)[:-8]], replica2part2dev_id=[], part_shift=24) with mock.patch("swift.common.storage_policy.RingData.load") as mock_ld, patch_policies( test_policies ), mock.patch("swift.common.storage_policy.whataremyips") as mock_whataremyips, temptree(ring_files) as tempdir: mock_whataremyips.return_value = my_ips cache = BindPortsCache(tempdir, bind_ip) self.assertEqual([mock.call(bind_ip)], mock_whataremyips.mock_calls) mock_whataremyips.reset_mock() mock_ld.side_effect = partial(_fake_load, stub_objs=devs_by_ring_name1) self.assertEqual(set([6006, 6008, 6011, 6010, 6014]), cache.all_bind_ports_for_node()) self.assertEqual( [ mock.call(os.path.join(tempdir, ring_files[0]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[1]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[2]), metadata_only=True), ], mock_ld.mock_calls, ) mock_ld.reset_mock() mock_ld.side_effect = partial(_fake_load, stub_objs=devs_by_ring_name2) self.assertEqual(set([6006, 6008, 6011, 6010, 6014]), cache.all_bind_ports_for_node()) self.assertEqual([], mock_ld.mock_calls) # but when all the file mtimes are made different, it'll # reload for gz_file in [os.path.join(tempdir, n) for n in ring_files]: os.utime(gz_file, (88, 88)) self.assertEqual(set([6016, 6020]), cache.all_bind_ports_for_node()) self.assertEqual( [ mock.call(os.path.join(tempdir, ring_files[0]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[1]), metadata_only=True), mock.call(os.path.join(tempdir, ring_files[2]), metadata_only=True), ], mock_ld.mock_calls, ) mock_ld.reset_mock() # Don't do something stupid like crash if a ring file is missing. os.unlink(os.path.join(tempdir, "object-2.ring.gz")) self.assertEqual(set([6016, 6020]), cache.all_bind_ports_for_node()) self.assertEqual([], mock_ld.mock_calls) # whataremyips() is only called in the constructor self.assertEqual([], mock_whataremyips.mock_calls)