def _setup_object(self, condition=None): attempts = [] for _ in range(50): account = 'a' container = 'c' obj = 'o-' + str(uuid.uuid4()) self._hash = utils.hash_path(account, container, obj) digest = binascii.unhexlify(self._hash) self.part = struct.unpack_from('>I', digest)[0] >> 24 self.next_part = struct.unpack_from('>I', digest)[0] >> 23 path = os.path.join(os.path.sep, account, container, obj) # There's 1/512 chance that both old and new parts will be 0; # that's not a terribly interesting case, as there's nothing to do attempts.append((self.part, self.next_part, 2**PART_POWER)) if (self.part != self.next_part and (condition(self.part) if condition else True)): break else: self.fail( 'Failed to setup object satisfying test preconditions %s' % attempts) shutil.rmtree(self.objects, ignore_errors=True) os.mkdir(self.objects) self.objdir = os.path.join(self.objects, str(self.part), self._hash[-3:], self._hash) os.makedirs(self.objdir) self.object_fname = utils.Timestamp.now().internal + ".data" self.objname = os.path.join(self.objdir, self.object_fname) with open(self.objname, "wb") as dummy: dummy.write(b"Hello World!") write_metadata(dummy, {'name': path, 'Content-Length': '12'}) self.policy = StoragePolicy(0, 'platinum', True) storage_policy._POLICIES = StoragePolicyCollection([self.policy]) self.part_dir = os.path.join(self.objects, str(self.part)) self.suffix_dir = os.path.join(self.part_dir, self._hash[-3:]) self.next_part_dir = os.path.join(self.objects, str(self.next_part)) self.next_suffix_dir = os.path.join(self.next_part_dir, self._hash[-3:]) self.expected_dir = os.path.join(self.next_suffix_dir, self._hash) self.expected_file = os.path.join(self.expected_dir, self.object_fname)
def test_headers_to_account_info_storage_policies(self): headers = { 'x-account-storage-policy-zero-object-count': '13', 'x-account-storage-policy-zero-container-count': '120', 'x-account-storage-policy-zero-bytes-used': '1002', 'x-account-storage-policy-one-object-count': '10', 'x-account-storage-policy-one-container-count': '20', } spc = StoragePolicyCollection( [StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False)]) with PatchPolicies(spc): resp = headers_to_account_info(headers.items(), 200) self.assertEqual(resp['storage_policies'][0]['object_count'], 13) self.assertEqual(resp['storage_policies'][0]['container_count'], 120) self.assertEqual(resp['storage_policies'][0]['bytes'], 1002) self.assertEqual(resp['storage_policies'][1]['object_count'], 10) self.assertEqual(resp['storage_policies'][1]['container_count'], 20) self.assertEqual(resp['storage_policies'][1]['bytes'], 0)
def test_names_are_normalized(self): test_policies = [ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'ZERO', False) ] self.assertRaises(PolicyError, StoragePolicyCollection, test_policies) policies = StoragePolicyCollection( [StoragePolicy(0, 'zEro', True), StoragePolicy(1, 'One', False)]) pol0 = policies[0] pol1 = policies[1] for name in ('zero', 'ZERO', 'zErO', 'ZeRo'): self.assertEqual(pol0, policies.get_by_name(name)) self.assertEqual(policies.get_by_name(name).name, 'zEro') for name in ('one', 'ONE', 'oNe', 'OnE'): self.assertEqual(pol1, policies.get_by_name(name)) self.assertEqual(policies.get_by_name(name).name, 'One')
def test_policies_type_attribute(self): test_policies = [ StoragePolicy(0, 'zero', is_default=True), StoragePolicy(1, 'one'), StoragePolicy(2, 'two'), StoragePolicy(3, 'three', is_deprecated=True), ECStoragePolicy(10, 'ten', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=3), ] policies = StoragePolicyCollection(test_policies) self.assertEqual(policies.get_by_index(0).policy_type, REPL_POLICY) self.assertEqual(policies.get_by_index(1).policy_type, REPL_POLICY) self.assertEqual(policies.get_by_index(2).policy_type, REPL_POLICY) self.assertEqual(policies.get_by_index(3).policy_type, REPL_POLICY) self.assertEqual(policies.get_by_index(10).policy_type, EC_POLICY)
def setUp(self): skip_if_no_xattrs() self.logger = FakeLogger() self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') shutil.rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) os.mkdir(self.devices) self.rb = ring.RingBuilder(8, 6.0, 1) for i in range(6): ip = "127.0.0.%s" % i self.rb.add_dev({'id': i, 'region': 0, 'zone': 0, 'weight': 1, 'ip': ip, 'port': 10000, 'device': 'sda1'}) self.rb.rebalance(seed=1) self.existing_device = 'sda1' os.mkdir(os.path.join(self.devices, self.existing_device)) self.objects = os.path.join(self.devices, self.existing_device, 'objects') os.mkdir(self.objects) self._hash = utils.hash_path('a/c/o') digest = binascii.unhexlify(self._hash) part = struct.unpack_from('>I', digest)[0] >> 24 self.next_part = struct.unpack_from('>I', digest)[0] >> 23 self.objdir = os.path.join( self.objects, str(part), self._hash[-3:], self._hash) os.makedirs(self.objdir) self.object_fname = "1278553064.00000.data" self.objname = os.path.join(self.objdir, self.object_fname) with open(self.objname, "wb") as dummy: dummy.write(b"Hello World!") write_metadata(dummy, {'name': '/a/c/o', 'Content-Length': '12'}) test_policies = [StoragePolicy(0, 'platin', True)] storage_policy._POLICIES = StoragePolicyCollection(test_policies) self.expected_dir = os.path.join( self.objects, str(self.next_part), self._hash[-3:], self._hash) self.expected_file = os.path.join(self.expected_dir, self.object_fname)
def test_deprecate_policies(self): # deprecation specified test_policies = [ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False, is_deprecated=True) ] policies = StoragePolicyCollection(test_policies) self.assertEquals(policies.default, test_policies[0]) self.assertEquals(policies.default.name, 'zero') self.assertEquals(len(policies), 3) # multiple policies requires default test_policies = [ StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', False, is_deprecated=True), StoragePolicy(2, 'two', False) ] self.assertRaisesWithMessage(PolicyError, 'Unable to find default policy', StoragePolicyCollection, test_policies)
def test_validate_ring(self): test_policies = [ ECStoragePolicy(0, 'ec8-2', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=8, ec_nparity=2, object_ring=FakeRing(replicas=8), is_default=True), ECStoragePolicy(1, 'ec10-4', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4, object_ring=FakeRing(replicas=10)), ECStoragePolicy(2, 'ec4-2', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=4, ec_nparity=2, object_ring=FakeRing(replicas=7)), ] policies = StoragePolicyCollection(test_policies) for policy in policies: msg = 'EC ring for policy %s needs to be configured with ' \ 'exactly %d nodes.' % \ (policy.name, policy.ec_ndata + policy.ec_nparity) self.assertRaisesWithMessage(RingValidationError, msg, policy._validate_ring)
def test_storage_policy_repr(self): test_policies = [ StoragePolicy(0, 'aay', True), StoragePolicy(1, 'bee', False), StoragePolicy(2, 'cee', False) ] policies = StoragePolicyCollection(test_policies) for policy in policies: policy_repr = repr(policy) self.assert_(policy.__class__.__name__ in policy_repr) self.assert_('is_default=%s' % policy.is_default in policy_repr) self.assert_('is_deprecated=%s' % policy.is_deprecated in policy_repr) self.assert_(policy.name in policy_repr) collection_repr = repr(policies) collection_repr_lines = collection_repr.splitlines() self.assert_(policies.__class__.__name__ in collection_repr_lines[0]) self.assertEqual(len(policies), len(collection_repr_lines[1:-1])) for policy, line in zip(policies, collection_repr_lines[1:-1]): self.assert_(repr(policy) in line) with patch_policies(policies): self.assertEqual(repr(POLICIES), collection_repr)
def test_storage_policy_get_info(self): test_policies = [ StoragePolicy(0, 'zero', is_default=True), StoragePolicy(1, 'one', is_deprecated=True), ECStoragePolicy(10, 'ten', ec_type='jerasure_rs_vand', ec_ndata=10, ec_nparity=3), ECStoragePolicy(11, 'done', is_deprecated=True, ec_type='jerasure_rs_vand', ec_ndata=10, ec_nparity=3), ] policies = StoragePolicyCollection(test_policies) expected = { # default replication (0, True): { 'name': 'zero', 'default': True, 'deprecated': False, 'policy_type': REPL_POLICY }, (0, False): { 'name': 'zero', 'default': True, }, # deprecated replication (1, True): { 'name': 'one', 'default': False, 'deprecated': True, 'policy_type': REPL_POLICY }, (1, False): { 'name': 'one', 'deprecated': True, }, # enabled ec (10, True): { 'name': 'ten', 'default': False, 'deprecated': False, 'policy_type': EC_POLICY, 'ec_type': 'jerasure_rs_vand', 'ec_num_data_fragments': 10, 'ec_num_parity_fragments': 3, 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE, }, (10, False): { 'name': 'ten', }, # deprecated ec (11, True): { 'name': 'done', 'default': False, 'deprecated': True, 'policy_type': EC_POLICY, 'ec_type': 'jerasure_rs_vand', 'ec_num_data_fragments': 10, 'ec_num_parity_fragments': 3, 'ec_object_segment_size': DEFAULT_EC_OBJECT_SEGMENT_SIZE, }, (11, False): { 'name': 'done', 'deprecated': True, }, } self.maxDiff = None for policy in policies: expected_info = expected[(int(policy), True)] self.assertEqual(policy.get_info(config=True), expected_info) expected_info = expected[(int(policy), False)] self.assertEqual(policy.get_info(config=False), expected_info)
def test_add_remove_names(self): test_policies = [StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False)] policies = StoragePolicyCollection(test_policies) # add names policies.add_policy_alias(1, 'tahi') self.assertEqual(policies.get_by_name('tahi'), test_policies[1]) policies.add_policy_alias(2, 'rua', 'dos') self.assertEqual(policies.get_by_name('rua'), test_policies[2]) self.assertEqual(policies.get_by_name('dos'), test_policies[2]) self.assertRaisesWithMessage(PolicyError, 'Invalid name', policies.add_policy_alias, 2, 'double\n') self.assertRaisesWithMessage(PolicyError, 'Invalid name', policies.add_policy_alias, 2, '') # try to add existing name self.assertRaisesWithMessage(PolicyError, 'Duplicate name', policies.add_policy_alias, 2, 'two') self.assertRaisesWithMessage(PolicyError, 'Duplicate name', policies.add_policy_alias, 1, 'two') # remove name policies.remove_policy_alias('tahi') self.assertEqual(policies.get_by_name('tahi'), None) # remove only name self.assertRaisesWithMessage(PolicyError, 'Policies must have at least one name.', policies.remove_policy_alias, 'zero') # remove non-existent name self.assertRaisesWithMessage(PolicyError, 'No policy with name', policies.remove_policy_alias, 'three') # remove default name policies.remove_policy_alias('two') self.assertEqual(policies.get_by_name('two'), None) self.assertEqual(policies.get_by_index(2).name, 'rua') # change default name to a new name policies.change_policy_primary_name(2, 'two') self.assertEqual(policies.get_by_name('two'), test_policies[2]) self.assertEqual(policies.get_by_index(2).name, 'two') # change default name to an existing alias policies.change_policy_primary_name(2, 'dos') self.assertEqual(policies.get_by_index(2).name, 'dos') # change default name to a bad new name self.assertRaisesWithMessage(PolicyError, 'Invalid name', policies.change_policy_primary_name, 2, 'bad\nname') # change default name to a name belonging to another policy self.assertRaisesWithMessage(PolicyError, 'Other policy', policies.change_policy_primary_name, 1, 'dos')
def test_multiple_names_EC(self): # checking duplicate names on insert test_policies_ec = [ ECStoragePolicy( 0, 'ec8-2', aliases='zeus, jupiter', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=8, ec_nparity=2, object_ring=FakeRing(replicas=8), is_default=True), ECStoragePolicy( 1, 'ec10-4', aliases='ec8-2', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4, object_ring=FakeRing(replicas=10))] self.assertRaises(PolicyError, StoragePolicyCollection, test_policies_ec) # checking correct retrival using other names good_test_policies_EC = [ ECStoragePolicy(0, 'ec8-2', aliases='zeus, jupiter', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=8, ec_nparity=2, object_ring=FakeRing(replicas=8), is_default=True), ECStoragePolicy(1, 'ec10-4', aliases='athena, minerva', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=10, ec_nparity=4, object_ring=FakeRing(replicas=10)), ECStoragePolicy(2, 'ec4-2', aliases='poseidon, neptune', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=4, ec_nparity=2, object_ring=FakeRing(replicas=7)), ] ec_policies = StoragePolicyCollection(good_test_policies_EC) for name in ('ec8-2', 'zeus', 'jupiter'): self.assertEqual(ec_policies.get_by_name(name), ec_policies[0]) for name in ('ec10-4', 'athena', 'minerva'): self.assertEqual(ec_policies.get_by_name(name), ec_policies[1]) # Testing parsing of conf files/text good_ec_conf = self._conf(""" [storage-policy:0] name = ec8-2 aliases = zeus, jupiter policy_type = erasure_coding ec_type = %(ec_type)s default = yes ec_num_data_fragments = 8 ec_num_parity_fragments = 2 [storage-policy:1] name = ec10-4 aliases = poseidon, neptune policy_type = erasure_coding ec_type = %(ec_type)s ec_num_data_fragments = 10 ec_num_parity_fragments = 4 """ % {'ec_type': DEFAULT_TEST_EC_TYPE}) ec_policies = parse_storage_policies(good_ec_conf) self.assertEqual(ec_policies.get_by_name('ec8-2'), ec_policies[0]) self.assertEqual(ec_policies.get_by_name('ec10-4'), ec_policies.get_by_name('poseidon')) name_repeat_ec_conf = self._conf(""" [storage-policy:0] name = ec8-2 aliases = ec8-2 policy_type = erasure_coding ec_type = %(ec_type)s default = yes ec_num_data_fragments = 8 ec_num_parity_fragments = 2 """ % {'ec_type': DEFAULT_TEST_EC_TYPE}) # Test on line below should not generate errors. Repeat of main # name under aliases is permitted during construction # but only because automated testing requires it. ec_policies = parse_storage_policies(name_repeat_ec_conf) bad_ec_conf = self._conf(""" [storage-policy:0] name = ec8-2 aliases = zeus, zeus policy_type = erasure_coding ec_type = %(ec_type)s default = yes ec_num_data_fragments = 8 ec_num_parity_fragments = 2 """ % {'ec_type': DEFAULT_TEST_EC_TYPE}) self.assertRaisesWithMessage(PolicyError, 'is already assigned to this policy', parse_storage_policies, bad_ec_conf)
def test_multiple_names(self): # checking duplicate on insert test_policies = [StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False, aliases='zero')] self.assertRaises(PolicyError, StoragePolicyCollection, test_policies) # checking correct retrival using other names test_policies = [StoragePolicy(0, 'zero', True, aliases='cero, kore'), StoragePolicy(1, 'one', False, aliases='uno, tahi'), StoragePolicy(2, 'two', False, aliases='dos, rua')] policies = StoragePolicyCollection(test_policies) for name in ('zero', 'cero', 'kore'): self.assertEqual(policies.get_by_name(name), test_policies[0]) for name in ('two', 'dos', 'rua'): self.assertEqual(policies.get_by_name(name), test_policies[2]) # Testing parsing of conf files/text good_conf = self._conf(""" [storage-policy:0] name = one aliases = uno, tahi default = yes """) policies = parse_storage_policies(good_conf) self.assertEqual(policies.get_by_name('one'), policies[0]) self.assertEqual(policies.get_by_name('one'), policies.get_by_name('tahi')) name_repeat_conf = self._conf(""" [storage-policy:0] name = one aliases = one default = yes """) # Test on line below should not generate errors. Repeat of main # name under aliases is permitted during construction # but only because automated testing requires it. policies = parse_storage_policies(name_repeat_conf) extra_commas_conf = self._conf(""" [storage-policy:0] name = one aliases = ,,one, , default = yes """) # Extra blank entries should be silently dropped policies = parse_storage_policies(extra_commas_conf) bad_conf = self._conf(""" [storage-policy:0] name = one aliases = uno, uno default = yes """) self.assertRaisesWithMessage(PolicyError, 'is already assigned to this policy', parse_storage_policies, bad_conf)