def test_nounce_reconciliation(self): source_port = self.ports[0] source_device = '127.0.0.1:%s' % source_port source_client = self.client_map[source_port] target_port = self.ports[1] target_device = '127.0.0.1:%s' % target_port target_client = self.client_map[target_port] # two requests with identical timestamp and different nounce req_timestamp = time.time() self.put_object(source_device, 'obj1', timestamp=req_timestamp) self.put_object(target_device, 'obj1', timestamp=req_timestamp) source_meta, _body = self.get_object(source_device, 'obj1') target_meta, _body = self.get_object(target_device, 'obj1') self.assertNotEqual(source_meta, target_meta) source_nounce = source_meta.pop('X-Kinetic-Chunk-Nounce') target_nounce = target_meta.pop('X-Kinetic-Chunk-Nounce') self.assertNotEqual(source_nounce, target_nounce) self.assertEqual(source_meta, target_meta) # peek at keys with source_client as client: source_resp = client.getKeyRange('chunks.', 'objects/') source_keys = source_resp.wait() with target_client as client: target_resp = client.getKeyRange('chunks.', 'objects/') target_keys = target_resp.wait() self.assertEqual(len(source_keys), len(target_keys)) for source_key, target_key in zip(source_keys, target_keys): source_policy_index, source_hash, source_nounce = \ replicator.split_key(source_key) target_policy_index, target_hash, target_nounce = \ replicator.split_key(target_key) self.assertEqual(source_policy_index, target_policy_index) self.assertEqual(source_hash, target_hash) self.assertNotEqual(source_nounce, target_nounce) original_key_count = len(source_keys) # perform replication, should more or less no-op self.daemon._replicate(source_device) with source_client as client: source_resp = client.getKeyRange('chunks.', 'objects/') source_keys = source_resp.wait() with target_client as client: target_resp = client.getKeyRange('chunks.', 'objects/') target_keys = target_resp.wait() self.assertEqual(len(source_keys), len(target_keys)) for source_key, target_key in zip(source_keys, target_keys): source_policy_index, source_hash, source_nounce = \ replicator.split_key(source_key) target_policy_index, target_hash, target_nounce = \ replicator.split_key(target_key) self.assertEqual(source_policy_index, target_policy_index) self.assertEqual(source_hash, target_hash) self.assertNotEqual(source_nounce, target_nounce) post_replication_key_count = len(source_keys) self.assertEquals(original_key_count, post_replication_key_count)
def test_nounce_reconciliation(self): source_port = self.ports[0] source_device = '127.0.0.1:%s' % source_port source_client = self.client_map[source_port] target_port = self.ports[1] target_device = '127.0.0.1:%s' % target_port target_client = self.client_map[target_port] # two requests with identical timestamp and different nounce req_timestamp = time.time() self.put_object(source_device, 'obj1', timestamp=req_timestamp) self.put_object(target_device, 'obj1', timestamp=req_timestamp) source_meta, _body = self.get_object(source_device, 'obj1') target_meta, _body = self.get_object(target_device, 'obj1') self.assertNotEqual(source_meta, target_meta) source_nounce = source_meta.pop('X-Kinetic-Chunk-Nounce') target_nounce = target_meta.pop('X-Kinetic-Chunk-Nounce') self.assertNotEqual(source_nounce, target_nounce) self.assertEqual(source_meta, target_meta) # peek at keys source_resp = source_client.getKeyRange('chunks.', 'objects/') source_keys = source_resp.wait() target_resp = target_client.getKeyRange('chunks.', 'objects/') target_keys = target_resp.wait() self.assertEqual(len(source_keys), len(target_keys)) for source_key, target_key in zip(source_keys, target_keys): source_key_info = replicator.split_key(source_key) target_key_info = replicator.split_key(target_key) for key in source_key_info: if key == 'nounce': continue self.assertEqual(source_key_info[key], target_key_info[key]) self.assertNotEqual(source_key_info['nounce'], target_key_info['nounce']) original_key_count = len(source_keys) # perform replication, should more or less no-op self.daemon._replicate(source_device) source_resp = source_client.getKeyRange('chunks.', 'objects/') source_keys = source_resp.wait() target_resp = target_client.getKeyRange('chunks.', 'objects/') target_keys = target_resp.wait() self.assertEqual(len(source_keys), len(target_keys)) for source_key, target_key in zip(source_keys, target_keys): source_key_info = replicator.split_key(source_key) target_key_info = replicator.split_key(target_key) for key in source_key_info: if key == 'nounce': continue self.assertEqual(source_key_info[key], target_key_info[key]) self.assertNotEqual(source_key_info['nounce'], target_key_info['nounce']) post_replication_key_count = len(source_keys) self.assertEquals(original_key_count, post_replication_key_count)
def find_frags(self): frags = defaultdict(list) for port in self.ports: conn = self.client_map[port] resp = conn.getKeyRange('objects.', 'objects/') for key in resp.wait(): key_info = replicator.split_key(key) frags[port].append(key_info['frag_index']) return frags
def test_split_key(self): hash_ = swift_utils.hash_path('a', 'c', 'o') t = swift_utils.Timestamp(time.time()) nonce = uuid.uuid4() key = 'objects.%s.%s.data.%s' % (hash_, t.internal, nonce) expected = { 'hashpath': hash_, 'frag_index': None, 'nonce': str(nonce), 'policy': storage_policy.POLICIES.legacy, 'timestamp': t.internal, 'ext': 'data', } try: self.assertEqual(replicator.split_key(key), expected) except AssertionError as e: msg = '%s for key %r' % (e, key) self.fail(msg)
def test_multiple_polices(self): hash_ = swift_utils.hash_path('a', 'c', 'o') t = swift_utils.Timestamp(time.time()) nonce = uuid.uuid4() key_template = '%(prefix)s.%(hash)s.%(timestamp)s.%(ext)s' \ '.%(nonce)s%(trailing_frag)s' with utils.patch_policies(with_ec_default=True): for p in storage_policy.POLICIES: if p.policy_type == storage_policy.EC_POLICY: frag_index = random.randint(0, 10) trailing_frag = '-%d' % (frag_index) else: frag_index = None trailing_frag = '' ext = random.choice(['data', 'ts']) object_prefix = storage_policy.get_policy_string('objects', p) key = key_template % { 'prefix': object_prefix, 'hash': hash_, 'timestamp': t.internal, 'ext': ext, 'nonce': nonce, 'trailing_frag': trailing_frag, } expected = { 'hashpath': hash_, 'policy': p, 'nonce': str(nonce), 'frag_index': frag_index, 'timestamp': t.internal, 'ext': ext, } try: self.assertEqual(replicator.split_key(key), expected) except AssertionError as e: msg = '%s\n\n ... for key %r' % (e, key) self.fail(msg)