def get_object_ring(self, policy_idx): """ Get the ring object to use based on its policy. :policy_idx: policy index as defined in swift.conf :returns: appropriate ring object """ return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def get_object_ring(self, policy_idx): """ Get the ring identified by the policy index :param policy_idx: Storage policy index :returns: A ring matching the storage policy """ return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) Manager(['account-reaper']).once() get_to_final_state() for policy, container, obj in all_objects: cpart, cnodes = self.container_ring.get_nodes( self.account, container) for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, node)) object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) for node in nodes: try: direct_get_object(node, part, self.account, container, obj) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy))
def _get_object_info(self, account, container, obj, number): obj_conf = self.configs['object-server'] config_path = obj_conf[number] options = utils.readconf(config_path, 'app:object-server') swift_dir = options.get('swift_dir', '/etc/swift') ring = POLICIES.get_object_ring(int(self.policy), swift_dir) part, nodes = ring.get_nodes(account, container, obj) for node in nodes: # assumes one to one mapping if node['port'] == int(options.get('bind_port')): device = node['device'] break else: return None mgr = DiskFileManager(options, get_logger(options)) disk_file = mgr.get_diskfile(device, part, account, container, obj, self.policy) info = disk_file.read_metadata() return info
def test_merge_storage_policy_index(self): # generic split brain self.brain.stop_primary_half() self.brain.put_container() self.brain.start_primary_half() self.brain.stop_handoff_half() self.brain.put_container() self.brain.put_object() self.brain.start_handoff_half() # make sure we have some manner of split brain container_part, container_nodes = self.container_ring.get_nodes( self.account, self.container_name) head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assert_(len(found_policy_indexes) > 1, 'primary nodes did not disagree about policy index %r' % head_responses) # find our object orig_policy_index = None for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': policy_index}) except direct_client.ClientException as err: continue orig_policy_index = policy_index break if orig_policy_index is not None: break else: self.fail('Unable to find /%s/%s/%s in %r' % ( self.account, self.container_name, self.object_name, found_policy_indexes)) get_to_final_state() Manager(['container-reconciler']).once() # validate containers head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assert_(len(found_policy_indexes) == 1, 'primary nodes disagree about policy index %r' % head_responses) expected_policy_index = found_policy_indexes.pop() self.assertNotEqual(orig_policy_index, expected_policy_index) # validate object placement orig_policy_ring = POLICIES.get_object_ring(orig_policy_index, '/etc/swift') for node in orig_policy_ring.devs: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': orig_policy_index}) except direct_client.ClientException as err: if err.http_status == HTTP_NOT_FOUND: continue raise else: self.fail('Found /%s/%s/%s in %s' % ( self.account, self.container_name, self.object_name, orig_policy_index)) # use proxy to access object (bad container info might be cached...) timeout = time.time() + TIMEOUT while time.time() < timeout: try: metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise time.sleep(1) else: break else: self.fail('could not HEAD /%s/%s/%s/ from policy %s ' 'after %s seconds.' % ( self.account, self.container_name, self.object_name, expected_policy_index, TIMEOUT))
def test_reconcile_symlink(self): if 'symlink' not in self.cluster_info: raise unittest.SkipTest("Symlink not enabled in proxy; can't test " "symlink reconciliation") wrong_policy = random.choice(ENABLED_POLICIES) policy = random.choice( [p for p in ENABLED_POLICIES if p is not wrong_policy]) # get an old container stashed self.brain.stop_primary_half() self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some target data target_name = self.get_object_name('target') self.brain.client.put_object(self.container_name, target_name, {}, b'this is the target data') # write the symlink self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) symlink_name = self.get_object_name('symlink') self.brain.client.put_object( self.container_name, symlink_name, { 'X-Symlink-Target': '%s/%s' % (self.container_name, target_name), 'Content-Type': 'application/symlink', }, b'') # at this point we have a broken symlink (the container_info has the # proxy looking for the target in the wrong policy) with self.assertRaises(ClientException) as ctx: self.brain.client.get_object(self.container_name, symlink_name) self.assertEqual(ctx.exception.http_status, 404) # of course the symlink itself is fine metadata, body = self.brain.client.get_object( self.container_name, symlink_name, query_string='symlink=get') self.assertEqual( metadata['x-symlink-target'], utils.quote('%s/%s' % (self.container_name, target_name))) self.assertEqual(metadata['content-type'], 'application/symlink') self.assertEqual(body, b'') # ... although in the wrong policy object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, symlink_name) for node in nodes: metadata = direct_client.direct_head_object( node, part, self.account, self.container_name, symlink_name, headers={'X-Backend-Storage-Policy-Index': int(wrong_policy)}) self.assertEqual( metadata['X-Object-Sysmeta-Symlink-Target'], utils.quote('%s/%s' % (self.container_name, target_name))) # let the reconciler run self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache self.brain.client.post_container(self.container_name, {}) # now the symlink works metadata, body = self.brain.client.get_object(self.container_name, symlink_name) self.assertEqual(body, b'this is the target data') # and it's in the correct policy object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, symlink_name) for node in nodes: metadata = direct_client.direct_head_object( node, part, self.account, self.container_name, symlink_name, headers={'X-Backend-Storage-Policy-Index': int(policy)}) self.assertEqual( metadata['X-Object-Sysmeta-Symlink-Target'], utils.quote('%s/%s' % (self.container_name, target_name)))
def test_reconcile_manifest(self): manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() policy = random.choice(ENABLED_POLICIES) self.brain.put_container(policy.idx) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() wrong_policy = random.choice([p for p in ENABLED_POLICIES if p is not policy]) self.brain.put_container(wrong_policy.idx) # write some more parts for i in range(10, 20): write_part(i) # write manifest try: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') except ClientException as err: # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(err.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEquals(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual(body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def print_item_locations(ring, ring_name=None, account=None, container=None, obj=None, **kwargs): """ Display placement information for an item based on ring lookup. If a ring is provided it always takes precedence, but warnings will be emitted if it doesn't match other optional arguments like the policy_name or ring_name. If no ring is provided the ring_name and/or policy_name will be used to lookup the ring. :param ring: a ring instance :param ring_name: server type, or storage policy ring name if object ring :param account: account name :param container: container name :param obj: object name :param partition: part number for non path lookups :param policy_name: name of storage policy to use to lookup the ring :param all_nodes: include all handoff nodes. If false, only the N primary nodes and first N handoffs will be printed. """ policy_name = kwargs.get("policy_name", None) part = kwargs.get("partition", None) all_nodes = kwargs.get("all", False) swift_dir = kwargs.get("swift_dir", "/etc/swift") if ring and policy_name: policy = POLICIES.get_by_name(policy_name) if policy: if ring_name != policy.ring_name: print "Attention! mismatch between ring and policy detected!" else: print "Attention! Policy %s is not valid" % policy_name policy_index = None if ring is None and (obj or part): if not policy_name: print "Need a ring or policy" raise InfoSystemExit() policy = POLICIES.get_by_name(policy_name) if not policy: print "No policy named %r" % policy_name raise InfoSystemExit() policy_index = int(policy) ring = POLICIES.get_object_ring(policy_index, swift_dir) ring_name = (POLICIES.get_by_name(policy_name)).ring_name if account is None and (container is not None or obj is not None): print "No account specified" raise InfoSystemExit() if container is None and obj is not None: print "No container specified" raise InfoSystemExit() if account is None and part is None: print "No target specified" raise InfoSystemExit() loc = "<type>" if part and ring_name: if "-" in ring_name and ring_name.startswith("object"): loc = "objects-" + ring_name.split("-", 1)[1] else: loc = ring_name + "s" if account and container and obj: loc = "objects" if "-" in ring_name and ring_name.startswith("object"): policy_index = int(ring_name.rsplit("-", 1)[1]) loc = "objects-%d" % policy_index if account and container and not obj: loc = "containers" if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name="container") else: if ring_name != "container": print "Attention! mismatch between ring and item detected!" if account and not container and not obj: loc = "accounts" if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name="account") else: if ring_name != "account": print "Attention! mismatch between ring and item detected!" print "\nAccount \t%s" % account print "Container\t%s" % container print "Object \t%s\n\n" % obj print_ring_locations(ring, loc, account, container, obj, part, all_nodes, policy_index=policy_index)
def PUT(self, req): version, acc, con, obj = split_path(req.path, 1, 4, True) stor_policy = req.headers['storage_policy'] ring = POLICIES.get_object_ring(stor_policy, '/etc/swift') #broker = self._get_metadata_broker() #broker.initialize() #Handle Container PUT if not obj: hsh = hash_path(acc, con) part = ring.get_part(acc, con) db_dir = storage_directory(swift.container.backend.DATADIR, part, hsh) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: path = os.path.join(self.root + item, db_dir, hsh + '.db') #TODO: move kwargs kwargs = { 'account': acc, 'container': con, 'logger': self.logger } md_broker = swift.container.backend.ContainerBroker( path, **kwargs) md = md_broker.get_info() md.update( (key, value) for key, (value, timestamp ) in md_broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('container', key)) sys_md = format_con_metadata(md) user_md = format_custom_metadata(md) if 'X-Container-Read' in req.headers: sys_md[ 'container_read_permissions'] = req.headers[ 'X-Container-Read'] if 'X-Container-Write' in req.headers: sys_md[ 'container_write_permissions'] = req.headers[ 'X-Container-Write'] #TODO: insert container_last_activity_time #TODO: split meta user/sys #TODO: insert meta self.broker.insert_container_md([sys_md]) return except DatabaseConnectionError as e: self.logger.warn("DatabaseConnectionError: " + e.path + "\n") pass except: self.logger.warn("%s: %s\n" % (str( sys.exc_info()[0]), str(sys.exc_info()[1]))) pass #handle object PUT else: part = ring.get_part(acc, con, obj) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: df = self.diskfile_mgr.get_diskfile( item, part, acc, con, obj, stor_policy) md = df.read_metadata() sys_md = format_obj_metadata(md) #df._data_file is a direct path to the objects data sys_md['object_location'] = df._data_file user_md = format_custom_metadata(md) #TODO: insert user meta and sys meta self.broker.insert_object_md([sys_md]) except: self.logger.warn("%s: %s\n" % (str( sys.exc_info()[0]), str(sys.exc_info()[1]))) pass return
def test_merge_storage_policy_index(self): # generic split brain self.brain.stop_primary_half() self.brain.put_container() self.brain.start_primary_half() self.brain.stop_handoff_half() self.brain.put_container() self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'}, contents='VERIFY') self.brain.start_handoff_half() # make sure we have some manner of split brain container_part, container_nodes = self.container_ring.get_nodes( self.account, self.container_name) head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assertTrue( len(found_policy_indexes) > 1, 'primary nodes did not disagree about policy index %r' % head_responses) # find our object orig_policy_index = None for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': policy_index }) except direct_client.ClientException as err: continue orig_policy_index = policy_index break if orig_policy_index is not None: break else: self.fail('Unable to find /%s/%s/%s in %r' % (self.account, self.container_name, self.object_name, found_policy_indexes)) self.get_to_final_state() Manager(['container-reconciler']).once() # validate containers head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assertTrue( len(found_policy_indexes) == 1, 'primary nodes disagree about policy index %r' % head_responses) expected_policy_index = found_policy_indexes.pop() self.assertNotEqual(orig_policy_index, expected_policy_index) # validate object placement orig_policy_ring = POLICIES.get_object_ring(orig_policy_index, '/etc/swift') for node in orig_policy_ring.devs: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': orig_policy_index }) except direct_client.ClientException as err: if err.http_status == HTTP_NOT_FOUND: continue raise else: self.fail('Found /%s/%s/%s in %s' % (self.account, self.container_name, self.object_name, orig_policy_index)) # verify that the object data read by external client is correct headers, data = self._get_object_patiently(expected_policy_index) self.assertEqual('VERIFY', data) self.assertEqual('custom-meta', headers['x-object-meta-test'])
def test_reconcile_manifest(self): info_url = "%s://%s/info" % (urlparse(self.url).scheme, urlparse(self.url).netloc) proxy_conn = client.http_connection(info_url) cluster_info = client.get_capabilities(proxy_conn) if 'slo' not in cluster_info: raise SkipTest("SLO not enabled in proxy; " "can't test manifest reconciliation") # this test is not only testing a split brain scenario on # multiple policies with mis-placed objects - it even writes out # a static large object directly to the storage nodes while the # objects are unavailably mis-placed from *behind* the proxy and # doesn't know how to do that for EC_POLICY (clayg: why did you # guys let me write a test that does this!?) - so we force # wrong_policy (where the manifest gets written) to be one of # any of your configured REPL_POLICY (we know you have one # because this is a ReplProbeTest) wrong_policy = random.choice(POLICIES_BY_TYPE[REPL_POLICY]) policy = random.choice([p for p in ENABLED_POLICIES if p is not wrong_policy]) manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # write some more parts for i in range(10, 20): write_part(i) # write manifest with self.assertRaises(ClientException) as catcher: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(catcher.exception.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEqual(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual(body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def test_reconcile_manifest(self): manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() policy = random.choice(ENABLED_POLICIES) self.brain.put_container(policy.idx) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() wrong_policy = random.choice( [p for p in ENABLED_POLICIES if p is not policy]) self.brain.put_container(wrong_policy.idx) # write some more parts for i in range(10, 20): write_part(i) # write manifest try: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') except ClientException as err: # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(err.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEquals(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object(self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual( body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def test_merge_storage_policy_index(self): # generic split brain self.brain.stop_primary_half() self.brain.put_container() self.brain.start_primary_half() self.brain.stop_handoff_half() self.brain.put_container() self.brain.put_object() self.brain.start_handoff_half() # make sure we have some manner of split brain container_part, container_nodes = self.container_ring.get_nodes( self.account, self.container_name) head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assert_( len(found_policy_indexes) > 1, 'primary nodes did not disagree about policy index %r' % head_responses) # find our object orig_policy_index = None for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': policy_index }) except direct_client.ClientException as err: continue orig_policy_index = policy_index break if orig_policy_index is not None: break else: self.fail('Unable to find /%s/%s/%s in %r' % (self.account, self.container_name, self.object_name, found_policy_indexes)) get_to_final_state() Manager(['container-reconciler']).once() # validate containers head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assert_( len(found_policy_indexes) == 1, 'primary nodes disagree about policy index %r' % head_responses) expected_policy_index = found_policy_indexes.pop() self.assertNotEqual(orig_policy_index, expected_policy_index) # validate object placement orig_policy_ring = POLICIES.get_object_ring(orig_policy_index, '/etc/swift') for node in orig_policy_ring.devs: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': orig_policy_index }) except direct_client.ClientException as err: if err.http_status == HTTP_NOT_FOUND: continue raise else: self.fail('Found /%s/%s/%s in %s' % (self.account, self.container_name, self.object_name, orig_policy_index)) # use proxy to access object (bad container info might be cached...) timeout = time.time() + TIMEOUT while time.time() < timeout: try: metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise time.sleep(1) else: break else: self.fail('could not HEAD /%s/%s/%s/ from policy %s ' 'after %s seconds.' % (self.account, self.container_name, self.object_name, expected_policy_index, TIMEOUT))
def POST(self, req): version, acc, con, obj = split_path(req.path, 1, 4, True) stor_policy = req.headers['storage_policy'] ring = POLICIES.get_object_ring(stor_policy, '/etc/swift') if not con and not obj: meta_type = 'account' kwargs = {'account':acc, 'logger':self.logger} data_dir = swift.account.backend.DATADIR hsh = hash_path(acc) part = ring.get_part(acc) db_dir = storage_directory(data_dir, part, hsh) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: path = os.path.join(self.root + item, db_dir, hsh + '.db') broker = swift.account.backend.AccountBroker(path, **kwargs) md = broker.get_info() md.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta(meta_type, key)) sys_md = format_acc_metadata(md) user_md = format_custom_metadata(md) #TODO: call overwrite_account_metadata #TODO: call overwrite_custom_metadata return except: self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1]))) pass #Handle Container POST elif not obj: meta_type = 'container' kwargs = {'account':acc, 'container':con, 'logger':self.logger} data_dir = swift.container.backend.DATADIR try: hsh = hash_path(acc, con) part = ring.get_part(acc, con) db_dir = storage_directory(data_dir, part, hsh) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: path = os.path.join(self.root + item, db_dir, hsh + '.db') broker = swift.container.backend.ContainerBroker(path, **kwargs) md = broker.get_info() md.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('container', key)) sys_md = format_con_metadata(md) user_md = format_custom_metadata(md) if 'X-Container-Read' in req.headers: sys_md['container_read_permissions'] = req.headers['X-Container-Read'] if 'X-Container-Write' in req.headers: sys_md['container_write_permissions'] = req.headers['X-Container-Write'] #TODO: call overwrite_container_metadata #TODO: call overwrite_custom_metadata return except DatabaseConnectionError as e: self.logger.warn("DatabaseConnectionError: " + e.path + "\n") pass except: self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1]))) pass else: part = ring.get_part(acc, con, obj) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: df = self.diskfile_mgr.get_diskfile(item, part, acc, con, obj, stor_policy) md = df.read_metadata() sys_md = format_obj_metadata(md) user_md = format_custom_metadata(md) #TODO: call overwrite_object_metadata #TODO: call overwrite_custom_metadata except: self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1]))) pass return
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) # run the reaper Manager(['account-reaper']).once() for policy, container, obj in all_objects: # verify that any container deletes were at same timestamp cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: # Container replicas may not yet be deleted if we have a # policy with object replicas < container replicas, so # ignore successful HEAD. We'll check for all replicas to # be deleted again after running the replicators. pass self.assertEqual(1, len(delete_times), delete_times) # verify that all object deletes were at same timestamp object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times)) # run replicators and updaters self.get_to_final_state() for policy, container, obj in all_objects: # verify that ALL container replicas are now deleted cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, cnode)) # sanity check that object state is still consistent... object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times))
def PUT(self, req): version, acc, con, obj = split_path(req.path, 1, 4, True) stor_policy = req.headers['storage_policy'] ring = POLICIES.get_object_ring(stor_policy, '/etc/swift') #broker = self._get_metadata_broker() #broker.initialize() #Handle Container PUT if not obj: hsh = hash_path(acc, con) part = ring.get_part(acc, con) db_dir = storage_directory(swift.container.backend.DATADIR, part, hsh) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: path = os.path.join(self.root + item, db_dir, hsh + '.db') #TODO: move kwargs kwargs = {'account':acc, 'container':con, 'logger':self.logger} md_broker= swift.container.backend.ContainerBroker(path, **kwargs) md = md_broker.get_info() md.update( (key, value) for key, (value, timestamp) in md_broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('container', key)) sys_md = format_con_metadata(md) user_md = format_custom_metadata(md) if 'X-Container-Read' in req.headers: sys_md['container_read_permissions'] = req.headers['X-Container-Read'] if 'X-Container-Write' in req.headers: sys_md['container_write_permissions'] = req.headers['X-Container-Write'] #TODO: insert container_last_activity_time #TODO: split meta user/sys #TODO: insert meta self.broker.insert_container_md([sys_md]) return except DatabaseConnectionError as e: self.logger.warn("DatabaseConnectionError: " + e.path + "\n") pass except: self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1]))) pass #handle object PUT else: part = ring.get_part(acc, con, obj) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: df = self.diskfile_mgr.get_diskfile(item, part, acc, con, obj, stor_policy) md = df.read_metadata() sys_md = format_obj_metadata(md) #df._data_file is a direct path to the objects data sys_md['object_location'] = df._data_file user_md = format_custom_metadata(md) #TODO: insert user meta and sys meta self.broker.insert_object_md([sys_md]) except: self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1]))) pass return
def _get_devices(self): return set([ d['device'] for policy in POLICIES for d in POLICIES.get_object_ring(int(policy), self.swift_dir).devs if d ])
def POST(self, req): version, acc, con, obj = split_path(req.path, 1, 4, True) stor_policy = req.headers['storage_policy'] ring = POLICIES.get_object_ring(stor_policy, '/etc/swift') if not con and not obj: meta_type = 'account' kwargs = {'account': acc, 'logger': self.logger} data_dir = swift.account.backend.DATADIR hsh = hash_path(acc) part = ring.get_part(acc) db_dir = storage_directory(data_dir, part, hsh) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: path = os.path.join(self.root + item, db_dir, hsh + '.db') broker = swift.account.backend.AccountBroker( path, **kwargs) md = broker.get_info() md.update((key, value) for key, ( value, timestamp) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta(meta_type, key)) sys_md = format_acc_metadata(md) user_md = format_custom_metadata(md) #TODO: call overwrite_account_metadata #TODO: call overwrite_custom_metadata return except: self.logger.warn("%s: %s\n" % (str( sys.exc_info()[0]), str(sys.exc_info()[1]))) pass #Handle Container POST elif not obj: meta_type = 'container' kwargs = {'account': acc, 'container': con, 'logger': self.logger} data_dir = swift.container.backend.DATADIR try: hsh = hash_path(acc, con) part = ring.get_part(acc, con) db_dir = storage_directory(data_dir, part, hsh) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: path = os.path.join(self.root + item, db_dir, hsh + '.db') broker = swift.container.backend.ContainerBroker( path, **kwargs) md = broker.get_info() md.update( (key, value) for key, (value, timestamp ) in broker.metadata.iteritems() if value != '' and is_sys_or_user_meta('container', key)) sys_md = format_con_metadata(md) user_md = format_custom_metadata(md) if 'X-Container-Read' in req.headers: sys_md[ 'container_read_permissions'] = req.headers[ 'X-Container-Read'] if 'X-Container-Write' in req.headers: sys_md[ 'container_write_permissions'] = req.headers[ 'X-Container-Write'] #TODO: call overwrite_container_metadata #TODO: call overwrite_custom_metadata return except DatabaseConnectionError as e: self.logger.warn("DatabaseConnectionError: " + e.path + "\n") pass except: self.logger.warn( "%s: %s\n" % (str(sys.exc_info()[0]), str(sys.exc_info()[1]))) pass else: part = ring.get_part(acc, con, obj) nodes = ring.get_part_nodes(part) for node in nodes: for item in self.devicelist: if node['device'] in item: try: df = self.diskfile_mgr.get_diskfile( item, part, acc, con, obj, stor_policy) md = df.read_metadata() sys_md = format_obj_metadata(md) user_md = format_custom_metadata(md) #TODO: call overwrite_object_metadata #TODO: call overwrite_custom_metadata except: self.logger.warn("%s: %s\n" % (str( sys.exc_info()[0]), str(sys.exc_info()[1]))) pass return
def test_reconcile_delete(self): # generic split brain self.brain.stop_primary_half() self.brain.put_container() self.brain.put_object() self.brain.start_primary_half() self.brain.stop_handoff_half() self.brain.put_container() self.brain.delete_object() self.brain.start_handoff_half() # make sure we have some manner of split brain container_part, container_nodes = self.container_ring.get_nodes( self.account, self.container_name) head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assertTrue( len(found_policy_indexes) > 1, 'primary nodes did not disagree about policy index %r' % head_responses) # find our object orig_policy_index = ts_policy_index = None for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': policy_index }) except direct_client.ClientException as err: if 'x-backend-timestamp' in err.http_headers: ts_policy_index = policy_index break else: orig_policy_index = policy_index break if not orig_policy_index: self.fail('Unable to find /%s/%s/%s in %r' % (self.account, self.container_name, self.object_name, found_policy_indexes)) if not ts_policy_index: self.fail('Unable to find tombstone /%s/%s/%s in %r' % (self.account, self.container_name, self.object_name, found_policy_indexes)) self.get_to_final_state() Manager(['container-reconciler']).once() # validate containers head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) new_found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assertTrue( len(new_found_policy_indexes) == 1, 'primary nodes disagree about policy index %r' % dict( (node['port'], metadata['X-Backend-Storage-Policy-Index']) for node, metadata in head_responses)) expected_policy_index = new_found_policy_indexes.pop() self.assertEqual(orig_policy_index, expected_policy_index) # validate object fully deleted for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': policy_index }) except direct_client.ClientException as err: if err.http_status == HTTP_NOT_FOUND: continue else: self.fail('Found /%s/%s/%s in %s on %s' % (self.account, self.container_name, self.object_name, orig_policy_index, node))
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) # run the reaper Manager(['account-reaper']).once() for policy, container, obj in all_objects: # verify that any container deletes were at same timestamp cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: # Container replicas may not yet be deleted if we have a # policy with object replicas < container replicas, so # ignore successful HEAD. We'll check for all replicas to # be deleted again after running the replicators. pass self.assertEqual(1, len(delete_times), delete_times) # verify that all object deletes were at same timestamp object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times)) # run replicators and updaters self.get_to_final_state() for policy, container, obj in all_objects: # verify that ALL container replicas are now deleted cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, cnode)) # sanity check that object state is still consistent... object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times))
def test_reconcile_manifest(self): info_url = "%s://%s/info" % (urlparse( self.url).scheme, urlparse(self.url).netloc) proxy_conn = client.http_connection(info_url) cluster_info = client.get_capabilities(proxy_conn) if 'slo' not in cluster_info: raise SkipTest("SLO not enabled in proxy; " "can't test manifest reconciliation") # this test is not only testing a split brain scenario on # multiple policies with mis-placed objects - it even writes out # a static large object directly to the storage nodes while the # objects are unavailably mis-placed from *behind* the proxy and # doesn't know how to do that for EC_POLICY (clayg: why did you # guys let me write a test that does this!?) - so we force # wrong_policy (where the manifest gets written) to be one of # any of your configured REPL_POLICY (we know you have one # because this is a ReplProbeTest) wrong_policy = random.choice(POLICIES_BY_TYPE[REPL_POLICY]) policy = random.choice( [p for p in ENABLED_POLICIES if p is not wrong_policy]) manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # write some more parts for i in range(10, 20): write_part(i) # write manifest with self.assertRaises(ClientException) as catcher: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(catcher.exception.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEqual(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object(self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual( body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift', policy_name=''): """ Display information about an object read from the datafile. Optionally verify the datafile content matches the ETag metadata. :param datafile: path on disk to object file :param check_etag: boolean, will read datafile content and verify computed checksum matches value stored in metadata. :param swift_dir: the path on disk to rings :param policy_name: optionally the name to use when finding the ring """ if not os.path.exists(datafile): print "Data file doesn't exist" raise InfoSystemExit() if not datafile.startswith(('/', './')): datafile = './' + datafile policy_index = None ring = None datadir = DATADIR_BASE # try to extract policy index from datafile disk path policy_index = int(extract_policy(datafile) or POLICIES.legacy) try: if policy_index: datadir += '-' + str(policy_index) ring = Ring(swift_dir, ring_name='object-' + str(policy_index)) elif policy_index == 0: ring = Ring(swift_dir, ring_name='object') except IOError: # no such ring pass if policy_name: policy = POLICIES.get_by_name(policy_name) if policy: policy_index_for_name = policy.idx if (policy_index is not None and policy_index_for_name is not None and policy_index != policy_index_for_name): print 'Attention: Ring does not match policy!' print 'Double check your policy name!' if not ring and policy_index_for_name: ring = POLICIES.get_object_ring(policy_index_for_name, swift_dir) datadir = get_data_dir(policy_index_for_name) with open(datafile, 'rb') as fp: try: metadata = read_metadata(fp) except EOFError: print "Invalid metadata" raise InfoSystemExit() etag = metadata.pop('ETag', '') length = metadata.pop('Content-Length', '') path = metadata.get('name', '') print_obj_metadata(metadata) # Optional integrity check; it's useful, but slow. file_len = None if check_etag: h = md5() file_len = 0 while True: data = fp.read(64 * 1024) if not data: break h.update(data) file_len += len(data) h = h.hexdigest() if etag: if h == etag: print 'ETag: %s (valid)' % etag else: print ("ETag: %s doesn't match file hash of %s!" % (etag, h)) else: print 'ETag: Not found in metadata' else: print 'ETag: %s (not checked)' % etag file_len = os.fstat(fp.fileno()).st_size if length: if file_len == int(length): print 'Content-Length: %s (valid)' % length else: print ("Content-Length: %s doesn't match file length of %s" % (length, file_len)) else: print 'Content-Length: Not found in metadata' account, container, obj = path.split('/', 3)[1:] if ring: print_ring_locations(ring, datadir, account, container, obj, policy_index=policy_index)
def test_reconcile_delete(self): # generic split brain self.brain.stop_primary_half() self.brain.put_container() self.brain.put_object() self.brain.start_primary_half() self.brain.stop_handoff_half() self.brain.put_container() self.brain.delete_object() self.brain.start_handoff_half() # make sure we have some manner of split brain container_part, container_nodes = self.container_ring.get_nodes( self.account, self.container_name) head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assertTrue( len(found_policy_indexes) > 1, 'primary nodes did not disagree about policy index %r' % head_responses) # find our object orig_policy_index = ts_policy_index = None for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': policy_index}) except direct_client.ClientException as err: if 'x-backend-timestamp' in err.http_headers: ts_policy_index = policy_index break else: orig_policy_index = policy_index break if not orig_policy_index: self.fail('Unable to find /%s/%s/%s in %r' % ( self.account, self.container_name, self.object_name, found_policy_indexes)) if not ts_policy_index: self.fail('Unable to find tombstone /%s/%s/%s in %r' % ( self.account, self.container_name, self.object_name, found_policy_indexes)) self.get_to_final_state() Manager(['container-reconciler']).once() # validate containers head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) new_found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assertTrue(len(new_found_policy_indexes) == 1, 'primary nodes disagree about policy index %r' % dict((node['port'], metadata['X-Backend-Storage-Policy-Index']) for node, metadata in head_responses)) expected_policy_index = new_found_policy_indexes.pop() self.assertEqual(orig_policy_index, expected_policy_index) # validate object fully deleted for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': policy_index}) except direct_client.ClientException as err: if err.http_status == HTTP_NOT_FOUND: continue else: self.fail('Found /%s/%s/%s in %s on %s' % ( self.account, self.container_name, self.object_name, orig_policy_index, node))
def get_object_ring(self, storage_policy_index): return POLICIES.get_object_ring(storage_policy_index, self.swift_dir)
def test_merge_storage_policy_index(self): # generic split brain self.brain.stop_primary_half() self.brain.put_container() self.brain.start_primary_half() self.brain.stop_handoff_half() self.brain.put_container() self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'}, contents='VERIFY') self.brain.start_handoff_half() # make sure we have some manner of split brain container_part, container_nodes = self.container_ring.get_nodes( self.account, self.container_name) head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assertTrue( len(found_policy_indexes) > 1, 'primary nodes did not disagree about policy index %r' % head_responses) # find our object orig_policy_index = None for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': policy_index}) except direct_client.ClientException as err: continue orig_policy_index = policy_index break if orig_policy_index is not None: break else: self.fail('Unable to find /%s/%s/%s in %r' % ( self.account, self.container_name, self.object_name, found_policy_indexes)) self.get_to_final_state() Manager(['container-reconciler']).once() # validate containers head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assertTrue(len(found_policy_indexes) == 1, 'primary nodes disagree about policy index %r' % head_responses) expected_policy_index = found_policy_indexes.pop() self.assertNotEqual(orig_policy_index, expected_policy_index) # validate object placement orig_policy_ring = POLICIES.get_object_ring(orig_policy_index, '/etc/swift') for node in orig_policy_ring.devs: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': orig_policy_index}) except direct_client.ClientException as err: if err.http_status == HTTP_NOT_FOUND: continue raise else: self.fail('Found /%s/%s/%s in %s' % ( self.account, self.container_name, self.object_name, orig_policy_index)) # verify that the object data read by external client is correct headers, data = self._get_object_patiently(expected_policy_index) self.assertEqual('VERIFY', data) self.assertEqual('custom-meta', headers['x-object-meta-test'])
def print_item_locations(ring, ring_name=None, account=None, container=None, obj=None, **kwargs): """ Display placement information for an item based on ring lookup. If a ring is provided it always takes precedence, but warnings will be emitted if it doesn't match other optional arguments like the policy_name or ring_name. If no ring is provided the ring_name and/or policy_name will be used to lookup the ring. :param ring: a ring instance :param ring_name: server type, or storage policy ring name if object ring :param account: account name :param container: container name :param obj: object name :param partition: part number for non path lookups :param policy_name: name of storage policy to use to lookup the ring :param all_nodes: include all handoff nodes. If false, only the N primary nodes and first N handoffs will be printed. """ policy_name = kwargs.get('policy_name', None) part = kwargs.get('partition', None) all_nodes = kwargs.get('all', False) swift_dir = kwargs.get('swift_dir', '/etc/swift') if ring and policy_name: policy = POLICIES.get_by_name(policy_name) if policy: if ring_name != policy.ring_name: print('Warning: mismatch between ring and policy name!') else: print('Warning: Policy %s is not valid' % policy_name) policy_index = None if ring is None and (obj or part): if not policy_name: print('Need a ring or policy') raise InfoSystemExit() policy = POLICIES.get_by_name(policy_name) if not policy: print('No policy named %r' % policy_name) raise InfoSystemExit() policy_index = int(policy) ring = POLICIES.get_object_ring(policy_index, swift_dir) ring_name = (POLICIES.get_by_name(policy_name)).ring_name if account is None and (container is not None or obj is not None): print('No account specified') raise InfoSystemExit() if container is None and obj is not None: print('No container specified') raise InfoSystemExit() if account is None and part is None: print('No target specified') raise InfoSystemExit() loc = '<type>' if part and ring_name: if '-' in ring_name and ring_name.startswith('object'): loc = 'objects-' + ring_name.split('-', 1)[1] else: loc = ring_name + 's' if account and container and obj: loc = 'objects' if '-' in ring_name and ring_name.startswith('object'): policy_index = int(ring_name.rsplit('-', 1)[1]) loc = 'objects-%d' % policy_index if account and container and not obj: loc = 'containers' if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name='container') else: if ring_name != 'container': print('Warning: account/container specified ' + 'but ring not named "container"') if account and not container and not obj: loc = 'accounts' if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name='account') else: if ring_name != 'account': print('Warning: account specified ' + 'but ring not named "account"') print('\nAccount \t%s' % account) print('Container\t%s' % container) print('Object \t%s\n\n' % obj) print_ring_locations(ring, loc, account, container, obj, part, all_nodes, policy_index=policy_index)
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift', policy_name=''): """ Display information about an object read from the datafile. Optionally verify the datafile content matches the ETag metadata. :param datafile: path on disk to object file :param check_etag: boolean, will read datafile content and verify computed checksum matches value stored in metadata. :param swift_dir: the path on disk to rings :param policy_name: optionally the name to use when finding the ring """ if not os.path.exists(datafile): print("Data file doesn't exist") raise InfoSystemExit() if not datafile.startswith(('/', './')): datafile = './' + datafile policy_index = None ring = None datadir = DATADIR_BASE # try to extract policy index from datafile disk path fullpath = os.path.abspath(datafile) policy_index = int(extract_policy(fullpath) or POLICIES.legacy) try: if policy_index: datadir += '-' + str(policy_index) ring = Ring(swift_dir, ring_name='object-' + str(policy_index)) elif policy_index == 0: ring = Ring(swift_dir, ring_name='object') except IOError: # no such ring pass if policy_name: policy = POLICIES.get_by_name(policy_name) if policy: policy_index_for_name = policy.idx if (policy_index is not None and policy_index_for_name is not None and policy_index != policy_index_for_name): print('Warning: Ring does not match policy!') print('Double check your policy name!') if not ring and policy_index_for_name: ring = POLICIES.get_object_ring(policy_index_for_name, swift_dir) datadir = get_data_dir(policy_index_for_name) with open(datafile, 'rb') as fp: try: metadata = read_metadata(fp) except EOFError: print("Invalid metadata") raise InfoSystemExit() etag = metadata.pop('ETag', '') length = metadata.pop('Content-Length', '') path = metadata.get('name', '') print_obj_metadata(metadata) # Optional integrity check; it's useful, but slow. file_len = None if check_etag: h = md5() file_len = 0 while True: data = fp.read(64 * 1024) if not data: break h.update(data) file_len += len(data) h = h.hexdigest() if etag: if h == etag: print('ETag: %s (valid)' % etag) else: print("ETag: %s doesn't match file hash of %s!" % (etag, h)) else: print('ETag: Not found in metadata') else: print('ETag: %s (not checked)' % etag) file_len = os.fstat(fp.fileno()).st_size if length: if file_len == int(length): print('Content-Length: %s (valid)' % length) else: print("Content-Length: %s doesn't match file length of %s" % (length, file_len)) else: print('Content-Length: Not found in metadata') account, container, obj = path.split('/', 3)[1:] if ring: print_ring_locations(ring, datadir, account, container, obj, policy_index=policy_index)
def test_reconcile_symlink(self): if 'symlink' not in self.cluster_info: raise unittest.SkipTest( "Symlink not enabled in proxy; can't test " "symlink reconciliation") wrong_policy = random.choice(ENABLED_POLICIES) policy = random.choice([p for p in ENABLED_POLICIES if p is not wrong_policy]) # get an old container stashed self.brain.stop_primary_half() self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some target data client.put_object(self.url, self.token, self.container_name, 'target', contents='this is the target data') # write the symlink self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) client.put_object( self.url, self.token, self.container_name, 'symlink', headers={ 'X-Symlink-Target': '%s/target' % self.container_name, 'Content-Type': 'application/symlink', }) # at this point we have a broken symlink (the container_info has the # proxy looking for the target in the wrong policy) with self.assertRaises(ClientException) as ctx: client.get_object(self.url, self.token, self.container_name, 'symlink') self.assertEqual(ctx.exception.http_status, 404) # of course the symlink itself is fine metadata, body = client.get_object(self.url, self.token, self.container_name, 'symlink', query_string='symlink=get') self.assertEqual(metadata['x-symlink-target'], '%s/target' % self.container_name) self.assertEqual(metadata['content-type'], 'application/symlink') self.assertEqual(body, '') # ... although in the wrong policy object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, 'symlink') for node in nodes: metadata = direct_client.direct_head_object( node, part, self.account, self.container_name, 'symlink', headers={'X-Backend-Storage-Policy-Index': int(wrong_policy)}) self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'], '%s/target' % self.container_name) # let the reconciler run self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # now the symlink works metadata, body = client.get_object(self.url, self.token, self.container_name, 'symlink') self.assertEqual(body, 'this is the target data') # and it's in the correct policy object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, 'symlink') for node in nodes: metadata = direct_client.direct_head_object( node, part, self.account, self.container_name, 'symlink', headers={'X-Backend-Storage-Policy-Index': int(policy)}) self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'], '%s/target' % self.container_name)
def print_item_locations(ring, ring_name=None, account=None, container=None, obj=None, **kwargs): """ Display placement information for an item based on ring lookup. If a ring is provided it always takes precedence, but warnings will be emitted if it doesn't match other optional arguments like the policy_name or ring_name. If no ring is provided the ring_name and/or policy_name will be used to lookup the ring. :param ring: a ring instance :param ring_name: server type, or storage policy ring name if object ring :param account: account name :param container: container name :param obj: object name :param partition: part number for non path lookups :param policy_name: name of storage policy to use to lookup the ring :param all_nodes: include all handoff nodes. If false, only the N primary nodes and first N handoffs will be printed. """ policy_name = kwargs.get('policy_name', None) part = kwargs.get('partition', None) all_nodes = kwargs.get('all', False) swift_dir = kwargs.get('swift_dir', '/etc/swift') if ring and policy_name: policy = POLICIES.get_by_name(policy_name) if policy: if ring_name != policy.ring_name: print 'Attention! mismatch between ring and policy detected!' else: print 'Attention! Policy %s is not valid' % policy_name policy_index = None if ring is None and (obj or part): if not policy_name: print 'Need a ring or policy' raise InfoSystemExit() policy = POLICIES.get_by_name(policy_name) if not policy: print 'No policy named %r' % policy_name raise InfoSystemExit() policy_index = int(policy) ring = POLICIES.get_object_ring(policy_index, swift_dir) ring_name = (POLICIES.get_by_name(policy_name)).ring_name if account is None and (container is not None or obj is not None): print 'No account specified' raise InfoSystemExit() if container is None and obj is not None: print 'No container specified' raise InfoSystemExit() if account is None and part is None: print 'No target specified' raise InfoSystemExit() loc = '<type>' if part and ring_name: if '-' in ring_name and ring_name.startswith('object'): loc = 'objects-' + ring_name.split('-', 1)[1] else: loc = ring_name + 's' if account and container and obj: loc = 'objects' if '-' in ring_name and ring_name.startswith('object'): policy_index = int(ring_name.rsplit('-', 1)[1]) loc = 'objects-%d' % policy_index if account and container and not obj: loc = 'containers' if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name='container') else: if ring_name != 'container': print 'Attention! mismatch between ring and item detected!' if account and not container and not obj: loc = 'accounts' if not any([ring, ring_name]): ring = Ring(swift_dir, ring_name='account') else: if ring_name != 'account': print 'Attention! mismatch between ring and item detected!' print '\nAccount \t%s' % account print 'Container\t%s' % container print 'Object \t%s\n\n' % obj print_ring_locations(ring, loc, account, container, obj, part, all_nodes, policy_index=policy_index)
def _get_devices(self): return set([ d['device'] for policy in POLICIES for d in POLICIES.get_object_ring(int(policy), self.swift_dir).devs ])