def get_object_ring(self, policy_idx):
"""
Get the ring object to use based on its policy.
:policy_idx: policy index as defined in swift.conf
:returns: appropriate ring object
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def get_object_ring(self, policy_idx):
"""
Get the ring identified by the policy index
:param policy_idx: Storage policy index
:returns: A ring matching the storage policy
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def test_sync(self):
all_objects = []
# upload some containers
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
body = 'test-body'
client.put_object(self.url, self.token, container, obj, body)
all_objects.append((policy, container, obj))
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
Manager(['account-reaper']).once()
get_to_final_state()
for policy, container, obj in all_objects:
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail('Found un-reaped /%s/%s on %r' %
(self.account, container, node))
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
for node in nodes:
try:
direct_get_object(node, part, self.account,
container, obj)
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
def _get_object_info(self, account, container, obj, number):
obj_conf = self.configs['object-server']
config_path = obj_conf[number]
options = utils.readconf(config_path, 'app:object-server')
swift_dir = options.get('swift_dir', '/etc/swift')
ring = POLICIES.get_object_ring(int(self.policy), swift_dir)
part, nodes = ring.get_nodes(account, container, obj)
for node in nodes:
# assumes one to one mapping
if node['port'] == int(options.get('bind_port')):
device = node['device']
break
else:
return None
mgr = DiskFileManager(options, get_logger(options))
disk_file = mgr.get_diskfile(device, part, account, container, obj,
self.policy)
info = disk_file.read_metadata()
return info
def test_merge_storage_policy_index(self):
# generic split brain
self.brain.stop_primary_half()
self.brain.put_container()
self.brain.start_primary_half()
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.put_object()
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assert_(len(found_policy_indexes) > 1,
'primary nodes did not disagree about policy index %r' %
head_responses)
# find our object
orig_policy_index = None
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index':
policy_index})
except direct_client.ClientException as err:
continue
orig_policy_index = policy_index
break
if orig_policy_index is not None:
break
else:
self.fail('Unable to find /%s/%s/%s in %r' % (
self.account, self.container_name, self.object_name,
found_policy_indexes))
get_to_final_state()
Manager(['container-reconciler']).once()
# validate containers
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assert_(len(found_policy_indexes) == 1,
'primary nodes disagree about policy index %r' %
head_responses)
expected_policy_index = found_policy_indexes.pop()
self.assertNotEqual(orig_policy_index, expected_policy_index)
# validate object placement
orig_policy_ring = POLICIES.get_object_ring(orig_policy_index,
'/etc/swift')
for node in orig_policy_ring.devs:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name, headers={
'X-Backend-Storage-Policy-Index': orig_policy_index})
except direct_client.ClientException as err:
if err.http_status == HTTP_NOT_FOUND:
continue
raise
else:
self.fail('Found /%s/%s/%s in %s' % (
self.account, self.container_name, self.object_name,
orig_policy_index))
# use proxy to access object (bad container info might be cached...)
timeout = time.time() + TIMEOUT
while time.time() < timeout:
try:
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
time.sleep(1)
else:
break
else:
self.fail('could not HEAD /%s/%s/%s/ from policy %s '
'after %s seconds.' % (
self.account, self.container_name, self.object_name,
expected_policy_index, TIMEOUT))
def test_reconcile_symlink(self):
if 'symlink' not in self.cluster_info:
raise unittest.SkipTest("Symlink not enabled in proxy; can't test "
"symlink reconciliation")
wrong_policy = random.choice(ENABLED_POLICIES)
policy = random.choice(
[p for p in ENABLED_POLICIES if p is not wrong_policy])
# get an old container stashed
self.brain.stop_primary_half()
self.brain.put_container(int(policy))
self.brain.start_primary_half()
# write some target data
target_name = self.get_object_name('target')
self.brain.client.put_object(self.container_name, target_name, {},
b'this is the target data')
# write the symlink
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
symlink_name = self.get_object_name('symlink')
self.brain.client.put_object(
self.container_name, symlink_name, {
'X-Symlink-Target': '%s/%s' %
(self.container_name, target_name),
'Content-Type': 'application/symlink',
}, b'')
# at this point we have a broken symlink (the container_info has the
# proxy looking for the target in the wrong policy)
with self.assertRaises(ClientException) as ctx:
self.brain.client.get_object(self.container_name, symlink_name)
self.assertEqual(ctx.exception.http_status, 404)
# of course the symlink itself is fine
metadata, body = self.brain.client.get_object(
self.container_name, symlink_name, query_string='symlink=get')
self.assertEqual(
metadata['x-symlink-target'],
utils.quote('%s/%s' % (self.container_name, target_name)))
self.assertEqual(metadata['content-type'], 'application/symlink')
self.assertEqual(body, b'')
# ... although in the wrong policy
object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift')
part, nodes = object_ring.get_nodes(self.account, self.container_name,
symlink_name)
for node in nodes:
metadata = direct_client.direct_head_object(
node,
part,
self.account,
self.container_name,
symlink_name,
headers={'X-Backend-Storage-Policy-Index': int(wrong_policy)})
self.assertEqual(
metadata['X-Object-Sysmeta-Symlink-Target'],
utils.quote('%s/%s' % (self.container_name, target_name)))
# let the reconciler run
self.brain.start_handoff_half()
self.get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
self.brain.client.post_container(self.container_name, {})
# now the symlink works
metadata, body = self.brain.client.get_object(self.container_name,
symlink_name)
self.assertEqual(body, b'this is the target data')
# and it's in the correct policy
object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift')
part, nodes = object_ring.get_nodes(self.account, self.container_name,
symlink_name)
for node in nodes:
metadata = direct_client.direct_head_object(
node,
part,
self.account,
self.container_name,
symlink_name,
headers={'X-Backend-Storage-Policy-Index': int(policy)})
self.assertEqual(
metadata['X-Object-Sysmeta-Symlink-Target'],
utils.quote('%s/%s' % (self.container_name, target_name)))
def test_reconcile_manifest(self):
manifest_data = []
def write_part(i):
body = 'VERIFY%0.2d' % i + '\x00' * 1048576
part_name = 'manifest_part_%0.2d' % i
manifest_entry = {
"path": "/%s/%s" % (self.container_name, part_name),
"etag": md5(body).hexdigest(),
"size_bytes": len(body),
}
client.put_object(self.url, self.token, self.container_name,
part_name, contents=body)
manifest_data.append(manifest_entry)
# get an old container stashed
self.brain.stop_primary_half()
policy = random.choice(ENABLED_POLICIES)
self.brain.put_container(policy.idx)
self.brain.start_primary_half()
# write some parts
for i in range(10):
write_part(i)
self.brain.stop_handoff_half()
wrong_policy = random.choice([p for p in ENABLED_POLICIES
if p is not policy])
self.brain.put_container(wrong_policy.idx)
# write some more parts
for i in range(10, 20):
write_part(i)
# write manifest
try:
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
except ClientException as err:
# so as it works out, you can't really upload a multi-part
# manifest for objects that are currently misplaced - you have to
# wait until they're all available - which is about the same as
# some other failure that causes data to be unavailable to the
# proxy at the time of upload
self.assertEqual(err.http_status, 400)
# but what the heck, we'll sneak one in just to see what happens...
direct_manifest_name = self.object_name + '-direct-test'
object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, direct_manifest_name)
container_part = self.container_ring.get_part(self.account,
self.container_name)
def translate_direct(data):
return {
'hash': data['etag'],
'bytes': data['size_bytes'],
'name': data['path'],
}
direct_manifest_data = map(translate_direct, manifest_data)
headers = {
'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n
in self.container_ring.devs),
'x-container-device': ','.join(n['device'] for n in
self.container_ring.devs),
'x-container-partition': container_part,
'X-Backend-Storage-Policy-Index': wrong_policy.idx,
'X-Static-Large-Object': 'True',
}
for node in nodes:
direct_client.direct_put_object(
node, part, self.account, self.container_name,
direct_manifest_name,
contents=utils.json.dumps(direct_manifest_data),
headers=headers)
break # one should do it...
self.brain.start_handoff_half()
get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# let's see how that direct upload worked out...
metadata, body = client.get_object(
self.url, self.token, self.container_name, direct_manifest_name,
query_string='multipart-manifest=get')
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
for i, entry in enumerate(utils.json.loads(body)):
for key in ('hash', 'bytes', 'name'):
self.assertEquals(entry[key], direct_manifest_data[i][key])
metadata, body = client.get_object(
self.url, self.token, self.container_name, direct_manifest_name)
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
self.assertEqual(body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576
for i in range(20)))
# and regular upload should work now too
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
def print_item_locations(ring, ring_name=None, account=None, container=None, obj=None, **kwargs):
"""
Display placement information for an item based on ring lookup.
If a ring is provided it always takes precedence, but warnings will be
emitted if it doesn't match other optional arguments like the policy_name
or ring_name.
If no ring is provided the ring_name and/or policy_name will be used to
lookup the ring.
:param ring: a ring instance
:param ring_name: server type, or storage policy ring name if object ring
:param account: account name
:param container: container name
:param obj: object name
:param partition: part number for non path lookups
:param policy_name: name of storage policy to use to lookup the ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
"""
policy_name = kwargs.get("policy_name", None)
part = kwargs.get("partition", None)
all_nodes = kwargs.get("all", False)
swift_dir = kwargs.get("swift_dir", "/etc/swift")
if ring and policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
if ring_name != policy.ring_name:
print "Attention! mismatch between ring and policy detected!"
else:
print "Attention! Policy %s is not valid" % policy_name
policy_index = None
if ring is None and (obj or part):
if not policy_name:
print "Need a ring or policy"
raise InfoSystemExit()
policy = POLICIES.get_by_name(policy_name)
if not policy:
print "No policy named %r" % policy_name
raise InfoSystemExit()
policy_index = int(policy)
ring = POLICIES.get_object_ring(policy_index, swift_dir)
ring_name = (POLICIES.get_by_name(policy_name)).ring_name
if account is None and (container is not None or obj is not None):
print "No account specified"
raise InfoSystemExit()
if container is None and obj is not None:
print "No container specified"
raise InfoSystemExit()
if account is None and part is None:
print "No target specified"
raise InfoSystemExit()
loc = "<type>"
if part and ring_name:
if "-" in ring_name and ring_name.startswith("object"):
loc = "objects-" + ring_name.split("-", 1)[1]
else:
loc = ring_name + "s"
if account and container and obj:
loc = "objects"
if "-" in ring_name and ring_name.startswith("object"):
policy_index = int(ring_name.rsplit("-", 1)[1])
loc = "objects-%d" % policy_index
if account and container and not obj:
loc = "containers"
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name="container")
else:
if ring_name != "container":
print "Attention! mismatch between ring and item detected!"
if account and not container and not obj:
loc = "accounts"
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name="account")
else:
if ring_name != "account":
print "Attention! mismatch between ring and item detected!"
print "\nAccount \t%s" % account
print "Container\t%s" % container
print "Object \t%s\n\n" % obj
print_ring_locations(ring, loc, account, container, obj, part, all_nodes, policy_index=policy_index)
def PUT(self, req):
version, acc, con, obj = split_path(req.path, 1, 4, True)
stor_policy = req.headers['storage_policy']
ring = POLICIES.get_object_ring(stor_policy, '/etc/swift')
#broker = self._get_metadata_broker()
#broker.initialize()
#Handle Container PUT
if not obj:
hsh = hash_path(acc, con)
part = ring.get_part(acc, con)
db_dir = storage_directory(swift.container.backend.DATADIR, part,
hsh)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
path = os.path.join(self.root + item, db_dir,
hsh + '.db')
#TODO: move kwargs
kwargs = {
'account': acc,
'container': con,
'logger': self.logger
}
md_broker = swift.container.backend.ContainerBroker(
path, **kwargs)
md = md_broker.get_info()
md.update(
(key, value)
for key, (value, timestamp
) in md_broker.metadata.iteritems()
if value != ''
and is_sys_or_user_meta('container', key))
sys_md = format_con_metadata(md)
user_md = format_custom_metadata(md)
if 'X-Container-Read' in req.headers:
sys_md[
'container_read_permissions'] = req.headers[
'X-Container-Read']
if 'X-Container-Write' in req.headers:
sys_md[
'container_write_permissions'] = req.headers[
'X-Container-Write']
#TODO: insert container_last_activity_time
#TODO: split meta user/sys
#TODO: insert meta
self.broker.insert_container_md([sys_md])
return
except DatabaseConnectionError as e:
self.logger.warn("DatabaseConnectionError: " +
e.path + "\n")
pass
except:
self.logger.warn("%s: %s\n" % (str(
sys.exc_info()[0]), str(sys.exc_info()[1])))
pass
#handle object PUT
else:
part = ring.get_part(acc, con, obj)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
df = self.diskfile_mgr.get_diskfile(
item, part, acc, con, obj, stor_policy)
md = df.read_metadata()
sys_md = format_obj_metadata(md)
#df._data_file is a direct path to the objects data
sys_md['object_location'] = df._data_file
user_md = format_custom_metadata(md)
#TODO: insert user meta and sys meta
self.broker.insert_object_md([sys_md])
except:
self.logger.warn("%s: %s\n" % (str(
sys.exc_info()[0]), str(sys.exc_info()[1])))
pass
return
def test_merge_storage_policy_index(self):
# generic split brain
self.brain.stop_primary_half()
self.brain.put_container()
self.brain.start_primary_half()
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'},
contents='VERIFY')
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assertTrue(
len(found_policy_indexes) > 1,
'primary nodes did not disagree about policy index %r' %
head_responses)
# find our object
orig_policy_index = None
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(self.account,
self.container_name,
self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node,
part,
self.account,
self.container_name,
self.object_name,
headers={
'X-Backend-Storage-Policy-Index': policy_index
})
except direct_client.ClientException as err:
continue
orig_policy_index = policy_index
break
if orig_policy_index is not None:
break
else:
self.fail('Unable to find /%s/%s/%s in %r' %
(self.account, self.container_name, self.object_name,
found_policy_indexes))
self.get_to_final_state()
Manager(['container-reconciler']).once()
# validate containers
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assertTrue(
len(found_policy_indexes) == 1,
'primary nodes disagree about policy index %r' % head_responses)
expected_policy_index = found_policy_indexes.pop()
self.assertNotEqual(orig_policy_index, expected_policy_index)
# validate object placement
orig_policy_ring = POLICIES.get_object_ring(orig_policy_index,
'/etc/swift')
for node in orig_policy_ring.devs:
try:
direct_client.direct_head_object(
node,
part,
self.account,
self.container_name,
self.object_name,
headers={
'X-Backend-Storage-Policy-Index': orig_policy_index
})
except direct_client.ClientException as err:
if err.http_status == HTTP_NOT_FOUND:
continue
raise
else:
self.fail('Found /%s/%s/%s in %s' %
(self.account, self.container_name, self.object_name,
orig_policy_index))
# verify that the object data read by external client is correct
headers, data = self._get_object_patiently(expected_policy_index)
self.assertEqual('VERIFY', data)
self.assertEqual('custom-meta', headers['x-object-meta-test'])
def test_reconcile_manifest(self):
info_url = "%s://%s/info" % (urlparse(self.url).scheme,
urlparse(self.url).netloc)
proxy_conn = client.http_connection(info_url)
cluster_info = client.get_capabilities(proxy_conn)
if 'slo' not in cluster_info:
raise SkipTest("SLO not enabled in proxy; "
"can't test manifest reconciliation")
# this test is not only testing a split brain scenario on
# multiple policies with mis-placed objects - it even writes out
# a static large object directly to the storage nodes while the
# objects are unavailably mis-placed from *behind* the proxy and
# doesn't know how to do that for EC_POLICY (clayg: why did you
# guys let me write a test that does this!?) - so we force
# wrong_policy (where the manifest gets written) to be one of
# any of your configured REPL_POLICY (we know you have one
# because this is a ReplProbeTest)
wrong_policy = random.choice(POLICIES_BY_TYPE[REPL_POLICY])
policy = random.choice([p for p in ENABLED_POLICIES
if p is not wrong_policy])
manifest_data = []
def write_part(i):
body = 'VERIFY%0.2d' % i + '\x00' * 1048576
part_name = 'manifest_part_%0.2d' % i
manifest_entry = {
"path": "/%s/%s" % (self.container_name, part_name),
"etag": md5(body).hexdigest(),
"size_bytes": len(body),
}
client.put_object(self.url, self.token, self.container_name,
part_name, contents=body)
manifest_data.append(manifest_entry)
# get an old container stashed
self.brain.stop_primary_half()
self.brain.put_container(int(policy))
self.brain.start_primary_half()
# write some parts
for i in range(10):
write_part(i)
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# write some more parts
for i in range(10, 20):
write_part(i)
# write manifest
with self.assertRaises(ClientException) as catcher:
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
# so as it works out, you can't really upload a multi-part
# manifest for objects that are currently misplaced - you have to
# wait until they're all available - which is about the same as
# some other failure that causes data to be unavailable to the
# proxy at the time of upload
self.assertEqual(catcher.exception.http_status, 400)
# but what the heck, we'll sneak one in just to see what happens...
direct_manifest_name = self.object_name + '-direct-test'
object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, direct_manifest_name)
container_part = self.container_ring.get_part(self.account,
self.container_name)
def translate_direct(data):
return {
'hash': data['etag'],
'bytes': data['size_bytes'],
'name': data['path'],
}
direct_manifest_data = map(translate_direct, manifest_data)
headers = {
'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n
in self.container_ring.devs),
'x-container-device': ','.join(n['device'] for n in
self.container_ring.devs),
'x-container-partition': container_part,
'X-Backend-Storage-Policy-Index': wrong_policy.idx,
'X-Static-Large-Object': 'True',
}
for node in nodes:
direct_client.direct_put_object(
node, part, self.account, self.container_name,
direct_manifest_name,
contents=utils.json.dumps(direct_manifest_data),
headers=headers)
break # one should do it...
self.brain.start_handoff_half()
self.get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# let's see how that direct upload worked out...
metadata, body = client.get_object(
self.url, self.token, self.container_name, direct_manifest_name,
query_string='multipart-manifest=get')
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
for i, entry in enumerate(utils.json.loads(body)):
for key in ('hash', 'bytes', 'name'):
self.assertEqual(entry[key], direct_manifest_data[i][key])
metadata, body = client.get_object(
self.url, self.token, self.container_name, direct_manifest_name)
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
self.assertEqual(body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576
for i in range(20)))
# and regular upload should work now too
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
def test_reconcile_manifest(self):
manifest_data = []
def write_part(i):
body = 'VERIFY%0.2d' % i + '\x00' * 1048576
part_name = 'manifest_part_%0.2d' % i
manifest_entry = {
"path": "/%s/%s" % (self.container_name, part_name),
"etag": md5(body).hexdigest(),
"size_bytes": len(body),
}
client.put_object(self.url,
self.token,
self.container_name,
part_name,
contents=body)
manifest_data.append(manifest_entry)
# get an old container stashed
self.brain.stop_primary_half()
policy = random.choice(ENABLED_POLICIES)
self.brain.put_container(policy.idx)
self.brain.start_primary_half()
# write some parts
for i in range(10):
write_part(i)
self.brain.stop_handoff_half()
wrong_policy = random.choice(
[p for p in ENABLED_POLICIES if p is not policy])
self.brain.put_container(wrong_policy.idx)
# write some more parts
for i in range(10, 20):
write_part(i)
# write manifest
try:
client.put_object(self.url,
self.token,
self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
except ClientException as err:
# so as it works out, you can't really upload a multi-part
# manifest for objects that are currently misplaced - you have to
# wait until they're all available - which is about the same as
# some other failure that causes data to be unavailable to the
# proxy at the time of upload
self.assertEqual(err.http_status, 400)
# but what the heck, we'll sneak one in just to see what happens...
direct_manifest_name = self.object_name + '-direct-test'
object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift')
part, nodes = object_ring.get_nodes(self.account, self.container_name,
direct_manifest_name)
container_part = self.container_ring.get_part(self.account,
self.container_name)
def translate_direct(data):
return {
'hash': data['etag'],
'bytes': data['size_bytes'],
'name': data['path'],
}
direct_manifest_data = map(translate_direct, manifest_data)
headers = {
'x-container-host':
','.join('%s:%s' % (n['ip'], n['port'])
for n in self.container_ring.devs),
'x-container-device':
','.join(n['device'] for n in self.container_ring.devs),
'x-container-partition':
container_part,
'X-Backend-Storage-Policy-Index':
wrong_policy.idx,
'X-Static-Large-Object':
'True',
}
for node in nodes:
direct_client.direct_put_object(
node,
part,
self.account,
self.container_name,
direct_manifest_name,
contents=utils.json.dumps(direct_manifest_data),
headers=headers)
break # one should do it...
self.brain.start_handoff_half()
get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# let's see how that direct upload worked out...
metadata, body = client.get_object(
self.url,
self.token,
self.container_name,
direct_manifest_name,
query_string='multipart-manifest=get')
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
for i, entry in enumerate(utils.json.loads(body)):
for key in ('hash', 'bytes', 'name'):
self.assertEquals(entry[key], direct_manifest_data[i][key])
metadata, body = client.get_object(self.url, self.token,
self.container_name,
direct_manifest_name)
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
self.assertEqual(
body,
''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20)))
# and regular upload should work now too
client.put_object(self.url,
self.token,
self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
metadata = client.head_object(self.url, self.token,
self.container_name, self.object_name)
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
def test_merge_storage_policy_index(self):
# generic split brain
self.brain.stop_primary_half()
self.brain.put_container()
self.brain.start_primary_half()
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.put_object()
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assert_(
len(found_policy_indexes) > 1,
'primary nodes did not disagree about policy index %r' %
head_responses)
# find our object
orig_policy_index = None
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(self.account,
self.container_name,
self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node,
part,
self.account,
self.container_name,
self.object_name,
headers={
'X-Backend-Storage-Policy-Index': policy_index
})
except direct_client.ClientException as err:
continue
orig_policy_index = policy_index
break
if orig_policy_index is not None:
break
else:
self.fail('Unable to find /%s/%s/%s in %r' %
(self.account, self.container_name, self.object_name,
found_policy_indexes))
get_to_final_state()
Manager(['container-reconciler']).once()
# validate containers
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assert_(
len(found_policy_indexes) == 1,
'primary nodes disagree about policy index %r' % head_responses)
expected_policy_index = found_policy_indexes.pop()
self.assertNotEqual(orig_policy_index, expected_policy_index)
# validate object placement
orig_policy_ring = POLICIES.get_object_ring(orig_policy_index,
'/etc/swift')
for node in orig_policy_ring.devs:
try:
direct_client.direct_head_object(
node,
part,
self.account,
self.container_name,
self.object_name,
headers={
'X-Backend-Storage-Policy-Index': orig_policy_index
})
except direct_client.ClientException as err:
if err.http_status == HTTP_NOT_FOUND:
continue
raise
else:
self.fail('Found /%s/%s/%s in %s' %
(self.account, self.container_name, self.object_name,
orig_policy_index))
# use proxy to access object (bad container info might be cached...)
timeout = time.time() + TIMEOUT
while time.time() < timeout:
try:
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
time.sleep(1)
else:
break
else:
self.fail('could not HEAD /%s/%s/%s/ from policy %s '
'after %s seconds.' %
(self.account, self.container_name, self.object_name,
expected_policy_index, TIMEOUT))
def POST(self, req):
version, acc, con, obj = split_path(req.path, 1, 4, True)
stor_policy = req.headers['storage_policy']
ring = POLICIES.get_object_ring(stor_policy, '/etc/swift')
if not con and not obj:
meta_type = 'account'
kwargs = {'account':acc, 'logger':self.logger}
data_dir = swift.account.backend.DATADIR
hsh = hash_path(acc)
part = ring.get_part(acc)
db_dir = storage_directory(data_dir, part, hsh)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
path = os.path.join(self.root + item, db_dir, hsh + '.db')
broker = swift.account.backend.AccountBroker(path, **kwargs)
md = broker.get_info()
md.update(
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
if value != '' and is_sys_or_user_meta(meta_type, key))
sys_md = format_acc_metadata(md)
user_md = format_custom_metadata(md)
#TODO: call overwrite_account_metadata
#TODO: call overwrite_custom_metadata
return
except:
self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1])))
pass
#Handle Container POST
elif not obj:
meta_type = 'container'
kwargs = {'account':acc, 'container':con, 'logger':self.logger}
data_dir = swift.container.backend.DATADIR
try:
hsh = hash_path(acc, con)
part = ring.get_part(acc, con)
db_dir = storage_directory(data_dir, part, hsh)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
path = os.path.join(self.root + item, db_dir, hsh + '.db')
broker = swift.container.backend.ContainerBroker(path, **kwargs)
md = broker.get_info()
md.update(
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
if value != '' and is_sys_or_user_meta('container', key))
sys_md = format_con_metadata(md)
user_md = format_custom_metadata(md)
if 'X-Container-Read' in req.headers:
sys_md['container_read_permissions'] = req.headers['X-Container-Read']
if 'X-Container-Write' in req.headers:
sys_md['container_write_permissions'] = req.headers['X-Container-Write']
#TODO: call overwrite_container_metadata
#TODO: call overwrite_custom_metadata
return
except DatabaseConnectionError as e:
self.logger.warn("DatabaseConnectionError: " + e.path + "\n")
pass
except:
self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1])))
pass
else:
part = ring.get_part(acc, con, obj)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
df = self.diskfile_mgr.get_diskfile(item, part, acc, con, obj, stor_policy)
md = df.read_metadata()
sys_md = format_obj_metadata(md)
user_md = format_custom_metadata(md)
#TODO: call overwrite_object_metadata
#TODO: call overwrite_custom_metadata
except:
self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1])))
pass
return
def test_sync(self):
all_objects = []
# upload some containers
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
body = 'test-body'
client.put_object(self.url, self.token, container, obj, body)
all_objects.append((policy, container, obj))
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
# run the reaper
Manager(['account-reaper']).once()
for policy, container, obj in all_objects:
# verify that any container deletes were at same timestamp
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
delete_times = set()
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEqual(err.http_status, 404)
delete_time = err.http_headers.get(
'X-Backend-DELETE-Timestamp')
# 'X-Backend-DELETE-Timestamp' confirms it was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
# Container replicas may not yet be deleted if we have a
# policy with object replicas < container replicas, so
# ignore successful HEAD. We'll check for all replicas to
# be deleted again after running the replicators.
pass
self.assertEqual(1, len(delete_times), delete_times)
# verify that all object deletes were at same timestamp
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
headers = {'X-Backend-Storage-Policy-Index': int(policy)}
delete_times = set()
for node in nodes:
try:
direct_get_object(node, part, self.account,
container, obj, headers=headers)
except ClientException as err:
self.assertEqual(err.http_status, 404)
delete_time = err.http_headers.get('X-Backend-Timestamp')
# 'X-Backend-Timestamp' confirms obj was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
self.assertEqual(1, len(delete_times))
# run replicators and updaters
self.get_to_final_state()
for policy, container, obj in all_objects:
# verify that ALL container replicas are now deleted
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
delete_times = set()
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEqual(err.http_status, 404)
delete_time = err.http_headers.get(
'X-Backend-DELETE-Timestamp')
# 'X-Backend-DELETE-Timestamp' confirms it was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s on %r' %
(self.account, container, cnode))
# sanity check that object state is still consistent...
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
headers = {'X-Backend-Storage-Policy-Index': int(policy)}
delete_times = set()
for node in nodes:
try:
direct_get_object(node, part, self.account,
container, obj, headers=headers)
except ClientException as err:
self.assertEqual(err.http_status, 404)
delete_time = err.http_headers.get('X-Backend-Timestamp')
# 'X-Backend-Timestamp' confirms obj was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
self.assertEqual(1, len(delete_times))
def PUT(self, req):
version, acc, con, obj = split_path(req.path, 1, 4, True)
stor_policy = req.headers['storage_policy']
ring = POLICIES.get_object_ring(stor_policy, '/etc/swift')
#broker = self._get_metadata_broker()
#broker.initialize()
#Handle Container PUT
if not obj:
hsh = hash_path(acc, con)
part = ring.get_part(acc, con)
db_dir = storage_directory(swift.container.backend.DATADIR, part, hsh)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
path = os.path.join(self.root + item, db_dir, hsh + '.db')
#TODO: move kwargs
kwargs = {'account':acc, 'container':con, 'logger':self.logger}
md_broker= swift.container.backend.ContainerBroker(path, **kwargs)
md = md_broker.get_info()
md.update(
(key, value)
for key, (value, timestamp) in md_broker.metadata.iteritems()
if value != '' and is_sys_or_user_meta('container', key))
sys_md = format_con_metadata(md)
user_md = format_custom_metadata(md)
if 'X-Container-Read' in req.headers:
sys_md['container_read_permissions'] = req.headers['X-Container-Read']
if 'X-Container-Write' in req.headers:
sys_md['container_write_permissions'] = req.headers['X-Container-Write']
#TODO: insert container_last_activity_time
#TODO: split meta user/sys
#TODO: insert meta
self.broker.insert_container_md([sys_md])
return
except DatabaseConnectionError as e:
self.logger.warn("DatabaseConnectionError: " + e.path + "\n")
pass
except:
self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1])))
pass
#handle object PUT
else:
part = ring.get_part(acc, con, obj)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
df = self.diskfile_mgr.get_diskfile(item, part, acc, con, obj, stor_policy)
md = df.read_metadata()
sys_md = format_obj_metadata(md)
#df._data_file is a direct path to the objects data
sys_md['object_location'] = df._data_file
user_md = format_custom_metadata(md)
#TODO: insert user meta and sys meta
self.broker.insert_object_md([sys_md])
except:
self.logger.warn("%s: %s\n"%(str(sys.exc_info()[0]),str(sys.exc_info()[1])))
pass
return
def _get_devices(self):
return set([
d['device'] for policy in POLICIES for d in
POLICIES.get_object_ring(int(policy), self.swift_dir).devs
if d
])
def POST(self, req):
version, acc, con, obj = split_path(req.path, 1, 4, True)
stor_policy = req.headers['storage_policy']
ring = POLICIES.get_object_ring(stor_policy, '/etc/swift')
if not con and not obj:
meta_type = 'account'
kwargs = {'account': acc, 'logger': self.logger}
data_dir = swift.account.backend.DATADIR
hsh = hash_path(acc)
part = ring.get_part(acc)
db_dir = storage_directory(data_dir, part, hsh)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
path = os.path.join(self.root + item, db_dir,
hsh + '.db')
broker = swift.account.backend.AccountBroker(
path, **kwargs)
md = broker.get_info()
md.update((key, value) for key, (
value,
timestamp) in broker.metadata.iteritems()
if value != ''
and is_sys_or_user_meta(meta_type, key))
sys_md = format_acc_metadata(md)
user_md = format_custom_metadata(md)
#TODO: call overwrite_account_metadata
#TODO: call overwrite_custom_metadata
return
except:
self.logger.warn("%s: %s\n" % (str(
sys.exc_info()[0]), str(sys.exc_info()[1])))
pass
#Handle Container POST
elif not obj:
meta_type = 'container'
kwargs = {'account': acc, 'container': con, 'logger': self.logger}
data_dir = swift.container.backend.DATADIR
try:
hsh = hash_path(acc, con)
part = ring.get_part(acc, con)
db_dir = storage_directory(data_dir, part, hsh)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
path = os.path.join(self.root + item, db_dir,
hsh + '.db')
broker = swift.container.backend.ContainerBroker(
path, **kwargs)
md = broker.get_info()
md.update(
(key, value)
for key, (value, timestamp
) in broker.metadata.iteritems()
if value != ''
and is_sys_or_user_meta('container', key))
sys_md = format_con_metadata(md)
user_md = format_custom_metadata(md)
if 'X-Container-Read' in req.headers:
sys_md[
'container_read_permissions'] = req.headers[
'X-Container-Read']
if 'X-Container-Write' in req.headers:
sys_md[
'container_write_permissions'] = req.headers[
'X-Container-Write']
#TODO: call overwrite_container_metadata
#TODO: call overwrite_custom_metadata
return
except DatabaseConnectionError as e:
self.logger.warn("DatabaseConnectionError: " +
e.path + "\n")
pass
except:
self.logger.warn(
"%s: %s\n" %
(str(sys.exc_info()[0]), str(sys.exc_info()[1])))
pass
else:
part = ring.get_part(acc, con, obj)
nodes = ring.get_part_nodes(part)
for node in nodes:
for item in self.devicelist:
if node['device'] in item:
try:
df = self.diskfile_mgr.get_diskfile(
item, part, acc, con, obj, stor_policy)
md = df.read_metadata()
sys_md = format_obj_metadata(md)
user_md = format_custom_metadata(md)
#TODO: call overwrite_object_metadata
#TODO: call overwrite_custom_metadata
except:
self.logger.warn("%s: %s\n" % (str(
sys.exc_info()[0]), str(sys.exc_info()[1])))
pass
return
def test_reconcile_delete(self):
# generic split brain
self.brain.stop_primary_half()
self.brain.put_container()
self.brain.put_object()
self.brain.start_primary_half()
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.delete_object()
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assertTrue(
len(found_policy_indexes) > 1,
'primary nodes did not disagree about policy index %r' %
head_responses)
# find our object
orig_policy_index = ts_policy_index = None
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(self.account,
self.container_name,
self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node,
part,
self.account,
self.container_name,
self.object_name,
headers={
'X-Backend-Storage-Policy-Index': policy_index
})
except direct_client.ClientException as err:
if 'x-backend-timestamp' in err.http_headers:
ts_policy_index = policy_index
break
else:
orig_policy_index = policy_index
break
if not orig_policy_index:
self.fail('Unable to find /%s/%s/%s in %r' %
(self.account, self.container_name, self.object_name,
found_policy_indexes))
if not ts_policy_index:
self.fail('Unable to find tombstone /%s/%s/%s in %r' %
(self.account, self.container_name, self.object_name,
found_policy_indexes))
self.get_to_final_state()
Manager(['container-reconciler']).once()
# validate containers
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
new_found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for node,
metadata in head_responses)
self.assertTrue(
len(new_found_policy_indexes) == 1,
'primary nodes disagree about policy index %r' % dict(
(node['port'], metadata['X-Backend-Storage-Policy-Index'])
for node, metadata in head_responses))
expected_policy_index = new_found_policy_indexes.pop()
self.assertEqual(orig_policy_index, expected_policy_index)
# validate object fully deleted
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(self.account,
self.container_name,
self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node,
part,
self.account,
self.container_name,
self.object_name,
headers={
'X-Backend-Storage-Policy-Index': policy_index
})
except direct_client.ClientException as err:
if err.http_status == HTTP_NOT_FOUND:
continue
else:
self.fail('Found /%s/%s/%s in %s on %s' %
(self.account, self.container_name,
self.object_name, orig_policy_index, node))
def test_sync(self):
all_objects = []
# upload some containers
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url,
self.token,
container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
body = 'test-body'
client.put_object(self.url, self.token, container, obj, body)
all_objects.append((policy, container, obj))
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
# run the reaper
Manager(['account-reaper']).once()
for policy, container, obj in all_objects:
# verify that any container deletes were at same timestamp
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
delete_times = set()
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEquals(err.http_status, 404)
delete_time = err.http_headers.get(
'X-Backend-DELETE-Timestamp')
# 'X-Backend-DELETE-Timestamp' confirms it was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
# Container replicas may not yet be deleted if we have a
# policy with object replicas < container replicas, so
# ignore successful HEAD. We'll check for all replicas to
# be deleted again after running the replicators.
pass
self.assertEqual(1, len(delete_times), delete_times)
# verify that all object deletes were at same timestamp
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
headers = {'X-Backend-Storage-Policy-Index': int(policy)}
delete_times = set()
for node in nodes:
try:
direct_get_object(node,
part,
self.account,
container,
obj,
headers=headers)
except ClientException as err:
self.assertEquals(err.http_status, 404)
delete_time = err.http_headers.get('X-Backend-Timestamp')
# 'X-Backend-Timestamp' confirms obj was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
self.assertEqual(1, len(delete_times))
# run replicators and updaters
self.get_to_final_state()
for policy, container, obj in all_objects:
# verify that ALL container replicas are now deleted
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
delete_times = set()
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEquals(err.http_status, 404)
delete_time = err.http_headers.get(
'X-Backend-DELETE-Timestamp')
# 'X-Backend-DELETE-Timestamp' confirms it was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s on %r' %
(self.account, container, cnode))
# sanity check that object state is still consistent...
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
headers = {'X-Backend-Storage-Policy-Index': int(policy)}
delete_times = set()
for node in nodes:
try:
direct_get_object(node,
part,
self.account,
container,
obj,
headers=headers)
except ClientException as err:
self.assertEquals(err.http_status, 404)
delete_time = err.http_headers.get('X-Backend-Timestamp')
# 'X-Backend-Timestamp' confirms obj was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
self.assertEqual(1, len(delete_times))
def test_reconcile_manifest(self):
info_url = "%s://%s/info" % (urlparse(
self.url).scheme, urlparse(self.url).netloc)
proxy_conn = client.http_connection(info_url)
cluster_info = client.get_capabilities(proxy_conn)
if 'slo' not in cluster_info:
raise SkipTest("SLO not enabled in proxy; "
"can't test manifest reconciliation")
# this test is not only testing a split brain scenario on
# multiple policies with mis-placed objects - it even writes out
# a static large object directly to the storage nodes while the
# objects are unavailably mis-placed from *behind* the proxy and
# doesn't know how to do that for EC_POLICY (clayg: why did you
# guys let me write a test that does this!?) - so we force
# wrong_policy (where the manifest gets written) to be one of
# any of your configured REPL_POLICY (we know you have one
# because this is a ReplProbeTest)
wrong_policy = random.choice(POLICIES_BY_TYPE[REPL_POLICY])
policy = random.choice(
[p for p in ENABLED_POLICIES if p is not wrong_policy])
manifest_data = []
def write_part(i):
body = 'VERIFY%0.2d' % i + '\x00' * 1048576
part_name = 'manifest_part_%0.2d' % i
manifest_entry = {
"path": "/%s/%s" % (self.container_name, part_name),
"etag": md5(body).hexdigest(),
"size_bytes": len(body),
}
client.put_object(self.url,
self.token,
self.container_name,
part_name,
contents=body)
manifest_data.append(manifest_entry)
# get an old container stashed
self.brain.stop_primary_half()
self.brain.put_container(int(policy))
self.brain.start_primary_half()
# write some parts
for i in range(10):
write_part(i)
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# write some more parts
for i in range(10, 20):
write_part(i)
# write manifest
with self.assertRaises(ClientException) as catcher:
client.put_object(self.url,
self.token,
self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
# so as it works out, you can't really upload a multi-part
# manifest for objects that are currently misplaced - you have to
# wait until they're all available - which is about the same as
# some other failure that causes data to be unavailable to the
# proxy at the time of upload
self.assertEqual(catcher.exception.http_status, 400)
# but what the heck, we'll sneak one in just to see what happens...
direct_manifest_name = self.object_name + '-direct-test'
object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift')
part, nodes = object_ring.get_nodes(self.account, self.container_name,
direct_manifest_name)
container_part = self.container_ring.get_part(self.account,
self.container_name)
def translate_direct(data):
return {
'hash': data['etag'],
'bytes': data['size_bytes'],
'name': data['path'],
}
direct_manifest_data = map(translate_direct, manifest_data)
headers = {
'x-container-host':
','.join('%s:%s' % (n['ip'], n['port'])
for n in self.container_ring.devs),
'x-container-device':
','.join(n['device'] for n in self.container_ring.devs),
'x-container-partition':
container_part,
'X-Backend-Storage-Policy-Index':
wrong_policy.idx,
'X-Static-Large-Object':
'True',
}
for node in nodes:
direct_client.direct_put_object(
node,
part,
self.account,
self.container_name,
direct_manifest_name,
contents=utils.json.dumps(direct_manifest_data),
headers=headers)
break # one should do it...
self.brain.start_handoff_half()
self.get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# let's see how that direct upload worked out...
metadata, body = client.get_object(
self.url,
self.token,
self.container_name,
direct_manifest_name,
query_string='multipart-manifest=get')
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
for i, entry in enumerate(utils.json.loads(body)):
for key in ('hash', 'bytes', 'name'):
self.assertEqual(entry[key], direct_manifest_data[i][key])
metadata, body = client.get_object(self.url, self.token,
self.container_name,
direct_manifest_name)
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
self.assertEqual(
body,
''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20)))
# and regular upload should work now too
client.put_object(self.url,
self.token,
self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
metadata = client.head_object(self.url, self.token,
self.container_name, self.object_name)
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
policy_name=''):
"""
Display information about an object read from the datafile.
Optionally verify the datafile content matches the ETag metadata.
:param datafile: path on disk to object file
:param check_etag: boolean, will read datafile content and verify
computed checksum matches value stored in
metadata.
:param swift_dir: the path on disk to rings
:param policy_name: optionally the name to use when finding the ring
"""
if not os.path.exists(datafile):
print "Data file doesn't exist"
raise InfoSystemExit()
if not datafile.startswith(('/', './')):
datafile = './' + datafile
policy_index = None
ring = None
datadir = DATADIR_BASE
# try to extract policy index from datafile disk path
policy_index = int(extract_policy(datafile) or POLICIES.legacy)
try:
if policy_index:
datadir += '-' + str(policy_index)
ring = Ring(swift_dir, ring_name='object-' + str(policy_index))
elif policy_index == 0:
ring = Ring(swift_dir, ring_name='object')
except IOError:
# no such ring
pass
if policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
policy_index_for_name = policy.idx
if (policy_index is not None and
policy_index_for_name is not None and
policy_index != policy_index_for_name):
print 'Attention: Ring does not match policy!'
print 'Double check your policy name!'
if not ring and policy_index_for_name:
ring = POLICIES.get_object_ring(policy_index_for_name,
swift_dir)
datadir = get_data_dir(policy_index_for_name)
with open(datafile, 'rb') as fp:
try:
metadata = read_metadata(fp)
except EOFError:
print "Invalid metadata"
raise InfoSystemExit()
etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length', '')
path = metadata.get('name', '')
print_obj_metadata(metadata)
# Optional integrity check; it's useful, but slow.
file_len = None
if check_etag:
h = md5()
file_len = 0
while True:
data = fp.read(64 * 1024)
if not data:
break
h.update(data)
file_len += len(data)
h = h.hexdigest()
if etag:
if h == etag:
print 'ETag: %s (valid)' % etag
else:
print ("ETag: %s doesn't match file hash of %s!" %
(etag, h))
else:
print 'ETag: Not found in metadata'
else:
print 'ETag: %s (not checked)' % etag
file_len = os.fstat(fp.fileno()).st_size
if length:
if file_len == int(length):
print 'Content-Length: %s (valid)' % length
else:
print ("Content-Length: %s doesn't match file length of %s"
% (length, file_len))
else:
print 'Content-Length: Not found in metadata'
account, container, obj = path.split('/', 3)[1:]
if ring:
print_ring_locations(ring, datadir, account, container, obj,
policy_index=policy_index)
def test_reconcile_delete(self):
# generic split brain
self.brain.stop_primary_half()
self.brain.put_container()
self.brain.put_object()
self.brain.start_primary_half()
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.delete_object()
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assertTrue(
len(found_policy_indexes) > 1,
'primary nodes did not disagree about policy index %r' %
head_responses)
# find our object
orig_policy_index = ts_policy_index = None
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index':
policy_index})
except direct_client.ClientException as err:
if 'x-backend-timestamp' in err.http_headers:
ts_policy_index = policy_index
break
else:
orig_policy_index = policy_index
break
if not orig_policy_index:
self.fail('Unable to find /%s/%s/%s in %r' % (
self.account, self.container_name, self.object_name,
found_policy_indexes))
if not ts_policy_index:
self.fail('Unable to find tombstone /%s/%s/%s in %r' % (
self.account, self.container_name, self.object_name,
found_policy_indexes))
self.get_to_final_state()
Manager(['container-reconciler']).once()
# validate containers
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
new_found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for node,
metadata in head_responses)
self.assertTrue(len(new_found_policy_indexes) == 1,
'primary nodes disagree about policy index %r' %
dict((node['port'],
metadata['X-Backend-Storage-Policy-Index'])
for node, metadata in head_responses))
expected_policy_index = new_found_policy_indexes.pop()
self.assertEqual(orig_policy_index, expected_policy_index)
# validate object fully deleted
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index':
policy_index})
except direct_client.ClientException as err:
if err.http_status == HTTP_NOT_FOUND:
continue
else:
self.fail('Found /%s/%s/%s in %s on %s' % (
self.account, self.container_name, self.object_name,
orig_policy_index, node))
def get_object_ring(self, storage_policy_index):
return POLICIES.get_object_ring(storage_policy_index, self.swift_dir)
def test_merge_storage_policy_index(self):
# generic split brain
self.brain.stop_primary_half()
self.brain.put_container()
self.brain.start_primary_half()
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'},
contents='VERIFY')
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assertTrue(
len(found_policy_indexes) > 1,
'primary nodes did not disagree about policy index %r' %
head_responses)
# find our object
orig_policy_index = None
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index':
policy_index})
except direct_client.ClientException as err:
continue
orig_policy_index = policy_index
break
if orig_policy_index is not None:
break
else:
self.fail('Unable to find /%s/%s/%s in %r' % (
self.account, self.container_name, self.object_name,
found_policy_indexes))
self.get_to_final_state()
Manager(['container-reconciler']).once()
# validate containers
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
self.assertTrue(len(found_policy_indexes) == 1,
'primary nodes disagree about policy index %r' %
head_responses)
expected_policy_index = found_policy_indexes.pop()
self.assertNotEqual(orig_policy_index, expected_policy_index)
# validate object placement
orig_policy_ring = POLICIES.get_object_ring(orig_policy_index,
'/etc/swift')
for node in orig_policy_ring.devs:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name, headers={
'X-Backend-Storage-Policy-Index': orig_policy_index})
except direct_client.ClientException as err:
if err.http_status == HTTP_NOT_FOUND:
continue
raise
else:
self.fail('Found /%s/%s/%s in %s' % (
self.account, self.container_name, self.object_name,
orig_policy_index))
# verify that the object data read by external client is correct
headers, data = self._get_object_patiently(expected_policy_index)
self.assertEqual('VERIFY', data)
self.assertEqual('custom-meta', headers['x-object-meta-test'])
def print_item_locations(ring,
ring_name=None,
account=None,
container=None,
obj=None,
**kwargs):
"""
Display placement information for an item based on ring lookup.
If a ring is provided it always takes precedence, but warnings will be
emitted if it doesn't match other optional arguments like the policy_name
or ring_name.
If no ring is provided the ring_name and/or policy_name will be used to
lookup the ring.
:param ring: a ring instance
:param ring_name: server type, or storage policy ring name if object ring
:param account: account name
:param container: container name
:param obj: object name
:param partition: part number for non path lookups
:param policy_name: name of storage policy to use to lookup the ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
"""
policy_name = kwargs.get('policy_name', None)
part = kwargs.get('partition', None)
all_nodes = kwargs.get('all', False)
swift_dir = kwargs.get('swift_dir', '/etc/swift')
if ring and policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
if ring_name != policy.ring_name:
print('Warning: mismatch between ring and policy name!')
else:
print('Warning: Policy %s is not valid' % policy_name)
policy_index = None
if ring is None and (obj or part):
if not policy_name:
print('Need a ring or policy')
raise InfoSystemExit()
policy = POLICIES.get_by_name(policy_name)
if not policy:
print('No policy named %r' % policy_name)
raise InfoSystemExit()
policy_index = int(policy)
ring = POLICIES.get_object_ring(policy_index, swift_dir)
ring_name = (POLICIES.get_by_name(policy_name)).ring_name
if account is None and (container is not None or obj is not None):
print('No account specified')
raise InfoSystemExit()
if container is None and obj is not None:
print('No container specified')
raise InfoSystemExit()
if account is None and part is None:
print('No target specified')
raise InfoSystemExit()
loc = '<type>'
if part and ring_name:
if '-' in ring_name and ring_name.startswith('object'):
loc = 'objects-' + ring_name.split('-', 1)[1]
else:
loc = ring_name + 's'
if account and container and obj:
loc = 'objects'
if '-' in ring_name and ring_name.startswith('object'):
policy_index = int(ring_name.rsplit('-', 1)[1])
loc = 'objects-%d' % policy_index
if account and container and not obj:
loc = 'containers'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='container')
else:
if ring_name != 'container':
print('Warning: account/container specified ' +
'but ring not named "container"')
if account and not container and not obj:
loc = 'accounts'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='account')
else:
if ring_name != 'account':
print('Warning: account specified ' +
'but ring not named "account"')
print('\nAccount \t%s' % account)
print('Container\t%s' % container)
print('Object \t%s\n\n' % obj)
print_ring_locations(ring,
loc,
account,
container,
obj,
part,
all_nodes,
policy_index=policy_index)
def print_obj(datafile,
check_etag=True,
swift_dir='/etc/swift',
policy_name=''):
"""
Display information about an object read from the datafile.
Optionally verify the datafile content matches the ETag metadata.
:param datafile: path on disk to object file
:param check_etag: boolean, will read datafile content and verify
computed checksum matches value stored in
metadata.
:param swift_dir: the path on disk to rings
:param policy_name: optionally the name to use when finding the ring
"""
if not os.path.exists(datafile):
print("Data file doesn't exist")
raise InfoSystemExit()
if not datafile.startswith(('/', './')):
datafile = './' + datafile
policy_index = None
ring = None
datadir = DATADIR_BASE
# try to extract policy index from datafile disk path
fullpath = os.path.abspath(datafile)
policy_index = int(extract_policy(fullpath) or POLICIES.legacy)
try:
if policy_index:
datadir += '-' + str(policy_index)
ring = Ring(swift_dir, ring_name='object-' + str(policy_index))
elif policy_index == 0:
ring = Ring(swift_dir, ring_name='object')
except IOError:
# no such ring
pass
if policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
policy_index_for_name = policy.idx
if (policy_index is not None and policy_index_for_name is not None
and policy_index != policy_index_for_name):
print('Warning: Ring does not match policy!')
print('Double check your policy name!')
if not ring and policy_index_for_name:
ring = POLICIES.get_object_ring(policy_index_for_name,
swift_dir)
datadir = get_data_dir(policy_index_for_name)
with open(datafile, 'rb') as fp:
try:
metadata = read_metadata(fp)
except EOFError:
print("Invalid metadata")
raise InfoSystemExit()
etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length', '')
path = metadata.get('name', '')
print_obj_metadata(metadata)
# Optional integrity check; it's useful, but slow.
file_len = None
if check_etag:
h = md5()
file_len = 0
while True:
data = fp.read(64 * 1024)
if not data:
break
h.update(data)
file_len += len(data)
h = h.hexdigest()
if etag:
if h == etag:
print('ETag: %s (valid)' % etag)
else:
print("ETag: %s doesn't match file hash of %s!" %
(etag, h))
else:
print('ETag: Not found in metadata')
else:
print('ETag: %s (not checked)' % etag)
file_len = os.fstat(fp.fileno()).st_size
if length:
if file_len == int(length):
print('Content-Length: %s (valid)' % length)
else:
print("Content-Length: %s doesn't match file length of %s" %
(length, file_len))
else:
print('Content-Length: Not found in metadata')
account, container, obj = path.split('/', 3)[1:]
if ring:
print_ring_locations(ring,
datadir,
account,
container,
obj,
policy_index=policy_index)
def test_reconcile_symlink(self):
if 'symlink' not in self.cluster_info:
raise unittest.SkipTest(
"Symlink not enabled in proxy; can't test "
"symlink reconciliation")
wrong_policy = random.choice(ENABLED_POLICIES)
policy = random.choice([p for p in ENABLED_POLICIES
if p is not wrong_policy])
# get an old container stashed
self.brain.stop_primary_half()
self.brain.put_container(int(policy))
self.brain.start_primary_half()
# write some target data
client.put_object(self.url, self.token, self.container_name, 'target',
contents='this is the target data')
# write the symlink
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
client.put_object(
self.url, self.token, self.container_name, 'symlink',
headers={
'X-Symlink-Target': '%s/target' % self.container_name,
'Content-Type': 'application/symlink',
})
# at this point we have a broken symlink (the container_info has the
# proxy looking for the target in the wrong policy)
with self.assertRaises(ClientException) as ctx:
client.get_object(self.url, self.token, self.container_name,
'symlink')
self.assertEqual(ctx.exception.http_status, 404)
# of course the symlink itself is fine
metadata, body = client.get_object(self.url, self.token,
self.container_name, 'symlink',
query_string='symlink=get')
self.assertEqual(metadata['x-symlink-target'],
'%s/target' % self.container_name)
self.assertEqual(metadata['content-type'], 'application/symlink')
self.assertEqual(body, '')
# ... although in the wrong policy
object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, 'symlink')
for node in nodes:
metadata = direct_client.direct_head_object(
node, part, self.account, self.container_name, 'symlink',
headers={'X-Backend-Storage-Policy-Index': int(wrong_policy)})
self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'],
'%s/target' % self.container_name)
# let the reconciler run
self.brain.start_handoff_half()
self.get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# now the symlink works
metadata, body = client.get_object(self.url, self.token,
self.container_name, 'symlink')
self.assertEqual(body, 'this is the target data')
# and it's in the correct policy
object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, 'symlink')
for node in nodes:
metadata = direct_client.direct_head_object(
node, part, self.account, self.container_name, 'symlink',
headers={'X-Backend-Storage-Policy-Index': int(policy)})
self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'],
'%s/target' % self.container_name)
def print_item_locations(ring, ring_name=None, account=None, container=None,
obj=None, **kwargs):
"""
Display placement information for an item based on ring lookup.
If a ring is provided it always takes precedence, but warnings will be
emitted if it doesn't match other optional arguments like the policy_name
or ring_name.
If no ring is provided the ring_name and/or policy_name will be used to
lookup the ring.
:param ring: a ring instance
:param ring_name: server type, or storage policy ring name if object ring
:param account: account name
:param container: container name
:param obj: object name
:param partition: part number for non path lookups
:param policy_name: name of storage policy to use to lookup the ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
"""
policy_name = kwargs.get('policy_name', None)
part = kwargs.get('partition', None)
all_nodes = kwargs.get('all', False)
swift_dir = kwargs.get('swift_dir', '/etc/swift')
if ring and policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
if ring_name != policy.ring_name:
print 'Attention! mismatch between ring and policy detected!'
else:
print 'Attention! Policy %s is not valid' % policy_name
policy_index = None
if ring is None and (obj or part):
if not policy_name:
print 'Need a ring or policy'
raise InfoSystemExit()
policy = POLICIES.get_by_name(policy_name)
if not policy:
print 'No policy named %r' % policy_name
raise InfoSystemExit()
policy_index = int(policy)
ring = POLICIES.get_object_ring(policy_index, swift_dir)
ring_name = (POLICIES.get_by_name(policy_name)).ring_name
if account is None and (container is not None or obj is not None):
print 'No account specified'
raise InfoSystemExit()
if container is None and obj is not None:
print 'No container specified'
raise InfoSystemExit()
if account is None and part is None:
print 'No target specified'
raise InfoSystemExit()
loc = '<type>'
if part and ring_name:
if '-' in ring_name and ring_name.startswith('object'):
loc = 'objects-' + ring_name.split('-', 1)[1]
else:
loc = ring_name + 's'
if account and container and obj:
loc = 'objects'
if '-' in ring_name and ring_name.startswith('object'):
policy_index = int(ring_name.rsplit('-', 1)[1])
loc = 'objects-%d' % policy_index
if account and container and not obj:
loc = 'containers'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='container')
else:
if ring_name != 'container':
print 'Attention! mismatch between ring and item detected!'
if account and not container and not obj:
loc = 'accounts'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='account')
else:
if ring_name != 'account':
print 'Attention! mismatch between ring and item detected!'
print '\nAccount \t%s' % account
print 'Container\t%s' % container
print 'Object \t%s\n\n' % obj
print_ring_locations(ring, loc, account, container, obj, part, all_nodes,
policy_index=policy_index)
def _get_devices(self):
return set([
d['device'] for policy in POLICIES
for d in POLICIES.get_object_ring(int(policy), self.swift_dir).devs
])
