def test_main(self):
# Create container
# Kill container servers excepting two of the primaries
# Create container/obj
# Restart other primary server
# Assert it does not know about container/obj
# Run the object-updaters
# Assert the other primary server now knows about container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnode['port'], self.port2server, self.pids)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
start_server(cnode['port'], self.port2server, self.pids)
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
processes = []
for node in xrange(1, 5):
processes.append(Popen(['swift-object-updater',
'/etc/swift/object-server/%d.conf' % node,
'once']))
for process in processes:
process.wait()
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assert_(obj in objs)
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()
# check the re-started container server has update with override values
obj = direct_client.direct_get_container(cnodes[0], cpart,
self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server, self.pids)
kill_server((cnode['ip'], cnode['port']),
self.ipport2server, self.pids)
# Create container/obj
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
# Restart other primary server
start_server((cnode['ip'], cnode['port']),
self.ipport2server, self.pids)
# Assert it does not know about container/obj
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
# Run the object-updaters
Manager(['object-updater']).once()
# Assert the other primary server now knows about container/obj
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assert_(obj in objs)
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url, self.token, 'c1',
headers={'X-Storage-Policy':
self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1])
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()
# check the re-started container server has update with override values
obj = direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
def test_direct_get_container(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
headers = {'key': 'value'}
body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]'
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, headers, body)
resp_headers, resp = (
direct_client.direct_get_container(node, part, account, container))
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(headers, resp_headers)
self.assertEqual(json_loads(body), resp)
direct_client.http_connect = mock_http_connect(204, headers, body)
resp_headers, resp = (
direct_client.direct_get_container(node, part, account, container))
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(headers, resp_headers)
self.assertEqual([], resp)
direct_client.http_connect = was_http_connector
def test_two_nodes_fail(self):
# Create container1
# Kill container1 servers excepting one of the primaries
# Delete container1 directly to the one primary still up
# Restart other container1 servers
# Get to a final state
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
# Assert account level also indicates container1 is gone
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
kill_server(cnodes[1]['port'], self.port2server, self.pids)
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
start_server(cnodes[1]['port'], self.port2server, self.pids)
start_server(cnp_port, self.port2server, self.pids)
get_to_final_state()
for cnode in cnodes:
exc = None
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '0')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
def test_main(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
apart, anodes = self.account_ring.get_nodes(self.account)
anode = anodes[0]
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill(self.pids[self.port2server[cnode['port']]], SIGTERM)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
self.pids[self.port2server[cnode['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnode['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
ps = []
for n in xrange(1, 5):
ps.append(
Popen([
'swift-object-updater',
'/etc/swift/object-server/%d.conf' % n, 'once'
]))
for p in ps:
p.wait()
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
self.assert_(obj in objs)
def test_main(self):
# Create container
container = "container-%s" % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server)
kill_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Create container/obj
obj = "object-%s" % uuid4()
client.put_object(self.url, self.token, container, obj, "")
# Restart other primary server
start_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Assert it does not know about container/obj
self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1])
# Run the object-updaters
Manager(["object-updater"]).once()
# Assert the other primary server now knows about container/obj
objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
self.assertIn(obj, objs)
def test_two_nodes_fail(self):
# Create container1
# Kill container1 servers excepting one of the primaries
# Delete container1 directly to the one primary still up
# Restart other container1 servers
# Get to a final state
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
# Assert account level also indicates container1 is gone
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
kill_server(cnodes[1]['port'], self.port2server, self.pids)
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
start_server(cnodes[1]['port'], self.port2server, self.pids)
start_server(cnp_port, self.port2server, self.pids)
get_to_final_state()
for cnode in cnodes:
exc = None
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '0')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
def test_missing_container(self):
# In this test, we need to put container at handoff devices, so we
# need container devices more than replica count
if len(self.container_ring.devs) <= self.container_ring.replica_count:
raise SkipTest("Need devices more that replica count")
container = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
# Kill all primary container servers
for cnode in cnodes:
kill_server((cnode['ip'], cnode['port']), self.ipport2server)
# Create container, and all of its replicas are placed at handoff
# device
try:
client.put_container(self.url, self.token, container)
except ClientException as err:
# if the cluster doesn't have enough devices, swift may return
# error (ex. When we only have 4 devices in 3-replica cluster).
self.assertEqual(err.http_status, 503)
# Assert handoff device has a container replica
another_cnode = next(self.container_ring.get_more_nodes(cpart))
direct_client.direct_get_container(
another_cnode, cpart, self.account, container)
# Restart all primary container servers
for cnode in cnodes:
start_server((cnode['ip'], cnode['port']), self.ipport2server)
# Create container/obj
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
# Run the object-updater
Manager(['object-updater']).once()
# Run the container-replicator, and now, container replicas
# at handoff device get moved to primary servers
Manager(['container-replicator']).once()
# Assert container replicas in primary servers, just moved by
# replicator don't know about the object
for cnode in cnodes:
self.assertFalse(direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
# since the container is empty - we can delete it!
client.delete_container(self.url, self.token, container)
# Re-run the object-updaters and now container replicas in primary
# container servers should get updated
Manager(['object-updater']).once()
# Assert all primary container servers know about container/obj
for cnode in cnodes:
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assertIn(obj, objs)
def test_main(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
apart, anodes = self.account_ring.get_nodes(self.account)
anode = anodes[0]
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill(self.pids[self.port2server[cnode['port']]], SIGTERM)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
self.pids[self.port2server[cnode['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnode['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(not direct_client.direct_get_container(cnode, cpart,
self.account, container)[1])
ps = []
for n in xrange(1, 5):
ps.append(Popen(['swift-object-updater',
'/etc/swift/object-server/%d.conf' % n, 'once']))
for p in ps:
p.wait()
objs = [o['name'] for o in direct_client.direct_get_container(cnode,
cpart, self.account, container)[1]]
self.assert_(obj in objs)
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server)
kill_server((cnode['ip'], cnode['port']), self.ipport2server)
# Create container/obj
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
# Restart other primary server
start_server((cnode['ip'], cnode['port']), self.ipport2server)
# Assert it does not know about container/obj
self.assertFalse(
direct_client.direct_get_container(cnode, cpart, self.account,
container)[1])
# Run the object-updaters
Manager(['object-updater']).once()
# Assert the other primary server now knows about container/obj
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
self.assertTrue(obj in objs)
def test_async_updates_after_PUT_and_POST(self):
# verify correct update values when PUT update and POST updates are
# missed but then async updates are sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# PUT and POST to object while one container server is stopped so that
# we force async updates to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url,
self.token,
'c1',
'o1',
contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(self.account, 'c1', 'o1',
{'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual('Tomato',
int_client.get_object_metadata(
self.account, 'c1',
'o1')['x-object-meta-fruit']) # sanity
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# Run the object-updaters to send the async pendings
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_main(self):
# Create container
# Kill container servers excepting two of the primaries
# Create container/obj
# Restart other primary server
# Assert it does not know about container/obj
# Run the object-updaters
# Assert the other primary server now knows about container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnode['port'], self.port2server, self.pids)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
start_server(cnode['port'], self.port2server, self.pids)
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
processes = []
for node in xrange(1, 5):
processes.append(
Popen([
'swift-object-updater', self.configs['object'] % node,
'once'
]))
for process in processes:
process.wait()
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
self.assert_(obj in objs)
def test_missing_container(self):
# In this test, we need to put container at handoff devices, so we
# need container devices more than replica count
if len(self.container_ring.devs) <= self.container_ring.replica_count:
raise SkipTest("Need devices more that replica count")
container = "container-%s" % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
# Kill all primary container servers
for cnode in cnodes:
kill_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Create container, and all of its replicas are placed at handoff
# device
try:
client.put_container(self.url, self.token, container)
except ClientException as err:
# if the cluster doesn't have enough devices, swift may return
# error (ex. When we only have 4 devices in 3-replica cluster).
self.assertEqual(err.http_status, 503)
# Assert handoff device has a container replica
another_cnode = self.container_ring.get_more_nodes(cpart).next()
direct_client.direct_get_container(another_cnode, cpart, self.account, container)
# Restart all primary container servers
for cnode in cnodes:
start_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Create container/obj
obj = "object-%s" % uuid4()
client.put_object(self.url, self.token, container, obj, "")
# Run the object-updater
Manager(["object-updater"]).once()
# Run the container-replicator, and now, container replicas
# at handoff device get moved to primary servers
Manager(["container-replicator"]).once()
# Assert container replicas in primary servers, just moved by
# replicator don't know about the object
for cnode in cnodes:
self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1])
# since the container is empty - we can delete it!
client.delete_container(self.url, self.token, container)
# Re-run the object-updaters and now container replicas in primary
# container servers should get updated
Manager(["object-updater"]).once()
# Assert all primary container servers know about container/obj
for cnode in cnodes:
objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
self.assertIn(obj, objs)
def test_update_during_POST_only(self):
# verify correct update values when PUT update is missed but then a
# POST update succeeds *before* the PUT async pending update is sent
cpart, cnodes = self.container_ring.get_nodes(self.account, "c1")
client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
content = u"stuff"
client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype")
meta = client.head_object(self.url, self.token, "c1", "o1")
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1])
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(self.account, "c1", "o1", {"X-Object-Meta-Fruit": "Tomato"})
self.assertEqual(
"Tomato", int_client.get_object_metadata(self.account, "c1", "o1")["x-object-meta-fruit"]
) # sanity
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual("test/ctype", listing[0]["content_type"])
listing_etags.add(listing[0]["hash"])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, "c1")
self.assertEqual(1, len(listing))
self.assertEqual("o1", listing[0]["name"])
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual(meta["etag"], listing[0]["hash"])
self.assertEqual("test/ctype", listing[0]["content_type"])
# Run the object-updaters to send the async pending from the PUT
Manager(["object-updater"]).once()
# check container listing metadata is still correct
for cnode in cnodes:
listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual("test/ctype", listing[0]["content_type"])
listing_etags.add(listing[0]["hash"])
self.assertEqual(1, len(listing_etags))
def test_async_updates_after_PUT_and_POST(self):
# verify correct update values when PUT update and POST updates are
# missed but then async updates are sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url, self.token, 'c1',
headers={'X-Storage-Policy':
self.policy.name})
# PUT and POST to object while one container server is stopped so that
# we force async updates to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(
self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual(
'Tomato',
int_client.get_object_metadata(self.account, 'c1', 'o1')
['x-object-meta-fruit']) # sanity
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1])
# Run the object-updaters to send the async pendings
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url,
self.token,
'c1',
'o1',
contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def do_test(req_params):
headers = HeaderKeyDict({'key': 'value'})
body = (b'[{"hash": "8f4e3", "last_modified": "317260", '
b'"bytes": 209}]')
with mocked_http_conn(200, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container,
**req_params)
try:
self.assertEqual(conn.method, 'GET')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'],
self.user_agent)
self.assertEqual(headers, resp_headers)
self.assertEqual(json.loads(body), resp)
self.assertIn('format=json', conn.query_string)
for k, v in req_params.items():
if v is None:
self.assertNotIn('&%s' % k, conn.query_string)
else:
self.assertIn('&%s=%s' % (k, v), conn.query_string)
except AssertionError as err:
self.fail('Failed with params %s: %s' % (req_params, err))
def do_test(req_params):
headers = HeaderKeyDict({'key': 'value'})
body = ('[{"hash": "8f4e3", "last_modified": "317260", '
'"bytes": 209}]')
with mocked_http_conn(200, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container,
**req_params)
try:
self.assertEqual(conn.method, 'GET')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'],
self.user_agent)
self.assertEqual(headers, resp_headers)
self.assertEqual(json.loads(body), resp)
self.assertIn('format=json', conn.query_string)
for k, v in req_params.items():
if v is None:
self.assertNotIn('&%s' % k, conn.query_string)
else:
self.assertIn('&%s=%s' % (k, v), conn.query_string)
except AssertionError as err:
self.fail('Failed with params %s: %s' % (req_params, err))
def test_one_node_fails(self):
# Create container1
# Kill container1 servers excepting two of the primaries
# Delete container1
# Restart other container1 primary server
# Create container1/object1 (allowed because at least server thinks the
# container exists)
# Get to a final state
# Assert all container1 servers indicate container1 is alive and
# well with object1
# Assert account level also indicates container1 is alive and
# well with object1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container1, 'object1', '123')
get_to_final_state()
for cnode in cnodes:
self.assertEquals([
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container1)[1]
], ['object1'])
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '3')
def test_direct_get_container(self):
headers = HeaderKeyDict({'key': 'value'})
body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]'
with mocked_http_conn(200, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node,
self.part,
self.account,
self.container,
marker='marker',
prefix='prefix',
delimiter='delimiter',
limit=1000,
end_marker='endmarker',
reverse='on')
self.assertEqual(conn.req_headers['user-agent'],
'direct-client %s' % os.getpid())
self.assertEqual(headers, resp_headers)
self.assertEqual(json.loads(body), resp)
self.assertTrue('marker=marker' in conn.query_string)
self.assertTrue('delimiter=delimiter' in conn.query_string)
self.assertTrue('limit=1000' in conn.query_string)
self.assertTrue('prefix=prefix' in conn.query_string)
self.assertTrue('format=json' in conn.query_string)
self.assertTrue('end_marker=endmarker' in conn.query_string)
self.assertTrue('reverse=on' in conn.query_string)
def test_one_node_fails(self):
# Create container1
# Kill container1 servers excepting two of the primaries
# Delete container1
# Restart other container1 primary server
# Create container1/object1 (allowed because at least server thinks the
# container exists)
# Get to a final state
# Assert all container1 servers indicate container1 is alive and
# well with object1
# Assert account level also indicates container1 is alive and
# well with object1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container1, 'object1', '123')
get_to_final_state()
for cnode in cnodes:
self.assertEquals(
[o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container1)[1]],
['object1'])
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '3')
def test_update_during_POST_only(self):
# verify correct update values when PUT update is missed but then a
# POST update succeeds *before* the PUT async pending update is sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(self.account, 'c1', 'o1',
{'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual('Tomato',
int_client.get_object_metadata(
self.account, 'c1',
'o1')['x-object-meta-fruit']) # sanity
# check the re-started container server has update with override values
obj = direct_client.direct_get_container(cnodes[0], cpart,
self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
# Run the object-updaters to send the async pending from the PUT
Manager(['object-updater']).once()
# check container listing metadata is still correct
obj = direct_client.direct_get_container(cnodes[0], cpart,
self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
def test_two_nodes_fail(self):
# Create container1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting one of the primaries
cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server,
self.pids)
kill_server((cnodes[0]['ip'], cnodes[0]['port']),
self.ipport2server, self.pids)
kill_server((cnodes[1]['ip'], cnodes[1]['port']),
self.ipport2server, self.pids)
# Delete container1 directly to the one primary still up
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
# Restart other container1 servers
start_server((cnodes[0]['ip'], cnodes[0]['port']),
self.ipport2server, self.pids)
start_server((cnodes[1]['ip'], cnodes[1]['port']),
self.ipport2server, self.pids)
start_server(cnp_ipport, self.ipport2server, self.pids)
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
for cnode in cnodes:
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert account level also indicates container1 is gone
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '0')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
def test_two_nodes_fail(self):
# Create container1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting one of the primaries
cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server,
self.pids)
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server,
self.pids)
kill_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server,
self.pids)
# Delete container1 directly to the one primary still up
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
# Restart other container1 servers
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server,
self.pids)
start_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server,
self.pids)
start_server(cnp_ipport, self.ipport2server, self.pids)
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
for cnode in cnodes:
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert account level also indicates container1 is gone
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '0')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
def test_update_during_POST_only(self):
# verify correct update values when PUT update is missed but then a
# POST update succeeds *before* the PUT async pending update is sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url, self.token, 'c1',
headers={'X-Storage-Policy':
self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1])
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(
self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual(
'Tomato',
int_client.get_object_metadata(self.account, 'c1', 'o1')
['x-object-meta-fruit']) # sanity
# check the re-started container server has update with override values
obj = direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
# Run the object-updaters to send the async pending from the PUT
Manager(['object-updater']).once()
# check container listing metadata is still correct
obj = direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url, self.token, 'c1',
headers={'X-Storage-Policy':
self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1])
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_direct_get_container_no_content_does_not_decode_body(self):
headers = {}
body = ''
with mocked_http_conn(204, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(headers, resp_headers)
self.assertEqual([], resp)
def test_direct_get_container_no_content_does_not_decode_body(self):
headers = {}
body = ''
with mocked_http_conn(204, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(headers, resp_headers)
self.assertEqual([], resp)
def test_first_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in xrange(2):
kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
for x in xrange(2):
self.pids[self.port2server[cnodes[x]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
# the update that the object was deleted yet), the whole must 503
# (until every is synced up, then the delete would work).
exc = None
try:
client.delete_container(self.url, self.token, container)
except client.ClientException, err:
exc = err
def test_direct_get_container(self):
headers = HeaderKeyDict({'key': 'value'})
body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]'
with mocked_http_conn(200, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container)
self.assertEqual(conn.req_headers['user-agent'],
'direct-client %s' % os.getpid())
self.assertEqual(headers, resp_headers)
self.assertEqual(json.loads(body), resp)
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, "c1")
client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
content = u"stuff"
client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype")
meta = client.head_object(self.url, self.token, "c1", "o1")
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1])
# Run the object-updaters to be sure updates are done
Manager(["object-updater"]).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual("test/ctype", listing[0]["content_type"])
listing_etags.add(listing[0]["hash"])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, "c1")
self.assertEqual(1, len(listing))
self.assertEqual("o1", listing[0]["name"])
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual(meta["etag"], listing[0]["hash"])
self.assertEqual("test/ctype", listing[0]["content_type"])
def test_first_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in xrange(2):
kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
for x in xrange(2):
self.pids[self.port2server[cnodes[x]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
# the update that the object was deleted yet), the whole must 503
# (until every is synced up, then the delete would work).
exc = None
try:
client.delete_container(self.url, self.token, container)
except client.ClientException, err:
exc = err
def get_info_1(container_ring, obj_path, logger):
path_comps = split_path(obj_path, 1, 3, True)
account_name = path_comps[0]
container_name = path_comps[1]
obj_name = path_comps[2]
container_part, container_nodes = \
container_ring.get_nodes(account_name, container_name)
if not container_nodes:
raise ContainerError()
# Perhaps we should do something about the way we select the container
# nodes. For now we just shuffle. It spreads the load, but it does not
# improve upon the the case when some nodes are down, so auditor slows
# to a crawl (if this plugin is enabled).
random.shuffle(container_nodes)
dark_flag = 0
for node in container_nodes:
try:
headers, objs = direct_get_container(node,
container_part,
account_name,
container_name,
prefix=obj_name,
limit=1)
except (ClientException, Timeout):
# Something is wrong with that server, treat as an error.
continue
if not objs or objs[0]['name'] != obj_name:
dark_flag += 1
continue
return objs[0]
# We do not ask for a quorum of container servers to know the object.
# Even if 1 server knows the object, we return with the info above.
# So, we only end here when all servers either have no record of the
# object or error out. In such case, even one non-error server means
# that the object is dark.
if dark_flag:
return None
raise ContainerError()
def test_direct_get_container(self):
headers = HeaderKeyDict({'key': 'value'})
body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]'
with mocked_http_conn(200, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container,
marker='marker', prefix='prefix', delimiter='delimiter',
limit=1000)
self.assertEqual(conn.req_headers['user-agent'],
'direct-client %s' % os.getpid())
self.assertEqual(headers, resp_headers)
self.assertEqual(json.loads(body), resp)
self.assertTrue('marker=marker' in conn.query_string)
self.assertTrue('delimiter=delimiter' in conn.query_string)
self.assertTrue('limit=1000' in conn.query_string)
self.assertTrue('prefix=prefix' in conn.query_string)
self.assertTrue('format=json' in conn.query_string)
def test_one_node_fails(self):
# Create container1
container1 = "container-%s" % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting two of the primaries
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]["port"], self.port2server, self.pids)
# Delete container1
client.delete_container(self.url, self.token, container1)
# Restart other container1 primary server
start_server(cnodes[0]["port"], self.port2server, self.pids)
# Create container1/object1 (allowed because at least server thinks the
# container exists)
client.put_object(self.url, self.token, container1, "object1", "123")
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is alive and
# well with object1
for cnode in cnodes:
self.assertEquals(
[o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container1)[1]],
["object1"],
)
# Assert account level also indicates container1 is alive and
# well with object1
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers["x-account-container-count"], "1")
self.assertEquals(headers["x-account-object-count"], "1")
self.assertEquals(headers["x-account-bytes-used"], "3")
def get_info_1(container_ring, obj_path, logger):
path_comps = split_path(obj_path, 1, 3, True)
account_name = path_comps[0]
container_name = path_comps[1]
obj_name = path_comps[2]
container_part, container_nodes = \
container_ring.get_nodes(account_name, container_name)
if not container_nodes:
raise ContainerError()
# Perhaps we should do something about the way we select the container
# nodes. For now we just shuffle. It spreads the load, but it does not
# improve upon the the case when some nodes are down, so auditor slows
# to a crawl (if this plugin is enabled).
random.shuffle(container_nodes)
err_flag = 0
for node in container_nodes:
try:
headers, objs = direct_get_container(
node, container_part, account_name, container_name,
prefix=obj_name, limit=1)
except (ClientException, Timeout):
# Something is wrong with that server, treat as an error.
err_flag += 1
continue
if objs and objs[0]['name'] == obj_name:
return objs[0]
# We only report the object as dark if all known servers agree that it is.
if err_flag:
raise ContainerError()
return None
def test_main(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
apart, anodes = self.account_ring.get_nodes(self.account)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill(self.pids[self.port2server[node['port']]], SIGTERM)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
self.pids[self.port2server[node['port']]] = Popen([
'swift-object-server',
'/etc/swift/object-server/%d.conf' %
((node['port'] - 6000) / 10)]).pid
sleep(2)
# We've indirectly verified the handoff node has the object, but let's
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart,
self.account, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(cnode, cpart,
self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([
'swift-object-server',
'/etc/swift/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
exc = False
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Previously downed object server had test object')
# Run the extra server last so it'll remove its extra partition
ps = []
for n in onodes:
ps.append(Popen(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once']))
for p in ps:
p.wait()
call(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'])
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Handoff object server still had test object')
# Because POST has changed to a COPY by default, POSTs will succeed on all up
# nodes now if at least one up node has the object.
# kill(self.pids[self.port2server[onode['port']]], SIGTERM)
# client.post_object(self.url, self.token, container, obj,
# headers={'x-object-meta-probe': 'value'})
# oheaders = client.head_object(self.url, self.token, container, obj)
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception('Metadata incorrect, was %s' % repr(oheaders))
# exc = False
# try:
# direct_client.direct_get_object(another_onode, opart, self.account,
# container, obj)
# except Exception:
# exc = True
# if not exc:
# raise Exception('Handoff server claimed it had the object when '
# 'it should not have it')
# self.pids[self.port2server[onode['port']]] = Popen([
# 'swift-object-server',
# '/etc/swift/object-server/%d.conf' %
# ((onode['port'] - 6000) / 10)]).pid
# sleep(2)
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') == 'value':
# raise Exception('Previously downed object server had the new '
# 'metadata when it should not have it')
# # Run the extra server last so it'll remove its extra partition
# ps = []
# for n in onodes:
# ps.append(Popen(['swift-object-replicator',
# '/etc/swift/object-server/%d.conf' %
# ((n['port'] - 6000) / 10), 'once']))
# for p in ps:
# p.wait()
# call(['swift-object-replicator',
# '/etc/swift/object-server/%d.conf' %
# ((another_onode['port'] - 6000) / 10), 'once'])
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception(
# 'Previously downed object server did not have the new metadata')
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, obj)
exc = False
try:
client.head_object(self.url, self.token, container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Regular object HEAD was still successful')
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([
'swift-object-server',
'/etc/swift/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
direct_client.direct_get_object(onode, opart, self.account, container,
obj)
# Run the extra server last so it'll remove its extra partition
ps = []
for n in onodes:
ps.append(Popen(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once']))
for p in ps:
p.wait()
call(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'])
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Handoff object server still had the object')
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Kill one container/obj primary server
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
# We've indirectly verified the handoff node has the container/object,
# but let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
# Bring the first container/obj primary server back up
start_server(onode['port'], self.port2server, self.pids)
# Assert that it doesn't have container/obj yet
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
kill_server(onode['port'], self.port2server, self.pids)
# Delete container/obj
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
# Assert we can't head container/obj
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert container/obj is not in the container listing, both indirectly
# and directly
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
# Restart the first container/obj primary server again
start_server(onode['port'], self.port2server, self.pids)
# Assert it still has container/obj
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
# Create container
# Kill one container/obj primary server
# Delete the "objects" directory on the primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication for first container/obj primary server
# Run object replication for handoff node
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
obj_dir = '%s/objects' % self._get_objects_dir(onode)
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart,
self.account, container,
obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj not in objs:
raise Exception('Container listing did not know about object')
timeout = time.time() + 5
found_objs_on_cnode = []
while time.time() < timeout:
for cnode in [
c for c in cnodes if cnodes not in found_objs_on_cnode
]:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj in objs:
found_objs_on_cnode.append(cnode)
if len(found_objs_on_cnode) >= len(cnodes):
break
time.sleep(0.3)
if len(found_objs_on_cnode) < len(cnodes):
missing = [
'%s:%s' % (cnode['ip'], cnode['port']) for cnode in cnodes
if cnode not in found_objs_on_cnode
]
raise Exception('Container servers %r did not know about object' %
missing)
start_server(onode['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
self.assertFalse(os.path.exists(obj_dir))
try:
port_num = onode['replication_port']
except KeyError:
port_num = onode['port']
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
call([
'swift-object-replicator',
self.configs['object-replicator'] % ((port_num - 6000) / 10),
'once'
])
call([
'swift-object-replicator', self.configs['object-replicator'] %
((another_port_num - 6000) / 10), 'once'
])
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
def test_first_node_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
self.pids[self.port2server[cnodes[0]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[0]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# And, of course, our new object2 exists.
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url,
self.token,
container,
headers={'X-Storage-Policy': self.policy.name})
# Kill one container/obj primary server
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
onode = onodes[0]
kill_server((onode['ip'], onode['port']), self.ipport2server,
self.pids)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server((node['ip'], node['port']), self.ipport2server,
self.pids)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server((node['ip'], node['port']), self.ipport2server,
self.pids)
# We've indirectly verified the handoff node has the container/object,
# but let's directly verify it.
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert container listing (via proxy and directly) has container/obj
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
# Bring the first container/obj primary server back up
start_server((onode['ip'], onode['port']), self.ipport2server,
self.pids)
# Assert that it doesn't have container/obj yet
try:
direct_client.direct_get_object(
onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
kill_server((onode['ip'], onode['port']), self.ipport2server,
self.pids)
# Delete container/obj
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
# Assert we can't head container/obj
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert container/obj is not in the container listing, both indirectly
# and directly
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
# Restart the first container/obj primary server again
start_server((onode['ip'], onode['port']), self.ipport2server,
self.pids)
# Assert it still has container/obj
direct_client.direct_get_object(
onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
try:
direct_client.direct_get_object(
another_onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
apart, anodes = self.account_ring.get_nodes(self.account)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill(self.pids[self.port2server[node['port']]], SIGTERM)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
self.pids[self.port2server[node['port']]] = Popen([
'swift-object-server',
'/etc/swift/object-server/%d.conf' %
((node['port'] - 6000) / 10)]).pid
sleep(2)
# We've indirectly verified the handoff node has the object, but let's
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart,
self.account, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(cnode, cpart,
self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([
'swift-object-server',
'/etc/swift/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
exc = False
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Previously downed object server had test object')
# Run the extra server last so it'll remove its extra partition
ps = []
for n in onodes:
ps.append(Popen(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once']))
for p in ps:
p.wait()
call(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'])
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Handoff object server still had test object')
# Because POST has changed to a COPY by default, POSTs will succeed on all up
# nodes now if at least one up node has the object.
# kill(self.pids[self.port2server[onode['port']]], SIGTERM)
# client.post_object(self.url, self.token, container, obj,
# headers={'x-object-meta-probe': 'value'})
# oheaders = client.head_object(self.url, self.token, container, obj)
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception('Metadata incorrect, was %s' % repr(oheaders))
# exc = False
# try:
# direct_client.direct_get_object(another_onode, opart, self.account,
# container, obj)
# except Exception:
# exc = True
# if not exc:
# raise Exception('Handoff server claimed it had the object when '
# 'it should not have it')
# self.pids[self.port2server[onode['port']]] = Popen([
# 'swift-object-server',
# '/etc/swift/object-server/%d.conf' %
# ((onode['port'] - 6000) / 10)]).pid
# sleep(2)
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') == 'value':
# raise Exception('Previously downed object server had the new '
# 'metadata when it should not have it')
# # Run the extra server last so it'll remove its extra partition
# ps = []
# for n in onodes:
# ps.append(Popen(['swift-object-replicator',
# '/etc/swift/object-server/%d.conf' %
# ((n['port'] - 6000) / 10), 'once']))
# for p in ps:
# p.wait()
# call(['swift-object-replicator',
# '/etc/swift/object-server/%d.conf' %
# ((another_onode['port'] - 6000) / 10), 'once'])
# oheaders = direct_client.direct_get_object(onode, opart, self.account,
# container, obj)[0]
# if oheaders.get('x-object-meta-probe') != 'value':
# raise Exception(
# 'Previously downed object server did not have the new metadata')
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, obj)
exc = False
try:
client.head_object(self.url, self.token, container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Regular object HEAD was still successful')
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
self.pids[self.port2server[onode['port']]] = Popen([
'swift-object-server',
'/etc/swift/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
direct_client.direct_get_object(onode, opart, self.account, container,
obj)
# Run the extra server last so it'll remove its extra partition
ps = []
for n in onodes:
ps.append(Popen(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((n['port'] - 6000) / 10), 'once']))
for p in ps:
p.wait()
call(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'])
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except Exception:
exc = True
if not exc:
raise Exception('Handoff object server still had the object')
class TestContainerFailures(unittest.TestCase):
def setUp(self):
self.pids, self.port2server, self.account_ring, self.container_ring, \
self.object_ring, self.url, self.token, self.account = \
reset_environment()
def tearDown(self):
kill_pids(self.pids)
def test_first_node_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
self.pids[self.port2server[cnodes[0]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[0]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# And, of course, our new object2 exists.
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
def test_second_node_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
self.pids[self.port2server[cnodes[1]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[1]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# And, of course, our new object2 exists.
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
def test_first_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in xrange(2):
kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
for x in xrange(2):
self.pids[self.port2server[cnodes[x]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
# the update that the object was deleted yet), the whole must 503
# (until every is synced up, then the delete would work).
exc = None
try:
client.delete_container(self.url, self.token, container)
except client.ClientException, err:
exc = err
self.assert_(exc)
self.assert_(exc.http_status, 503)
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# And, of course, our new object2 exists.
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
def test_main(self):
# Create container
# Kill one container/obj primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
# Delete container/obj
# Assert we can't head container/obj
# Assert container/obj is not in the container listing, both indirectly
# and directly
# Restart the first container/obj primary server again
# Assert it still has container/obj
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert primary node no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
# We've indirectly verified the handoff node has the object, but let's
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart,
self.account, container,
obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException, err:
exc = err
self.assert_(exc)
self.assert_(exc.http_status, 503)
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [
o['name'] for o in direct_client.direct_get_container(
cnodes[0], cpart, self.account, container)[1]
])
# And, of course, our new object2 exists.
self.assert_(object2 in [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(
container in
[c['name'] for c in client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
def reap_container(self, account, account_partition, account_nodes,
container):
"""
Deletes the data and the container itself for the given container. This
will call :func:`reap_object` up to sqrt(self.concurrency) times
concurrently for the objects in the container.
If there is any exception while deleting a single object, the process
will continue for any other objects in the container and the failed
objects will be tried again the next time this function is called with
the same parameters.
If there is any exception while listing the objects for deletion, the
process will stop (but will obviously be tried again the next time this
function is called with the same parameters). This is a possibility
since the listing comes from querying just the primary remote container
server.
Once all objects have been attempted to be deleted, the container
itself will be attempted to be deleted by sending a delete request to
all container nodes. The format of the delete request is such that each
container server will update a corresponding account server, removing
the container from the account's listing.
This function returns nothing and should raise no exception but only
update various self.stats_* values for what occurs.
:param account: The name of the account for the container.
:param account_partition: The partition for the account on the account
ring.
:param account_nodes: The primary node dicts for the account.
:param container: The name of the container to delete.
* See also: :func:`swift.common.ring.Ring.get_nodes` for a description
of the account node dicts.
"""
account_nodes = list(account_nodes)
part, nodes = self.get_container_ring().get_nodes(account, container)
node = nodes[-1]
pool = GreenPool(size=self.object_concurrency)
marker = ''
while True:
objects = None
try:
objects = direct_get_container(
node,
part,
account,
container,
marker=marker,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout)[1]
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
except ClientException, err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
self.stats_return_codes[err.http_status / 100] = \
self.stats_return_codes.get(err.http_status / 100, 0) + 1
if not objects:
break
try:
for obj in objects:
if isinstance(obj['name'], unicode):
obj['name'] = obj['name'].encode('utf8')
pool.spawn(self.reap_object, account, container, part,
nodes, obj['name'])
pool.waitall()
except Exception:
self.logger.exception(
_('Exception with objects for container '
'%(container)s for account %(account)s'), {
'container': container,
'account': account
})
marker = objects[-1]['name']
def test_first_node_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
self.pids[self.port2server[cnodes[0]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[0]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
# Kill one container/obj primary server
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, b'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Stash the on disk data from a primary for future comparison with the
# handoff - this may not equal 'VERIFY' if for example the proxy has
# crypto enabled
direct_get_data = direct_client.direct_get_object(
onodes[1], opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server((node['ip'], node['port']), self.ipport2server)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server((node['ip'], node['port']), self.ipport2server)
# We've indirectly verified the handoff node has the container/object,
# but let's directly verify it.
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(direct_get_data, odata)
# drop a tempfile in the handoff's datadir, like it might have
# had if there was an rsync failure while it was previously a
# primary
handoff_device_path = self.device_dir(another_onode)
data_filename = None
for root, dirs, files in os.walk(handoff_device_path):
for filename in files:
if filename.endswith('.data'):
data_filename = filename
temp_filename = '.%s.6MbL6r' % data_filename
temp_filepath = os.path.join(root, temp_filename)
if not data_filename:
self.fail('Did not find any data files on %r' %
handoff_device_path)
open(temp_filepath, 'w')
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
# Bring the first container/obj primary server back up
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert that it doesn't have container/obj yet
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) // 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) // 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(direct_get_data, odata)
# and that it does *not* have a temporary rsync dropping!
found_data_filename = False
primary_device_path = self.device_dir(onode)
for root, dirs, files in os.walk(primary_device_path):
for filename in files:
if filename.endswith('.6MbL6r'):
self.fail('Found unexpected file %s' %
os.path.join(root, filename))
if filename == data_filename:
found_data_filename = True
self.assertTrue(found_data_filename,
'Did not find data file %r on %r' % (
data_filename, primary_device_path))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Delete container/obj
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
# Assert we can't head container/obj
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert container/obj is not in the container listing, both indirectly
# and directly
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
# Restart the first container/obj primary server again
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert it still has container/obj
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) // 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) // 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
# Create container
# Kill one container/obj primary server
# Delete the "objects" directory on the primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication for first container/obj primary server
# Run object replication for handoff node
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
obj_dir = '%s/objects' % self._get_objects_dir(onode)
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
timeout = time.time() + 5
found_objs_on_cnode = []
while time.time() < timeout:
for cnode in [c for c in cnodes if cnodes not in
found_objs_on_cnode]:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
found_objs_on_cnode.append(cnode)
if len(found_objs_on_cnode) >= len(cnodes):
break
time.sleep(0.3)
if len(found_objs_on_cnode) < len(cnodes):
missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in
cnodes if cnode not in found_objs_on_cnode]
raise Exception('Container servers %r did not know about object' %
missing)
start_server(onode['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
self.assertFalse(os.path.exists(obj_dir))
try:
port_num = onode['replication_port']
except KeyError:
port_num = onode['port']
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
call(['swift-object-replicator',
self.configs['object-replicator'] %
((port_num - 6000) / 10), 'once'])
call(['swift-object-replicator',
self.configs['object-replicator'] %
((another_port_num - 6000) / 10), 'once'])
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
def test_main(self):
# Create container
# Kill one container/obj primary server
# Delete the "objects" directory on the primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication for first container/obj primary server
# Run object replication for handoff node
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
obj_dir = '%s/objects' % self._get_objects_dir(onode)
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException, err:
exc = err
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
# Kill one container/obj primary server
kill_server(onode['port'], self.port2server, self.pids)
# Delete the default data directory for objects on the primary server
obj_dir = '%s/%s' % (self._get_objects_dir(onode),
get_data_dir(self.policy))
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
# Directly to handoff server assert we can get container/obj
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
timeout = time.time() + 5
found_objs_on_cnode = []
while time.time() < timeout:
for cnode in [c for c in cnodes if cnodes not in
found_objs_on_cnode]:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
found_objs_on_cnode.append(cnode)
if len(found_objs_on_cnode) >= len(cnodes):
break
time.sleep(0.3)
if len(found_objs_on_cnode) < len(cnodes):
missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in
cnodes if cnode not in found_objs_on_cnode]
raise Exception('Container servers %r did not know about object' %
missing)
# Bring the first container/obj primary server back up
start_server(onode['port'], self.port2server, self.pids)
# Assert that it doesn't have container/obj yet
self.assertFalse(os.path.exists(obj_dir))
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
self.assertFalse(os.path.exists(obj_dir))
else:
self.fail("Expected ClientException but didn't get it")
try:
port_num = onode['replication_port']
except KeyError:
port_num = onode['port']
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
# Run object replication for first container/obj primary server
num = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=num)
# Run object replication for handoff node
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
self.assert_(exc)
self.assert_(exc.http_status, 503)
# Unfortunately, the following might pass or fail, depending on the
# position of the account server associated with the first container
# server we had killed. If the associated happens to be the first
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
def reap_container(self, account, account_partition, account_nodes,
container):
"""
Deletes the data and the container itself for the given container. This
will call :func:`reap_object` up to sqrt(self.concurrency) times
concurrently for the objects in the container.
If there is any exception while deleting a single object, the process
will continue for any other objects in the container and the failed
objects will be tried again the next time this function is called with
the same parameters.
If there is any exception while listing the objects for deletion, the
process will stop (but will obviously be tried again the next time this
function is called with the same parameters). This is a possibility
since the listing comes from querying just the primary remote container
server.
Once all objects have been attempted to be deleted, the container
itself will be attempted to be deleted by sending a delete request to
all container nodes. The format of the delete request is such that each
container server will update a corresponding account server, removing
the container from the account's listing.
This function returns nothing and should raise no exception but only
update various self.stats_* values for what occurs.
:param account: The name of the account for the container.
:param account_partition: The partition for the account on the account
ring.
:param account_nodes: The primary node dicts for the account.
:param container: The name of the container to delete.
* See also: :func:`swift.common.ring.Ring.get_nodes` for a description
of the account node dicts.
"""
account_nodes = list(account_nodes)
part, nodes = self.get_container_ring().get_nodes(account, container)
node = nodes[-1]
pool = GreenPool(size=self.object_concurrency)
marker = ''
while True:
objects = None
try:
headers, objects = direct_get_container(
node, part, account, container,
marker=marker,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout)
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
self.stats_return_codes[err.http_status / 100] = \
self.stats_return_codes.get(err.http_status / 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status / 100,))
if not objects:
break
try:
policy_index = headers.get('X-Backend-Storage-Policy-Index', 0)
for obj in objects:
if isinstance(obj['name'], unicode):
obj['name'] = obj['name'].encode('utf8')
pool.spawn(self.reap_object, account, container, part,
nodes, obj['name'], policy_index)
pool.waitall()
except (Exception, Timeout):
self.logger.exception(_('Exception with objects for container '
'%(container)s for account %(account)s'
),
{'container': container,
'account': account})
marker = objects[-1]['name']
if marker == '':
break
successes = 0
failures = 0
for node in nodes:
anode = account_nodes.pop()
try:
direct_delete_container(
node, part, account, container,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout,
headers={'X-Account-Host': '%(ip)s:%(port)s' % anode,
'X-Account-Partition': str(account_partition),
'X-Account-Device': anode['device'],
'X-Account-Override-Deleted': 'yes'})
successes += 1
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
failures += 1
self.logger.increment('containers_failures')
self.stats_return_codes[err.http_status / 100] = \
self.stats_return_codes.get(err.http_status / 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status / 100,))
if successes > failures:
self.stats_containers_deleted += 1
self.logger.increment('containers_deleted')
elif not successes:
self.stats_containers_remaining += 1
self.logger.increment('containers_remaining')
else:
self.stats_containers_possibly_remaining += 1
self.logger.increment('containers_possibly_remaining')
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
# Kill one container/obj primary server
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server((node['ip'], node['port']), self.ipport2server)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server((node['ip'], node['port']), self.ipport2server)
# We've indirectly verified the handoff node has the container/object,
# but let's directly verify it.
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# drop a tempfile in the handoff's datadir, like it might have
# had if there was an rsync failure while it was previously a
# primary
handoff_device_path = self.device_dir('object', another_onode)
data_filename = None
for root, dirs, files in os.walk(handoff_device_path):
for filename in files:
if filename.endswith('.data'):
data_filename = filename
temp_filename = '.%s.6MbL6r' % data_filename
temp_filepath = os.path.join(root, temp_filename)
if not data_filename:
self.fail('Did not find any data files on %r' %
handoff_device_path)
open(temp_filepath, 'w')
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
# Bring the first container/obj primary server back up
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert that it doesn't have container/obj yet
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# and that it does *not* have a temporary rsync dropping!
found_data_filename = False
primary_device_path = self.device_dir('object', onode)
for root, dirs, files in os.walk(primary_device_path):
for filename in files:
if filename.endswith('.6MbL6r'):
self.fail('Found unexpected file %s' %
os.path.join(root, filename))
if filename == data_filename:
found_data_filename = True
self.assertTrue(found_data_filename,
'Did not find data file %r on %r' % (
data_filename, primary_device_path))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Delete container/obj
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
# Assert we can't head container/obj
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert container/obj is not in the container listing, both indirectly
# and directly
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
# Restart the first container/obj primary server again
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert it still has container/obj
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
