def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server, self.pids)
kill_server((cnode['ip'], cnode['port']),
self.ipport2server, self.pids)
# Create container/obj
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
# Restart other primary server
start_server((cnode['ip'], cnode['port']),
self.ipport2server, self.pids)
# Assert it does not know about container/obj
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
# Run the object-updaters
Manager(['object-updater']).once()
# Assert the other primary server now knows about container/obj
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assert_(obj in objs)
def test_main(self):
# Create container
# Kill container servers excepting two of the primaries
# Create container/obj
# Restart other primary server
# Assert it does not know about container/obj
# Run the object-updaters
# Assert the other primary server now knows about container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnode['port'], self.port2server, self.pids)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
start_server(cnode['port'], self.port2server, self.pids)
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
processes = []
for node in xrange(1, 5):
processes.append(Popen(['swift-object-updater',
'/etc/swift/object-server/%d.conf' % node,
'once']))
for process in processes:
process.wait()
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assert_(obj in objs)
def test_main(self):
# Create container
container = "container-%s" % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server)
kill_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Create container/obj
obj = "object-%s" % uuid4()
client.put_object(self.url, self.token, container, obj, "")
# Restart other primary server
start_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Assert it does not know about container/obj
self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1])
# Run the object-updaters
Manager(["object-updater"]).once()
# Assert the other primary server now knows about container/obj
objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
self.assertIn(obj, objs)
def test_main(self):
# Create container
# Kill container servers excepting two of the primaries
# Create container/obj
# Restart other primary server
# Assert it does not know about container/obj
# Run the object-updaters
# Assert the other primary server now knows about container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnode['port'], self.port2server, self.pids)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
start_server(cnode['port'], self.port2server, self.pids)
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
Manager(['object-updater']).once()
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
self.assert_(obj in objs)
def test_async_updates_after_PUT_and_POST(self):
# verify correct update values when PUT update and POST updates are
# missed but then async updates are sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# PUT and POST to object while one container server is stopped so that
# we force async updates to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url,
self.token,
'c1',
'o1',
contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(self.account, 'c1', 'o1',
{'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual('Tomato',
int_client.get_object_metadata(
self.account, 'c1',
'o1')['x-object-meta-fruit']) # sanity
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# Run the object-updaters to send the async pendings
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_main(self):
# Create container
# Kill container servers excepting two of the primaries
# Create container/obj
# Restart other primary server
# Assert it does not know about container/obj
# Run the object-updaters
# Assert the other primary server now knows about container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnode['port'], self.port2server, self.pids)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
start_server(cnode['port'], self.port2server, self.pids)
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
processes = []
for node in xrange(1, 5):
processes.append(
Popen([
'swift-object-updater',
'/etc/swift/object-server/%d.conf' % node, 'once'
]))
for process in processes:
process.wait()
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
self.assert_(obj in objs)
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url, self.token, 'c1',
headers={'X-Storage-Policy':
self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1])
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()
# check the re-started container server has update with override values
obj = direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server)
kill_server((cnode['ip'], cnode['port']), self.ipport2server)
# Create container/obj
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
# Restart other primary server
start_server((cnode['ip'], cnode['port']), self.ipport2server)
# Assert it does not know about container/obj
self.assertFalse(direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
# Run the object-updaters
Manager(['object-updater']).once()
# Assert the other primary server now knows about container/obj
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assertIn(obj, objs)
def test_one_node_fails(self):
# Create container1
# Kill container1 servers excepting two of the primaries
# Delete container1
# Restart other container1 primary server
# Create container1/object1 (allowed because at least server thinks the
# container exists)
# Get to a final state
# Assert all container1 servers indicate container1 is alive and
# well with object1
# Assert account level also indicates container1 is alive and
# well with object1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container1, 'object1', '123')
get_to_final_state()
for cnode in cnodes:
self.assertEquals([
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container1)[1]
], ['object1'])
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '3')
def test_missing_container(self):
# In this test, we need to put container at handoff devices, so we
# need container devices more than replica count
if len(self.container_ring.devs) <= self.container_ring.replica_count:
raise SkipTest("Need devices more that replica count")
container = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
# Kill all primary container servers
for cnode in cnodes:
kill_server((cnode['ip'], cnode['port']), self.ipport2server)
# Create container, and all of its replicas are placed at handoff
# device
try:
client.put_container(self.url, self.token, container)
except ClientException as err:
# if the cluster doesn't have enough devices, swift may return
# error (ex. When we only have 4 devices in 3-replica cluster).
self.assertEqual(err.http_status, 503)
# Assert handoff device has a container replica
another_cnode = next(self.container_ring.get_more_nodes(cpart))
direct_client.direct_get_container(
another_cnode, cpart, self.account, container)
# Restart all primary container servers
for cnode in cnodes:
start_server((cnode['ip'], cnode['port']), self.ipport2server)
# Create container/obj
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
# Run the object-updater
Manager(['object-updater']).once()
# Run the container-replicator, and now, container replicas
# at handoff device get moved to primary servers
Manager(['container-replicator']).once()
# Assert container replicas in primary servers, just moved by
# replicator don't know about the object
for cnode in cnodes:
self.assertFalse(direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
# since the container is empty - we can delete it!
client.delete_container(self.url, self.token, container)
# Re-run the object-updaters and now container replicas in primary
# container servers should get updated
Manager(['object-updater']).once()
# Assert all primary container servers know about container/obj
for cnode in cnodes:
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assertIn(obj, objs)
def test_one_node_fails(self):
# Create container1
# Kill container1 servers excepting two of the primaries
# Delete container1
# Restart other container1 primary server
# Create container1/object1 (allowed because at least server thinks the
# container exists)
# Get to a final state
# Assert all container1 servers indicate container1 is alive and
# well with object1
# Assert account level also indicates container1 is alive and
# well with object1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container1, 'object1', '123')
get_to_final_state()
for cnode in cnodes:
self.assertEquals(
[o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container1)[1]],
['object1'])
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '3')
def test_missing_container(self):
# In this test, we need to put container at handoff devices, so we
# need container devices more than replica count
if len(self.container_ring.devs) <= self.container_ring.replica_count:
raise SkipTest("Need devices more that replica count")
container = "container-%s" % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
# Kill all primary container servers
for cnode in cnodes:
kill_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Create container, and all of its replicas are placed at handoff
# device
try:
client.put_container(self.url, self.token, container)
except ClientException as err:
# if the cluster doesn't have enough devices, swift may return
# error (ex. When we only have 4 devices in 3-replica cluster).
self.assertEqual(err.http_status, 503)
# Assert handoff device has a container replica
another_cnode = self.container_ring.get_more_nodes(cpart).next()
direct_client.direct_get_container(another_cnode, cpart, self.account, container)
# Restart all primary container servers
for cnode in cnodes:
start_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Create container/obj
obj = "object-%s" % uuid4()
client.put_object(self.url, self.token, container, obj, "")
# Run the object-updater
Manager(["object-updater"]).once()
# Run the container-replicator, and now, container replicas
# at handoff device get moved to primary servers
Manager(["container-replicator"]).once()
# Assert container replicas in primary servers, just moved by
# replicator don't know about the object
for cnode in cnodes:
self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1])
# since the container is empty - we can delete it!
client.delete_container(self.url, self.token, container)
# Re-run the object-updaters and now container replicas in primary
# container servers should get updated
Manager(["object-updater"]).once()
# Assert all primary container servers know about container/obj
for cnode in cnodes:
objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
self.assertIn(obj, objs)
def test_missing_primaries(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url,
self.token,
container,
headers={'X-Storage-Policy': self.policy.name})
# Create container/obj (goes to all three primaries)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, b'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries obj server
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
for onode in onodes:
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Indirectly (i.e., through proxy) try to GET object, it should return
# a 503, since all primaries will Timeout and handoffs return a 404.
try:
client.get_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 503)
else:
self.fail("Expected ClientException but didn't get it")
# Restart the first container/obj primary server again
onode = onodes[0]
start_server((onode['ip'], onode['port']), self.ipport2server)
# Send a delete that will reach first primary and handoff.
# Sure, the DELETE will return a 404 since the handoff doesn't
# have a .data file, but object server will still write a
# Tombstone in the handoff node!
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
# kill the first container/obj primary server again
kill_server((onode['ip'], onode['port']), self.ipport2server)
# a new GET should return a 404, since all primaries will Timeout
# and the handoff will return a 404 but this time with a tombstone
try:
client.get_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_update_during_POST_only(self):
# verify correct update values when PUT update is missed but then a
# POST update succeeds *before* the PUT async pending update is sent
cpart, cnodes = self.container_ring.get_nodes(self.account, "c1")
client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
content = u"stuff"
client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype")
meta = client.head_object(self.url, self.token, "c1", "o1")
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1])
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(self.account, "c1", "o1", {"X-Object-Meta-Fruit": "Tomato"})
self.assertEqual(
"Tomato", int_client.get_object_metadata(self.account, "c1", "o1")["x-object-meta-fruit"]
) # sanity
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual("test/ctype", listing[0]["content_type"])
listing_etags.add(listing[0]["hash"])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, "c1")
self.assertEqual(1, len(listing))
self.assertEqual("o1", listing[0]["name"])
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual(meta["etag"], listing[0]["hash"])
self.assertEqual("test/ctype", listing[0]["content_type"])
# Run the object-updaters to send the async pending from the PUT
Manager(["object-updater"]).once()
# check container listing metadata is still correct
for cnode in cnodes:
listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual("test/ctype", listing[0]["content_type"])
listing_etags.add(listing[0]["hash"])
self.assertEqual(1, len(listing_etags))
def test_async_updates_after_PUT_and_POST(self):
# verify correct update values when PUT update and POST updates are
# missed but then async updates are sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url, self.token, 'c1',
headers={'X-Storage-Policy':
self.policy.name})
# PUT and POST to object while one container server is stopped so that
# we force async updates to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(
self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual(
'Tomato',
int_client.get_object_metadata(self.account, 'c1', 'o1')
['x-object-meta-fruit']) # sanity
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1])
# Run the object-updaters to send the async pendings
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url,
self.token,
'c1',
'o1',
contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_update_during_POST_only(self):
# verify correct update values when PUT update is missed but then a
# POST update succeeds *before* the PUT async pending update is sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(self.account, 'c1', 'o1',
{'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual('Tomato',
int_client.get_object_metadata(
self.account, 'c1',
'o1')['x-object-meta-fruit']) # sanity
# check the re-started container server has update with override values
obj = direct_client.direct_get_container(cnodes[0], cpart,
self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
# Run the object-updaters to send the async pending from the PUT
Manager(['object-updater']).once()
# check container listing metadata is still correct
obj = direct_client.direct_get_container(cnodes[0], cpart,
self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
def test_update_during_POST_only(self):
# verify correct update values when PUT update is missed but then a
# POST update succeeds *before* the PUT async pending update is sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url, self.token, 'c1',
headers={'X-Storage-Policy':
self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content)
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1])
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(
self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual(
'Tomato',
int_client.get_object_metadata(self.account, 'c1', 'o1')
['x-object-meta-fruit']) # sanity
# check the re-started container server has update with override values
obj = direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
# Run the object-updaters to send the async pending from the PUT
Manager(['object-updater']).once()
# check container listing metadata is still correct
obj = direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1][0]
self.assertEqual(meta['etag'], obj['hash'])
self.assertEqual(len(content), obj['bytes'])
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url, self.token, 'c1',
headers={'X-Storage-Policy':
self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url, self.token, 'c1', 'o1', contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(
cnodes[0], cpart, self.account, 'c1')[1])
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_two_nodes_fail(self):
# Create container1
# Kill container1 servers excepting one of the primaries
# Delete container1 directly to the one primary still up
# Restart other container1 servers
# Get to a final state
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
# Assert account level also indicates container1 is gone
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
kill_server(cnodes[1]['port'], self.port2server, self.pids)
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
start_server(cnodes[1]['port'], self.port2server, self.pids)
start_server(cnp_port, self.port2server, self.pids)
get_to_final_state()
for cnode in cnodes:
exc = None
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '0')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
def test_two_nodes_fail(self):
# Create container1
# Kill container1 servers excepting one of the primaries
# Delete container1 directly to the one primary still up
# Restart other container1 servers
# Get to a final state
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
# Assert account level also indicates container1 is gone
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
kill_server(cnodes[1]['port'], self.port2server, self.pids)
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
start_server(cnodes[1]['port'], self.port2server, self.pids)
start_server(cnp_port, self.port2server, self.pids)
get_to_final_state()
for cnode in cnodes:
exc = None
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '0')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, "c1")
client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
content = u"stuff"
client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype")
meta = client.head_object(self.url, self.token, "c1", "o1")
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1])
# Run the object-updaters to be sure updates are done
Manager(["object-updater"]).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual("test/ctype", listing[0]["content_type"])
listing_etags.add(listing[0]["hash"])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, "c1")
self.assertEqual(1, len(listing))
self.assertEqual("o1", listing[0]["name"])
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual(meta["etag"], listing[0]["hash"])
self.assertEqual("test/ctype", listing[0]["content_type"])
def test_one_node_fails(self):
# Create container1
container1 = "container-%s" % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting two of the primaries
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]["port"], self.port2server, self.pids)
# Delete container1
client.delete_container(self.url, self.token, container1)
# Restart other container1 primary server
start_server(cnodes[0]["port"], self.port2server, self.pids)
# Create container1/object1 (allowed because at least server thinks the
# container exists)
client.put_object(self.url, self.token, container1, "object1", "123")
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is alive and
# well with object1
for cnode in cnodes:
self.assertEquals(
[o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container1)[1]],
["object1"],
)
# Assert account level also indicates container1 is alive and
# well with object1
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers["x-account-container-count"], "1")
self.assertEquals(headers["x-account-object-count"], "1")
self.assertEquals(headers["x-account-bytes-used"], "3")
def test_two_nodes_fail(self):
# Create container1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting one of the primaries
cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server,
self.pids)
kill_server((cnodes[0]['ip'], cnodes[0]['port']),
self.ipport2server, self.pids)
kill_server((cnodes[1]['ip'], cnodes[1]['port']),
self.ipport2server, self.pids)
# Delete container1 directly to the one primary still up
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
# Restart other container1 servers
start_server((cnodes[0]['ip'], cnodes[0]['port']),
self.ipport2server, self.pids)
start_server((cnodes[1]['ip'], cnodes[1]['port']),
self.ipport2server, self.pids)
start_server(cnp_ipport, self.ipport2server, self.pids)
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
for cnode in cnodes:
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert account level also indicates container1 is gone
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '0')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
def test_two_nodes_fail(self):
# Create container1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting one of the primaries
cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server,
self.pids)
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server,
self.pids)
kill_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server,
self.pids)
# Delete container1 directly to the one primary still up
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
# Restart other container1 servers
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server,
self.pids)
start_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server,
self.pids)
start_server(cnp_ipport, self.ipport2server, self.pids)
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
for cnode in cnodes:
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert account level also indicates container1 is gone
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '0')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
def test_main(self):
# Create container
# Kill one container/obj primary server
# Delete the "objects" directory on the primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication for first container/obj primary server
# Run object replication for handoff node
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
obj_dir = '%s/objects' % self._get_objects_dir(onode)
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException, err:
exc = err
def test_main(self):
# Create container1 and container2
# Assert account level sees them
# Create container2/object1
# Assert account level doesn't see it yet
# Get to final state
# Assert account level now sees the container2/object1
# Kill account servers excepting two of the primaries
# Delete container1
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
# Put container2/object2
# Run container updaters
# Assert account level now knows about container2/object2
# Restart other primary account server
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
# Get to final state
# Assert that server is now up to date
container1 = 'container1'
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assert_(found1)
self.assert_(found2)
client.put_object(self.url, self.token, container2, 'object1', '1234')
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assert_(found1)
self.assert_(found2)
get_to_final_state()
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
self.assert_(found2)
apart, anodes = self.account_ring.get_nodes(self.account)
kill_nonprimary_server(anodes, self.port2server, self.pids)
kill_server(anodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
client.put_object(self.url, self.token, container2, 'object2', '12345')
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(not found1)
self.assert_(found2)
processes = []
for node in xrange(1, 5):
processes.append(
Popen([
'swift-container-updater',
'/etc/swift/container-server/%d.conf' % node, 'once'
]))
for process in processes:
process.wait()
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assert_(not found1)
self.assert_(found2)
start_server(anodes[0]['port'], self.port2server, self.pids)
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
self.assert_(found2)
get_to_final_state()
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assert_(not found1)
self.assert_(found2)
def test_main(self):
# Create container
# Kill one container/obj primary server
# Delete the "objects" directory on the primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication for first container/obj primary server
# Run object replication for handoff node
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
obj_dir = '%s/objects' % self._get_objects_dir(onode)
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
timeout = time.time() + 5
found_objs_on_cnode = []
while time.time() < timeout:
for cnode in [c for c in cnodes if cnodes not in
found_objs_on_cnode]:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
found_objs_on_cnode.append(cnode)
if len(found_objs_on_cnode) >= len(cnodes):
break
time.sleep(0.3)
if len(found_objs_on_cnode) < len(cnodes):
missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in
cnodes if cnode not in found_objs_on_cnode]
raise Exception('Container servers %r did not know about object' %
missing)
start_server(onode['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
self.assertFalse(os.path.exists(obj_dir))
try:
port_num = onode['replication_port']
except KeyError:
port_num = onode['port']
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
call(['swift-object-replicator',
self.configs['object-replicator'] %
((port_num - 6000) / 10), 'once'])
call(['swift-object-replicator',
self.configs['object-replicator'] %
((another_port_num - 6000) / 10), 'once'])
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except direct_client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
def test_main(self):
# Create container1 and container2
# Assert account level sees them
# Create container2/object1
# Assert account level doesn't see it yet
# Get to final state
# Assert account level now sees the container2/object1
# Kill account servers excepting two of the primaries
# Delete container1
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
# Put container2/object2
# Run container updaters
# Assert account level now knows about container2/object2
# Restart other primary account server
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
# Get to final state
# Assert that server is now up to date
container1 = 'container1'
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assert_(found1)
self.assert_(found2)
client.put_object(self.url, self.token, container2, 'object1', '1234')
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assert_(found1)
self.assert_(found2)
get_to_final_state()
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
self.assert_(found2)
apart, anodes = self.account_ring.get_nodes(self.account)
kill_nonprimary_server(anodes, self.port2server, self.pids)
kill_server(anodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
client.put_object(self.url, self.token, container2, 'object2', '12345')
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(not found1)
self.assert_(found2)
processes = []
for node in xrange(1, 5):
processes.append(Popen([
'swift-container-updater',
self.configs['container'] % node,
'once']))
for process in processes:
process.wait()
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assert_(not found1)
self.assert_(found2)
start_server(anodes[0]['port'], self.port2server, self.pids)
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
self.assert_(found2)
get_to_final_state()
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assert_(not found1)
self.assert_(found2)
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
# Kill one container/obj primary server
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, b'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Stash the on disk data from a primary for future comparison with the
# handoff - this may not equal 'VERIFY' if for example the proxy has
# crypto enabled
direct_get_data = direct_client.direct_get_object(
onodes[1], opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server((node['ip'], node['port']), self.ipport2server)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server((node['ip'], node['port']), self.ipport2server)
# We've indirectly verified the handoff node has the container/object,
# but let's directly verify it.
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(direct_get_data, odata)
# drop a tempfile in the handoff's datadir, like it might have
# had if there was an rsync failure while it was previously a
# primary
handoff_device_path = self.device_dir(another_onode)
data_filename = None
for root, dirs, files in os.walk(handoff_device_path):
for filename in files:
if filename.endswith('.data'):
data_filename = filename
temp_filename = '.%s.6MbL6r' % data_filename
temp_filepath = os.path.join(root, temp_filename)
if not data_filename:
self.fail('Did not find any data files on %r' %
handoff_device_path)
open(temp_filepath, 'w')
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
# Bring the first container/obj primary server back up
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert that it doesn't have container/obj yet
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) // 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) // 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(direct_get_data, odata)
# and that it does *not* have a temporary rsync dropping!
found_data_filename = False
primary_device_path = self.device_dir(onode)
for root, dirs, files in os.walk(primary_device_path):
for filename in files:
if filename.endswith('.6MbL6r'):
self.fail('Found unexpected file %s' %
os.path.join(root, filename))
if filename == data_filename:
found_data_filename = True
self.assertTrue(found_data_filename,
'Did not find data file %r on %r' % (
data_filename, primary_device_path))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Delete container/obj
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
# Assert we can't head container/obj
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert container/obj is not in the container listing, both indirectly
# and directly
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
# Restart the first container/obj primary server again
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert it still has container/obj
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) // 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) // 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
# Create container1 and container2
container1 = 'container1'
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
# Assert account level sees them
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '2')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
self.assertTrue(found1)
self.assertTrue(found2)
# Create container2/object1
client.put_object(self.url, self.token, container2, 'object1', '1234')
# Assert account level doesn't see it yet
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '2')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
self.assertTrue(found1)
self.assertTrue(found2)
# Get to final state
self.get_to_final_state()
# Assert account level now sees the container2/object1
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '2')
self.assertEqual(headers['x-account-object-count'], '1')
self.assertEqual(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 1)
self.assertEqual(container['bytes'], 4)
self.assertTrue(found1)
self.assertTrue(found2)
apart, anodes = self.account_ring.get_nodes(self.account)
kill_nonprimary_server(anodes, self.ipport2server)
kill_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server)
# Kill account servers excepting two of the primaries
# Delete container1
client.delete_container(self.url, self.token, container1)
# Put container2/object2
client.put_object(self.url, self.token, container2, 'object2', '12345')
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '1')
self.assertEqual(headers['x-account-object-count'], '1')
self.assertEqual(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 1)
self.assertEqual(container['bytes'], 4)
self.assertFalse(found1)
self.assertTrue(found2)
# Run container updaters
Manager(['container-updater']).once()
# Assert account level now knows about container2/object2
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '1')
self.assertEqual(headers['x-account-object-count'], '2')
self.assertEqual(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 2)
self.assertEqual(container['bytes'], 9)
self.assertFalse(found1)
self.assertTrue(found2)
# Restart other primary account server
start_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server)
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEqual(headers['x-account-container-count'], '2')
self.assertEqual(headers['x-account-object-count'], '1')
self.assertEqual(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 1)
self.assertEqual(container['bytes'], 4)
self.assertTrue(found1)
self.assertTrue(found2)
# Get to final state
self.get_to_final_state()
# Assert that server is now up to date
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEqual(headers['x-account-container-count'], '1')
self.assertEqual(headers['x-account-object-count'], '2')
self.assertEqual(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 2)
self.assertEqual(container['bytes'], 9)
self.assertEqual(container['bytes'], 9)
self.assertFalse(found1)
self.assertTrue(found2)
except direct_client.ClientException, err:
exc = err
self.assertEquals(exc.http_status, 404)
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
direct_client.direct_get_object(onode, opart, self.account, container,
obj)
# Run the extra server last so it'll remove its extra partition
processes = []
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
processes.append(Popen(['swift-object-replicator',
self.configs['object-replicator'] %
((port_num - 6000) / 10),
'once']))
for process in processes:
process.wait()
def test_main(self):
# Create container
# Kill one container/obj primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
# Delete container/obj
# Assert we can't head container/obj
# Assert container/obj is not in the container listing, both indirectly
# and directly
# Restart the first container/obj primary server again
# Assert it still has container/obj
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert primary node no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
# We've indirectly verified the handoff node has the object, but let's
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException, err:
exc = err
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
# Kill one container/obj primary server
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Delete the default data directory for objects on the primary server
obj_dir = '%s/%s' % (self._get_objects_dir(onode),
get_data_dir(self.policy))
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Stash the on disk data from a primary for future comparison with the
# handoff - this may not equal 'VERIFY' if for example the proxy has
# crypto enabled
direct_get_data = direct_client.direct_get_object(
onodes[1], opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server((node['ip'], node['port']), self.ipport2server)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server((node['ip'], node['port']), self.ipport2server)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
# Directly to handoff server assert we can get container/obj
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(direct_get_data, odata)
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
timeout = time.time() + 5
found_objs_on_cnode = []
while time.time() < timeout:
for cnode in [c for c in cnodes if cnodes not in
found_objs_on_cnode]:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
found_objs_on_cnode.append(cnode)
if len(found_objs_on_cnode) >= len(cnodes):
break
time.sleep(0.3)
if len(found_objs_on_cnode) < len(cnodes):
missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in
cnodes if cnode not in found_objs_on_cnode]
raise Exception('Container servers %r did not know about object' %
missing)
# Bring the first container/obj primary server back up
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert that it doesn't have container/obj yet
self.assertFalse(os.path.exists(obj_dir))
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
self.assertFalse(os.path.exists(obj_dir))
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication for first container/obj primary server
_, num = get_server_number(
(onode['ip'], onode.get('replication_port', onode['port'])),
self.ipport2server)
Manager(['object-replicator']).once(number=num)
# Run object replication for handoff node
_, another_num = get_server_number(
(another_onode['ip'],
another_onode.get('replication_port', another_onode['port'])),
self.ipport2server)
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(direct_get_data, odata)
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
# Kill one container/obj primary server
kill_server(onode['port'], self.port2server, self.pids)
# Delete the default data directory for objects on the primary server
obj_dir = '%s/%s' % (self._get_objects_dir(onode),
get_data_dir(self.policy))
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
# Directly to handoff server assert we can get container/obj
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
timeout = time.time() + 5
found_objs_on_cnode = []
while time.time() < timeout:
for cnode in [c for c in cnodes if cnodes not in
found_objs_on_cnode]:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
found_objs_on_cnode.append(cnode)
if len(found_objs_on_cnode) >= len(cnodes):
break
time.sleep(0.3)
if len(found_objs_on_cnode) < len(cnodes):
missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in
cnodes if cnode not in found_objs_on_cnode]
raise Exception('Container servers %r did not know about object' %
missing)
# Bring the first container/obj primary server back up
start_server(onode['port'], self.port2server, self.pids)
# Assert that it doesn't have container/obj yet
self.assertFalse(os.path.exists(obj_dir))
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
self.assertFalse(os.path.exists(obj_dir))
else:
self.fail("Expected ClientException but didn't get it")
try:
port_num = onode['replication_port']
except KeyError:
port_num = onode['port']
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
# Run object replication for first container/obj primary server
num = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=num)
# Run object replication for handoff node
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url,
self.token,
container,
headers={'X-Storage-Policy': self.policy.name})
# Kill one container/obj primary server
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
onode = onodes[0]
kill_server((onode['ip'], onode['port']), self.ipport2server,
self.pids)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server((node['ip'], node['port']), self.ipport2server,
self.pids)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server((node['ip'], node['port']), self.ipport2server,
self.pids)
# We've indirectly verified the handoff node has the container/object,
# but let's directly verify it.
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert container listing (via proxy and directly) has container/obj
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
# Bring the first container/obj primary server back up
start_server((onode['ip'], onode['port']), self.ipport2server,
self.pids)
# Assert that it doesn't have container/obj yet
try:
direct_client.direct_get_object(
onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
kill_server((onode['ip'], onode['port']), self.ipport2server,
self.pids)
# Delete container/obj
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
# Assert we can't head container/obj
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert container/obj is not in the container listing, both indirectly
# and directly
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
# Restart the first container/obj primary server again
start_server((onode['ip'], onode['port']), self.ipport2server,
self.pids)
# Assert it still has container/obj
direct_client.direct_get_object(
onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
try:
direct_client.direct_get_object(
another_onode,
opart,
self.account,
container,
obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
# Kill one container/obj primary server
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server((node['ip'], node['port']), self.ipport2server)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server((node['ip'], node['port']), self.ipport2server)
# We've indirectly verified the handoff node has the container/object,
# but let's directly verify it.
another_onode = next(self.object_ring.get_more_nodes(opart))
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# drop a tempfile in the handoff's datadir, like it might have
# had if there was an rsync failure while it was previously a
# primary
handoff_device_path = self.device_dir('object', another_onode)
data_filename = None
for root, dirs, files in os.walk(handoff_device_path):
for filename in files:
if filename.endswith('.data'):
data_filename = filename
temp_filename = '.%s.6MbL6r' % data_filename
temp_filepath = os.path.join(root, temp_filename)
if not data_filename:
self.fail('Did not find any data files on %r' %
handoff_device_path)
open(temp_filepath, 'w')
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
# Bring the first container/obj primary server back up
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert that it doesn't have container/obj yet
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# and that it does *not* have a temporary rsync dropping!
found_data_filename = False
primary_device_path = self.device_dir('object', onode)
for root, dirs, files in os.walk(primary_device_path):
for filename in files:
if filename.endswith('.6MbL6r'):
self.fail('Found unexpected file %s' %
os.path.join(root, filename))
if filename == data_filename:
found_data_filename = True
self.assertTrue(found_data_filename,
'Did not find data file %r on %r' % (
data_filename, primary_device_path))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Delete container/obj
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
# Assert we can't head container/obj
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert container/obj is not in the container listing, both indirectly
# and directly
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
# Restart the first container/obj primary server again
start_server((onode['ip'], onode['port']), self.ipport2server)
# Assert it still has container/obj
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_metadata_sync(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url,
self.token,
container,
headers={
'X-Storage-Policy': self.policy.name,
'X-Container-Meta-A': '1',
'X-Container-Meta-B': '1',
'X-Container-Meta-C': '1'
})
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes.pop()
# 2 of 3 container servers are temporarily down
for node in cnodes:
kill_server((node['ip'], node['port']), self.ipport2server)
# Put some meta on the lone server, to make sure it's merged properly
# This will 503 (since we don't have a quorum), but we don't care (!)
try:
client.post_container(self.url,
self.token,
container,
headers={
'X-Container-Meta-A': '2',
'X-Container-Meta-B': '2',
'X-Container-Meta-D': '2'
})
except ClientException:
pass
# object updates come to only one container server
for _ in range(self.object_puts):
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, 'VERIFY')
# 2 container servers make comeback
for node in cnodes:
start_server((node['ip'], node['port']), self.ipport2server)
# But, container-server which got object updates is down
kill_server((cnode['ip'], cnode['port']), self.ipport2server)
# Metadata update will be applied to 2 container servers
# (equal to quorum)
client.post_container(self.url,
self.token,
container,
headers={
'X-Container-Meta-B': '3',
'X-Container-Meta-E': '3'
})
# container-server which got object updates makes comeback
start_server((cnode['ip'], cnode['port']), self.ipport2server)
# other nodes have no objects
for node in cnodes:
resp_headers = direct_client.direct_head_container(
node, cpart, self.account, container)
self.assertIn(resp_headers.get('x-container-object-count'),
(None, '0', 0))
# If container-replicator on the node which got the object updates
# runs in first, db file may be replicated by rsync to other
# containers. In that case, the db file does not information about
# metadata, so metadata should be synced before replication
Manager(['container-replicator'
]).once(number=self.config_number(cnode))
expected_meta = {
'x-container-meta-a': '2',
'x-container-meta-b': '2',
'x-container-meta-c': '1',
'x-container-meta-d': '2',
}
# node that got the object updates still doesn't have the meta
resp_headers = direct_client.direct_head_container(
cnode, cpart, self.account, container)
for header, value in expected_meta.items():
self.assertIn(header, resp_headers)
self.assertEqual(value, resp_headers[header])
self.assertNotIn(resp_headers.get('x-container-object-count'),
(None, '0', 0))
expected_meta = {
'x-container-meta-a': '2',
'x-container-meta-b': '3',
'x-container-meta-c': '1',
'x-container-meta-d': '2',
'x-container-meta-e': '3',
}
# other nodes still have the meta, as well as objects
for node in cnodes:
resp_headers = direct_client.direct_head_container(
node, cpart, self.account, container)
for header, value in expected_meta.items():
self.assertIn(header, resp_headers)
self.assertEqual(value, resp_headers[header])
self.assertNotIn(resp_headers.get('x-container-object-count'),
(None, '0', 0))
# and after full pass on remaining nodes
for node in cnodes:
Manager(['container-replicator'
]).once(number=self.config_number(node))
# ... all is right
for node in cnodes + [cnode]:
resp_headers = direct_client.direct_head_container(
node, cpart, self.account, container)
for header, value in expected_meta.items():
self.assertIn(header, resp_headers)
self.assertEqual(value, resp_headers[header])
self.assertNotIn(resp_headers.get('x-container-object-count'),
(None, '0', 0))
def test_metadata_sync(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy': self.policy.name,
'X-Container-Meta-A': '1',
'X-Container-Meta-B': '1',
'X-Container-Meta-C': '1'})
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes.pop()
# 2 of 3 container servers are temporarily down
for node in cnodes:
kill_server((node['ip'], node['port']),
self.ipport2server)
# Put some meta on the lone server, to make sure it's merged properly
# This will 503 (since we don't have a quorum), but we don't care (!)
try:
client.post_container(self.url, self.token, container,
headers={'X-Container-Meta-A': '2',
'X-Container-Meta-B': '2',
'X-Container-Meta-D': '2'})
except ClientException:
pass
# object updates come to only one container server
for _ in range(self.object_puts):
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, 'VERIFY')
# 2 container servers make comeback
for node in cnodes:
start_server((node['ip'], node['port']),
self.ipport2server)
# But, container-server which got object updates is down
kill_server((cnode['ip'], cnode['port']),
self.ipport2server)
# Metadata update will be applied to 2 container servers
# (equal to quorum)
client.post_container(self.url, self.token, container,
headers={'X-Container-Meta-B': '3',
'X-Container-Meta-E': '3'})
# container-server which got object updates makes comeback
start_server((cnode['ip'], cnode['port']),
self.ipport2server)
# other nodes have no objects
for node in cnodes:
resp_headers = direct_client.direct_head_container(
node, cpart, self.account, container)
self.assertIn(resp_headers.get('x-container-object-count'),
(None, '0', 0))
# If container-replicator on the node which got the object updates
# runs in first, db file may be replicated by rsync to other
# containers. In that case, the db file does not information about
# metadata, so metadata should be synced before replication
Manager(['container-replicator']).once(
number=self.config_number(cnode))
expected_meta = {
'x-container-meta-a': '2',
'x-container-meta-b': '3',
'x-container-meta-c': '1',
'x-container-meta-d': '2',
'x-container-meta-e': '3',
}
# node that got the object updates now has the meta
resp_headers = direct_client.direct_head_container(
cnode, cpart, self.account, container)
for header, value in expected_meta.items():
self.assertIn(header, resp_headers)
self.assertEqual(value, resp_headers[header])
self.assertNotIn(resp_headers.get('x-container-object-count'),
(None, '0', 0))
# other nodes still have the meta, as well as objects
for node in cnodes:
resp_headers = direct_client.direct_head_container(
node, cpart, self.account, container)
for header, value in expected_meta.items():
self.assertIn(header, resp_headers)
self.assertEqual(value, resp_headers[header])
self.assertNotIn(resp_headers.get('x-container-object-count'),
(None, '0', 0))
# and after full pass on remaining nodes
for node in cnodes:
Manager(['container-replicator']).once(
number=self.config_number(node))
# ... all is right
for node in cnodes + [cnode]:
resp_headers = direct_client.direct_head_container(
node, cpart, self.account, container)
for header, value in expected_meta.items():
self.assertIn(header, resp_headers)
self.assertEqual(value, resp_headers[header])
self.assertNotIn(resp_headers.get('x-container-object-count'),
(None, '0', 0))
def test_stale_reads(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
# Kill one primary obj server
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Create container/obj (goes to two primaries and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Stash the on disk data from a primary for future comparison with the
# handoff - this may not equal 'VERIFY' if for example the proxy has
# crypto enabled
direct_get_data = direct_client.direct_get_object(
onodes[1], opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
# Restart the first container/obj primary server again
start_server((onode['ip'], onode['port']), self.ipport2server)
# send a delete request to primaries
client.delete_object(self.url, self.token, container, obj)
# there should be .ts files in all primaries now
for node in onodes:
try:
direct_client.direct_get_object(
node, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# verify that handoff still has the data, DELETEs should have gone
# only to primaries
another_onode = next(self.object_ring.get_more_nodes(opart))
handoff_data = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(handoff_data, direct_get_data)
# Indirectly (i.e., through proxy) try to GET object, it should return
# a 404, before bug #1560574, the proxy would return the stale object
# from the handoff
try:
client.get_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_stale_reads(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy':
self.policy.name})
# Kill one primary obj server
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Create container/obj (goes to two primaries and one handoff)
client.put_object(self.url, self.token, container, obj, b'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Stash the on disk data from a primary for future comparison with the
# handoff - this may not equal 'VERIFY' if for example the proxy has
# crypto enabled
direct_get_data = direct_client.direct_get_object(
onodes[1], opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
# Restart the first container/obj primary server again
start_server((onode['ip'], onode['port']), self.ipport2server)
# send a delete request to primaries
client.delete_object(self.url, self.token, container, obj)
# there should be .ts files in all primaries now
for node in onodes:
try:
direct_client.direct_get_object(
node, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# verify that handoff still has the data, DELETEs should have gone
# only to primaries
another_onode = next(self.object_ring.get_more_nodes(opart))
handoff_data = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(handoff_data, direct_get_data)
# Indirectly (i.e., through proxy) try to GET object, it should return
# a 404, before bug #1560574, the proxy would return the stale object
# from the handoff
try:
client.get_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
# Create container
# Kill one container/obj primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
# Delete container/obj
# Assert we can't head container/obj
# Assert container/obj is not in the container listing, both indirectly
# and directly
# Restart the first container/obj primary server again
# Assert it still has container/obj
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert primary node no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
# We've indirectly verified the handoff node has the object, but let's
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart,
self.account, container,
obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except direct_client.ClientException, err:
exc = err
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
direct_client.direct_get_object(onode, opart, self.account, container,
obj)
# Run the extra server last so it'll remove its extra partition
processes = []
for node in onodes:
processes.append(
Popen([
'swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((node['port'] - 6000) / 10), 'once'
]))
for process in processes:
process.wait()
call([
'swift-object-replicator',
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Kill one container/obj primary server
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
# We've indirectly verified the handoff node has the container/object,
# but let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
(cnode['ip'], cnode['port']))
# Bring the first container/obj primary server back up
start_server(onode['port'], self.port2server, self.pids)
# Assert that it doesn't have container/obj yet
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
# Assert the handoff server no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
kill_server(onode['port'], self.port2server, self.pids)
# Delete container/obj
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
# Assert we can't head container/obj
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert container/obj is not in the container listing, both indirectly
# and directly
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
# Restart the first container/obj primary server again
start_server(onode['port'], self.port2server, self.pids)
# Assert it still has container/obj
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
# Run object replication, ensuring we run the handoff node last so it
# will remove its extra handoff partition
for node in onodes:
try:
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
try:
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_main(self):
# Create container
# Kill one container/obj primary server
# Delete the "objects" directory on the primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication for first container/obj primary server
# Run object replication for handoff node
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
obj_dir = '%s/objects' % self._get_objects_dir(onode)
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node['port'], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
for node in onodes[1:]:
start_server(node['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
# We've indirectly verified the handoff node has the object, but
# let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart,
self.account, container,
obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [
o['name']
for o in client.get_container(self.url, self.token, container)[1]
]
if obj not in objs:
raise Exception('Container listing did not know about object')
timeout = time.time() + 5
found_objs_on_cnode = []
while time.time() < timeout:
for cnode in [
c for c in cnodes if cnodes not in found_objs_on_cnode
]:
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
if obj in objs:
found_objs_on_cnode.append(cnode)
if len(found_objs_on_cnode) >= len(cnodes):
break
time.sleep(0.3)
if len(found_objs_on_cnode) < len(cnodes):
missing = [
'%s:%s' % (cnode['ip'], cnode['port']) for cnode in cnodes
if cnode not in found_objs_on_cnode
]
raise Exception('Container servers %r did not know about object' %
missing)
start_server(onode['port'], self.port2server, self.pids)
self.assertFalse(os.path.exists(obj_dir))
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
self.assertFalse(os.path.exists(obj_dir))
try:
port_num = onode['replication_port']
except KeyError:
port_num = onode['port']
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
call([
'swift-object-replicator',
self.configs['object-replicator'] % ((port_num - 6000) / 10),
'once'
])
call([
'swift-object-replicator', self.configs['object-replicator'] %
((another_port_num - 6000) / 10), 'once'
])
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
def test_main(self):
# Create container
# Kill one container/obj primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
# Restart those other two container/obj primary servers
# Directly to handoff server assert we can get container/obj
# Assert container listing (via proxy and directly) has container/obj
# Bring the first container/obj primary server back up
# Assert that it doesn't have container/obj yet
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert the first container/obj primary server now has container/obj
# Assert the handoff server no longer has container/obj
# Kill the first container/obj primary server again (we have two
# primaries and the handoff up now)
# Delete container/obj
# Assert we can't head container/obj
# Assert container/obj is not in the container listing, both indirectly
# and directly
# Restart the first container/obj primary server again
# Assert it still has container/obj
# Run object replication, ensuring we run the handoff node last so it
# should remove its extra handoff partition
# Assert primary node no longer has container/obj
container = "container-%s" % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
obj = "object-%s" % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container, obj)
onode = onodes[0]
kill_server(onode["port"], self.port2server, self.pids)
client.put_object(self.url, self.token, container, obj, "VERIFY")
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != "VERIFY":
raise Exception("Object GET did not return VERIFY, instead it " "returned: %s" % repr(odata))
# Kill all primaries to ensure GET handoff works
for node in onodes[1:]:
kill_server(node["port"], self.port2server, self.pids)
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != "VERIFY":
raise Exception("Object GET did not return VERIFY, instead it " "returned: %s" % repr(odata))
for node in onodes[1:]:
start_server(node["port"], self.port2server, self.pids)
# We've indirectly verified the handoff node has the object, but let's
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(another_onode, opart, self.account, container, obj)[-1]
if odata != "VERIFY":
raise Exception("Direct object GET did not return VERIFY, instead " "it returned: %s" % repr(odata))
objs = [o["name"] for o in client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception("Container listing did not know about object")
for cnode in cnodes:
objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
if obj not in objs:
raise Exception("Container server %s:%s did not know about object" % (cnode["ip"], cnode["port"]))
start_server(onode["port"], self.port2server, self.pids)
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account, container, obj)
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
# Run the extra server last so it'll remove its extra partition
processes = []
for node in onodes:
try:
port_num = node["replication_port"]
except KeyError:
port_num = node["port"]
processes.append(
Popen(["swift-object-replicator", self.configs["object-replicator"] % ((port_num - 6000) / 10), "once"])
)
for process in processes:
process.wait()
try:
another_port_num = another_onode["replication_port"]
except KeyError:
another_port_num = another_onode["port"]
call(["swift-object-replicator", self.configs["object-replicator"] % ((another_port_num - 6000) / 10), "once"])
odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1]
if odata != "VERIFY":
raise Exception("Direct object GET did not return VERIFY, instead " "it returned: %s" % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account, container, obj)
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
kill_server(onode["port"], self.port2server, self.pids)
client.delete_object(self.url, self.token, container, obj)
exc = None
try:
client.head_object(self.url, self.token, container, obj)
except client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
objs = [o["name"] for o in client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception("Container listing still knew about object")
for cnode in cnodes:
objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception("Container server %s:%s still knew about object" % (cnode["ip"], cnode["port"]))
start_server(onode["port"], self.port2server, self.pids)
direct_client.direct_get_object(onode, opart, self.account, container, obj)
# Run the extra server last so it'll remove its extra partition
processes = []
for node in onodes:
try:
port_num = node["replication_port"]
except KeyError:
port_num = node["port"]
processes.append(
Popen(["swift-object-replicator", self.configs["object-replicator"] % ((port_num - 6000) / 10), "once"])
)
for process in processes:
process.wait()
call(["swift-object-replicator", self.configs["object-replicator"] % ((another_port_num - 6000) / 10), "once"])
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account, container, obj)
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
def test_main(self):
# Create container1 and container2
container1 = 'container1'
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
# Assert account level sees them
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assertTrue(found1)
self.assertTrue(found2)
# Create container2/object1
client.put_object(self.url, self.token, container2, 'object1', '1234')
# Assert account level doesn't see it yet
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assertTrue(found1)
self.assertTrue(found2)
# Get to final state
self.get_to_final_state()
# Assert account level now sees the container2/object1
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assertTrue(found1)
self.assertTrue(found2)
apart, anodes = self.account_ring.get_nodes(self.account)
kill_nonprimary_server(anodes, self.ipport2server, self.pids)
kill_server((anodes[0]['ip'], anodes[0]['port']),
self.ipport2server, self.pids)
# Kill account servers excepting two of the primaries
# Delete container1
client.delete_container(self.url, self.token, container1)
# Put container2/object2
client.put_object(self.url, self.token, container2, 'object2', '12345')
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assertTrue(not found1)
self.assertTrue(found2)
# Run container updaters
Manager(['container-updater']).once()
# Assert account level now knows about container2/object2
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assertTrue(not found1)
self.assertTrue(found2)
# Restart other primary account server
start_server((anodes[0]['ip'], anodes[0]['port']),
self.ipport2server, self.pids)
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assertTrue(found1)
self.assertTrue(found2)
# Get to final state
self.get_to_final_state()
# Assert that server is now up to date
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assertTrue(not found1)
self.assertTrue(found2)
def test_update_pending(self):
# Create container
container = 'contx'
client.put_container(self.url, self.token, container)
part, nodes = self.account_ring.get_nodes(self.account)
anode = nodes[0]
# Stop a quorum of account servers
# This allows the put to continue later.
kill_nonprimary_server(nodes, self.ipport2server)
kill_server((anode['ip'], anode['port']), self.ipport2server)
# Put object
# This creates an outstanding update.
client.put_object(self.url, self.token, container, 'object1', b'123')
cont_db_files = self.get_container_db_files(container)
self.assertEqual(len(cont_db_files), 3)
# Collect the observable state from containers
outstanding_files = []
for cfile in cont_db_files:
broker = ContainerBroker(cfile)
try:
info = broker.get_info()
except LockTimeout:
self.fail('LockTimeout at %s' % (cfile, ))
if Timestamp(info['put_timestamp']) <= 0:
self.fail('No put_timestamp at %s' % (cfile, ))
# Correct even if reported_put_timestamp is zero.
if info['put_timestamp'] > info['reported_put_timestamp']:
outstanding_files.append(cfile)
self.assertGreater(len(outstanding_files), 0)
# At this point the users shut everything down and screw up the
# hash in swift.conf. But we destroy the account DB instead.
files = self.get_account_db_files(self.account)
for afile in files:
os.unlink(afile)
# Restart the stopped primary server
start_server((anode['ip'], anode['port']), self.ipport2server)
# Make sure updaters run
Manager(['container-updater']).once()
# Collect the observable state from containers again and examine it
outstanding_files_new = []
for cfile in cont_db_files:
# We aren't catching DatabaseConnectionError, because
# we only want to approve of DBs that were quarantined,
# and not otherwise damaged. So if the code below throws
# an exception for other reason, we want the test to fail.
if not os.path.exists(cfile):
continue
broker = ContainerBroker(cfile)
try:
info = broker.get_info()
except LockTimeout:
self.fail('LockTimeout at %s' % (cfile, ))
if Timestamp(info['put_timestamp']) <= 0:
self.fail('No put_timestamp at %s' % (cfile, ))
# Correct even if reported_put_timestamp is zero.
if info['put_timestamp'] > info['reported_put_timestamp']:
outstanding_files_new.append(cfile)
self.assertLengthEqual(outstanding_files_new, 0)
self.get_to_final_state()
