def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server)
kill_server((cnode['ip'], cnode['port']), self.ipport2server)
# Create container/obj
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
# Restart other primary server
start_server((cnode['ip'], cnode['port']), self.ipport2server)
# Assert it does not know about container/obj
self.assertFalse(
direct_client.direct_get_container(cnode, cpart, self.account,
container)[1])
# Run the object-updaters
Manager(['object-updater']).once()
# Assert the other primary server now knows about container/obj
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
self.assertTrue(obj in objs)
def test_main(self):
# Create container
container = "container-%s" % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server)
kill_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Create container/obj
obj = "object-%s" % uuid4()
client.put_object(self.url, self.token, container, obj, "")
# Restart other primary server
start_server((cnode["ip"], cnode["port"]), self.ipport2server)
# Assert it does not know about container/obj
self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1])
# Run the object-updaters
Manager(["object-updater"]).once()
# Assert the other primary server now knows about container/obj
objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]]
self.assertIn(obj, objs)
def test_one_node_fails(self):
# Create container1
# Kill container1 servers excepting two of the primaries
# Delete container1
# Restart other container1 primary server
# Create container1/object1 (allowed because at least server thinks the
# container exists)
# Get to a final state
# Assert all container1 servers indicate container1 is alive and
# well with object1
# Assert account level also indicates container1 is alive and
# well with object1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container1, 'object1', '123')
get_to_final_state()
for cnode in cnodes:
self.assertEquals([
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container1)[1]
], ['object1'])
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '3')
def test_one_node_fails(self):
# Create container1
# Kill container1 servers excepting two of the primaries
# Delete container1
# Restart other container1 primary server
# Create container1/object1 (allowed because at least server thinks the
# container exists)
# Get to a final state
# Assert all container1 servers indicate container1 is alive and
# well with object1
# Assert account level also indicates container1 is alive and
# well with object1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
client.put_object(self.url, self.token, container1, 'object1', '123')
get_to_final_state()
for cnode in cnodes:
self.assertEquals(
[o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container1)[1]],
['object1'])
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '3')
def test_main(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Kill container servers excepting two of the primaries
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.ipport2server, self.pids)
kill_server((cnode['ip'], cnode['port']),
self.ipport2server, self.pids)
# Create container/obj
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
# Restart other primary server
start_server((cnode['ip'], cnode['port']),
self.ipport2server, self.pids)
# Assert it does not know about container/obj
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
# Run the object-updaters
Manager(['object-updater']).once()
# Assert the other primary server now knows about container/obj
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assert_(obj in objs)
def test_main(self):
# Create container
# Kill container servers excepting two of the primaries
# Create container/obj
# Restart other primary server
# Assert it does not know about container/obj
# Run the object-updaters
# Assert the other primary server now knows about container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnode['port'], self.port2server, self.pids)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
start_server(cnode['port'], self.port2server, self.pids)
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
processes = []
for node in xrange(1, 5):
processes.append(
Popen([
'swift-object-updater', self.configs['object'] % node,
'once'
]))
for process in processes:
process.wait()
objs = [
o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]
]
self.assert_(obj in objs)
def test_main(self):
# Create container
# Kill container servers excepting two of the primaries
# Create container/obj
# Restart other primary server
# Assert it does not know about container/obj
# Run the object-updaters
# Assert the other primary server now knows about container/obj
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
cnode = cnodes[0]
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnode['port'], self.port2server, self.pids)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, '')
start_server(cnode['port'], self.port2server, self.pids)
self.assert_(not direct_client.direct_get_container(
cnode, cpart, self.account, container)[1])
processes = []
for node in xrange(1, 5):
processes.append(Popen(['swift-object-updater',
'/etc/swift/object-server/%d.conf' % node,
'once']))
for process in processes:
process.wait()
objs = [o['name'] for o in direct_client.direct_get_container(
cnode, cpart, self.account, container)[1]]
self.assert_(obj in objs)
def test_two_nodes_fail(self):
# Create container1
# Kill container1 servers excepting one of the primaries
# Delete container1 directly to the one primary still up
# Restart other container1 servers
# Get to a final state
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
# Assert account level also indicates container1 is gone
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
kill_server(cnodes[1]['port'], self.port2server, self.pids)
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
start_server(cnodes[1]['port'], self.port2server, self.pids)
start_server(cnp_port, self.port2server, self.pids)
get_to_final_state()
for cnode in cnodes:
exc = None
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '0')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
def test_two_nodes_fail(self):
# Create container1
# Kill container1 servers excepting one of the primaries
# Delete container1 directly to the one primary still up
# Restart other container1 servers
# Get to a final state
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
# Assert account level also indicates container1 is gone
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]['port'], self.port2server, self.pids)
kill_server(cnodes[1]['port'], self.port2server, self.pids)
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
start_server(cnodes[0]['port'], self.port2server, self.pids)
start_server(cnodes[1]['port'], self.port2server, self.pids)
start_server(cnp_port, self.port2server, self.pids)
get_to_final_state()
for cnode in cnodes:
exc = None
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except client.ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '0')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
def test_one_node_fails(self):
# Create container1
container1 = "container-%s" % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting two of the primaries
kill_nonprimary_server(cnodes, self.port2server, self.pids)
kill_server(cnodes[0]["port"], self.port2server, self.pids)
# Delete container1
client.delete_container(self.url, self.token, container1)
# Restart other container1 primary server
start_server(cnodes[0]["port"], self.port2server, self.pids)
# Create container1/object1 (allowed because at least server thinks the
# container exists)
client.put_object(self.url, self.token, container1, "object1", "123")
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is alive and
# well with object1
for cnode in cnodes:
self.assertEquals(
[o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container1)[1]],
["object1"],
)
# Assert account level also indicates container1 is alive and
# well with object1
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers["x-account-container-count"], "1")
self.assertEquals(headers["x-account-object-count"], "1")
self.assertEquals(headers["x-account-bytes-used"], "3")
def test_two_nodes_fail(self):
# Create container1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting one of the primaries
cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server,
self.pids)
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server,
self.pids)
kill_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server,
self.pids)
# Delete container1 directly to the one primary still up
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
# Restart other container1 servers
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server,
self.pids)
start_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server,
self.pids)
start_server(cnp_ipport, self.ipport2server, self.pids)
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
for cnode in cnodes:
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert account level also indicates container1 is gone
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '0')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
def test_two_nodes_fail(self):
# Create container1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
# Kill container1 servers excepting one of the primaries
cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server,
self.pids)
kill_server((cnodes[0]['ip'], cnodes[0]['port']),
self.ipport2server, self.pids)
kill_server((cnodes[1]['ip'], cnodes[1]['port']),
self.ipport2server, self.pids)
# Delete container1 directly to the one primary still up
direct_client.direct_delete_container(cnodes[2], cpart, self.account,
container1)
# Restart other container1 servers
start_server((cnodes[0]['ip'], cnodes[0]['port']),
self.ipport2server, self.pids)
start_server((cnodes[1]['ip'], cnodes[1]['port']),
self.ipport2server, self.pids)
start_server(cnp_ipport, self.ipport2server, self.pids)
# Get to a final state
self.get_to_final_state()
# Assert all container1 servers indicate container1 is gone (happens
# because the one node that knew about the delete replicated to the
# others.)
for cnode in cnodes:
try:
direct_client.direct_get_container(cnode, cpart, self.account,
container1)
except ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
# Assert account level also indicates container1 is gone
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '0')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
def test_main(self):
# Create container1 and container2
container1 = 'container1'
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
# Assert account level sees them
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '2')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
self.assertTrue(found1)
self.assertTrue(found2)
# Create container2/object1
client.put_object(self.url, self.token, container2, 'object1', '1234')
# Assert account level doesn't see it yet
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '2')
self.assertEqual(headers['x-account-object-count'], '0')
self.assertEqual(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
self.assertTrue(found1)
self.assertTrue(found2)
# Get to final state
self.get_to_final_state()
# Assert account level now sees the container2/object1
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '2')
self.assertEqual(headers['x-account-object-count'], '1')
self.assertEqual(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEqual(container['count'], 0)
self.assertEqual(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 1)
self.assertEqual(container['bytes'], 4)
self.assertTrue(found1)
self.assertTrue(found2)
apart, anodes = self.account_ring.get_nodes(self.account)
kill_nonprimary_server(anodes, self.ipport2server)
kill_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server)
# Kill account servers excepting two of the primaries
# Delete container1
client.delete_container(self.url, self.token, container1)
# Put container2/object2
client.put_object(self.url, self.token, container2, 'object2', '12345')
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '1')
self.assertEqual(headers['x-account-object-count'], '1')
self.assertEqual(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 1)
self.assertEqual(container['bytes'], 4)
self.assertFalse(found1)
self.assertTrue(found2)
# Run container updaters
Manager(['container-updater']).once()
# Assert account level now knows about container2/object2
headers, containers = client.get_account(self.url, self.token)
self.assertEqual(headers['x-account-container-count'], '1')
self.assertEqual(headers['x-account-object-count'], '2')
self.assertEqual(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 2)
self.assertEqual(container['bytes'], 9)
self.assertFalse(found1)
self.assertTrue(found2)
# Restart other primary account server
start_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server)
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEqual(headers['x-account-container-count'], '2')
self.assertEqual(headers['x-account-object-count'], '1')
self.assertEqual(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 1)
self.assertEqual(container['bytes'], 4)
self.assertTrue(found1)
self.assertTrue(found2)
# Get to final state
self.get_to_final_state()
# Assert that server is now up to date
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEqual(headers['x-account-container-count'], '1')
self.assertEqual(headers['x-account-object-count'], '2')
self.assertEqual(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEqual(container['count'], 2)
self.assertEqual(container['bytes'], 9)
self.assertEqual(container['bytes'], 9)
self.assertFalse(found1)
self.assertTrue(found2)
def test_main(self):
# Create container1 and container2
# Assert account level sees them
# Create container2/object1
# Assert account level doesn't see it yet
# Get to final state
# Assert account level now sees the container2/object1
# Kill account servers excepting two of the primaries
# Delete container1
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
# Put container2/object2
# Run container updaters
# Assert account level now knows about container2/object2
# Restart other primary account server
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
# Get to final state
# Assert that server is now up to date
container1 = 'container1'
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assert_(found1)
self.assert_(found2)
client.put_object(self.url, self.token, container2, 'object1', '1234')
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assert_(found1)
self.assert_(found2)
get_to_final_state()
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
self.assert_(found2)
apart, anodes = self.account_ring.get_nodes(self.account)
kill_nonprimary_server(anodes, self.port2server, self.pids)
kill_server(anodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
client.put_object(self.url, self.token, container2, 'object2', '12345')
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(not found1)
self.assert_(found2)
processes = []
for node in xrange(1, 5):
processes.append(
Popen([
'swift-container-updater',
'/etc/swift/container-server/%d.conf' % node, 'once'
]))
for process in processes:
process.wait()
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assert_(not found1)
self.assert_(found2)
start_server(anodes[0]['port'], self.port2server, self.pids)
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
self.assert_(found2)
get_to_final_state()
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assert_(not found1)
self.assert_(found2)
def test_main(self):
# Create container1 and container2
# Assert account level sees them
# Create container2/object1
# Assert account level doesn't see it yet
# Get to final state
# Assert account level now sees the container2/object1
# Kill account servers excepting two of the primaries
# Delete container1
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
# Put container2/object2
# Run container updaters
# Assert account level now knows about container2/object2
# Restart other primary account server
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
# Get to final state
# Assert that server is now up to date
container1 = 'container1'
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assert_(found1)
self.assert_(found2)
client.put_object(self.url, self.token, container2, 'object1', '1234')
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assert_(found1)
self.assert_(found2)
get_to_final_state()
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
self.assert_(found2)
apart, anodes = self.account_ring.get_nodes(self.account)
kill_nonprimary_server(anodes, self.port2server, self.pids)
kill_server(anodes[0]['port'], self.port2server, self.pids)
client.delete_container(self.url, self.token, container1)
client.put_object(self.url, self.token, container2, 'object2', '12345')
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(not found1)
self.assert_(found2)
processes = []
for node in xrange(1, 5):
processes.append(Popen([
'swift-container-updater',
self.configs['container'] % node,
'once']))
for process in processes:
process.wait()
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assert_(not found1)
self.assert_(found2)
start_server(anodes[0]['port'], self.port2server, self.pids)
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
self.assert_(found2)
get_to_final_state()
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assert_(not found1)
self.assert_(found2)
def test_main(self):
# Create container1 and container2
container1 = 'container1'
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
# Assert account level sees them
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assertTrue(found1)
self.assertTrue(found2)
# Create container2/object1
client.put_object(self.url, self.token, container2, 'object1', '1234')
# Assert account level doesn't see it yet
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
self.assertTrue(found1)
self.assertTrue(found2)
# Get to final state
self.get_to_final_state()
# Assert account level now sees the container2/object1
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
self.assertEquals(container['count'], 0)
self.assertEquals(container['bytes'], 0)
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assertTrue(found1)
self.assertTrue(found2)
apart, anodes = self.account_ring.get_nodes(self.account)
kill_nonprimary_server(anodes, self.ipport2server, self.pids)
kill_server((anodes[0]['ip'], anodes[0]['port']),
self.ipport2server, self.pids)
# Kill account servers excepting two of the primaries
# Delete container1
client.delete_container(self.url, self.token, container1)
# Put container2/object2
client.put_object(self.url, self.token, container2, 'object2', '12345')
# Assert account level knows container1 is gone but doesn't know about
# container2/object2 yet
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assertTrue(not found1)
self.assertTrue(found2)
# Run container updaters
Manager(['container-updater']).once()
# Assert account level now knows about container2/object2
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assertTrue(not found1)
self.assertTrue(found2)
# Restart other primary account server
start_server((anodes[0]['ip'], anodes[0]['port']),
self.ipport2server, self.pids)
# Assert that server doesn't know about container1's deletion or the
# new container2/object2 yet
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assertTrue(found1)
self.assertTrue(found2)
# Get to final state
self.get_to_final_state()
# Assert that server is now up to date
headers, containers = \
direct_client.direct_get_account(anodes[0], apart, self.account)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for container in containers:
if container['name'] == container1:
found1 = True
elif container['name'] == container2:
found2 = True
self.assertEquals(container['count'], 2)
self.assertEquals(container['bytes'], 9)
self.assertTrue(not found1)
self.assertTrue(found2)
def test_update_pending(self):
# Create container
container = 'contx'
client.put_container(self.url, self.token, container)
part, nodes = self.account_ring.get_nodes(self.account)
anode = nodes[0]
# Stop a quorum of account servers
# This allows the put to continue later.
kill_nonprimary_server(nodes, self.ipport2server)
kill_server((anode['ip'], anode['port']), self.ipport2server)
# Put object
# This creates an outstanding update.
client.put_object(self.url, self.token, container, 'object1', b'123')
cont_db_files = self.get_container_db_files(container)
self.assertEqual(len(cont_db_files), 3)
# Collect the observable state from containers
outstanding_files = []
for cfile in cont_db_files:
broker = ContainerBroker(cfile)
try:
info = broker.get_info()
except LockTimeout:
self.fail('LockTimeout at %s' % (cfile, ))
if Timestamp(info['put_timestamp']) <= 0:
self.fail('No put_timestamp at %s' % (cfile, ))
# Correct even if reported_put_timestamp is zero.
if info['put_timestamp'] > info['reported_put_timestamp']:
outstanding_files.append(cfile)
self.assertGreater(len(outstanding_files), 0)
# At this point the users shut everything down and screw up the
# hash in swift.conf. But we destroy the account DB instead.
files = self.get_account_db_files(self.account)
for afile in files:
os.unlink(afile)
# Restart the stopped primary server
start_server((anode['ip'], anode['port']), self.ipport2server)
# Make sure updaters run
Manager(['container-updater']).once()
# Collect the observable state from containers again and examine it
outstanding_files_new = []
for cfile in cont_db_files:
# We aren't catching DatabaseConnectionError, because
# we only want to approve of DBs that were quarantined,
# and not otherwise damaged. So if the code below throws
# an exception for other reason, we want the test to fail.
if not os.path.exists(cfile):
continue
broker = ContainerBroker(cfile)
try:
info = broker.get_info()
except LockTimeout:
self.fail('LockTimeout at %s' % (cfile, ))
if Timestamp(info['put_timestamp']) <= 0:
self.fail('No put_timestamp at %s' % (cfile, ))
# Correct even if reported_put_timestamp is zero.
if info['put_timestamp'] > info['reported_put_timestamp']:
outstanding_files_new.append(cfile)
self.assertLengthEqual(outstanding_files_new, 0)
self.get_to_final_state()
