def test_main(self): # Create container # Kill container servers excepting two of the primaries # Create container/obj # Restart other primary server # Assert it does not know about container/obj # Run the object-updaters # Assert the other primary server now knows about container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnode['port'], self.port2server, self.pids) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') start_server(cnode['port'], self.port2server, self.pids) self.assert_(not direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) processes = [] for node in xrange(1, 5): processes.append(Popen(['swift-object-updater', '/etc/swift/object-server/%d.conf' % node, 'once'])) for process in processes: process.wait() objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] self.assert_(obj in objs)
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server has update with override values obj = direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) # Kill container servers excepting two of the primaries cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.ipport2server, self.pids) kill_server((cnode['ip'], cnode['port']), self.ipport2server, self.pids) # Create container/obj obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') # Restart other primary server start_server((cnode['ip'], cnode['port']), self.ipport2server, self.pids) # Assert it does not know about container/obj self.assert_(not direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) # Run the object-updaters Manager(['object-updater']).once() # Assert the other primary server now knows about container/obj objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] self.assert_(obj in objs)
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server has update with override values obj = direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def test_direct_get_container(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' headers = {'key': 'value'} body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]' was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, headers, body) resp_headers, resp = ( direct_client.direct_get_container(node, part, account, container)) headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(headers, resp_headers) self.assertEqual(json_loads(body), resp) direct_client.http_connect = mock_http_connect(204, headers, body) resp_headers, resp = ( direct_client.direct_get_container(node, part, account, container)) headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(headers, resp_headers) self.assertEqual([], resp) direct_client.http_connect = was_http_connector
def test_two_nodes_fail(self): # Create container1 # Kill container1 servers excepting one of the primaries # Delete container1 directly to the one primary still up # Restart other container1 servers # Get to a final state # Assert all container1 servers indicate container1 is gone (happens # because the one node that knew about the delete replicated to the # others.) # Assert account level also indicates container1 is gone container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) kill_server(cnodes[1]['port'], self.port2server, self.pids) direct_client.direct_delete_container(cnodes[2], cpart, self.account, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) start_server(cnodes[1]['port'], self.port2server, self.pids) start_server(cnp_port, self.port2server, self.pids) get_to_final_state() for cnode in cnodes: exc = None try: direct_client.direct_get_container(cnode, cpart, self.account, container1) except client.ClientException as err: exc = err self.assertEquals(exc.http_status, 404) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '0') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0')
def test_main(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) apart, anodes = self.account_ring.get_nodes(self.account) anode = anodes[0] cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill(self.pids[self.port2server[cnode['port']]], SIGTERM) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') self.pids[self.port2server[cnode['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnode['port'] - 6001) / 10)]).pid sleep(2) self.assert_(not direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) ps = [] for n in xrange(1, 5): ps.append( Popen([ 'swift-object-updater', '/etc/swift/object-server/%d.conf' % n, 'once' ])) for p in ps: p.wait() objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] self.assert_(obj in objs)
def test_main(self): # Create container container = "container-%s" % uuid4() client.put_container(self.url, self.token, container) # Kill container servers excepting two of the primaries cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.ipport2server) kill_server((cnode["ip"], cnode["port"]), self.ipport2server) # Create container/obj obj = "object-%s" % uuid4() client.put_object(self.url, self.token, container, obj, "") # Restart other primary server start_server((cnode["ip"], cnode["port"]), self.ipport2server) # Assert it does not know about container/obj self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1]) # Run the object-updaters Manager(["object-updater"]).once() # Assert the other primary server now knows about container/obj objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]] self.assertIn(obj, objs)
def test_missing_container(self): # In this test, we need to put container at handoff devices, so we # need container devices more than replica count if len(self.container_ring.devs) <= self.container_ring.replica_count: raise SkipTest("Need devices more that replica count") container = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container) # Kill all primary container servers for cnode in cnodes: kill_server((cnode['ip'], cnode['port']), self.ipport2server) # Create container, and all of its replicas are placed at handoff # device try: client.put_container(self.url, self.token, container) except ClientException as err: # if the cluster doesn't have enough devices, swift may return # error (ex. When we only have 4 devices in 3-replica cluster). self.assertEqual(err.http_status, 503) # Assert handoff device has a container replica another_cnode = next(self.container_ring.get_more_nodes(cpart)) direct_client.direct_get_container( another_cnode, cpart, self.account, container) # Restart all primary container servers for cnode in cnodes: start_server((cnode['ip'], cnode['port']), self.ipport2server) # Create container/obj obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') # Run the object-updater Manager(['object-updater']).once() # Run the container-replicator, and now, container replicas # at handoff device get moved to primary servers Manager(['container-replicator']).once() # Assert container replicas in primary servers, just moved by # replicator don't know about the object for cnode in cnodes: self.assertFalse(direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) # since the container is empty - we can delete it! client.delete_container(self.url, self.token, container) # Re-run the object-updaters and now container replicas in primary # container servers should get updated Manager(['object-updater']).once() # Assert all primary container servers know about container/obj for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] self.assertIn(obj, objs)
def test_main(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) apart, anodes = self.account_ring.get_nodes(self.account) anode = anodes[0] cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill(self.pids[self.port2server[cnode['port']]], SIGTERM) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') self.pids[self.port2server[cnode['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnode['port'] - 6001) / 10)]).pid sleep(2) self.assert_(not direct_client.direct_get_container(cnode, cpart, self.account, container)[1]) ps = [] for n in xrange(1, 5): ps.append(Popen(['swift-object-updater', '/etc/swift/object-server/%d.conf' % n, 'once'])) for p in ps: p.wait() objs = [o['name'] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]] self.assert_(obj in objs)
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) # Kill container servers excepting two of the primaries cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.ipport2server) kill_server((cnode['ip'], cnode['port']), self.ipport2server) # Create container/obj obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') # Restart other primary server start_server((cnode['ip'], cnode['port']), self.ipport2server) # Assert it does not know about container/obj self.assertFalse( direct_client.direct_get_container(cnode, cpart, self.account, container)[1]) # Run the object-updaters Manager(['object-updater']).once() # Assert the other primary server now knows about container/obj objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] self.assertTrue(obj in objs)
def test_async_updates_after_PUT_and_POST(self): # verify correct update values when PUT update and POST updates are # missed but then async updates are sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # PUT and POST to object while one container server is stopped so that # we force async updates to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata(self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual('Tomato', int_client.get_object_metadata( self.account, 'c1', 'o1')['x-object-meta-fruit']) # sanity # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to send the async pendings Manager(['object-updater']).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type'])
def test_main(self): # Create container # Kill container servers excepting two of the primaries # Create container/obj # Restart other primary server # Assert it does not know about container/obj # Run the object-updaters # Assert the other primary server now knows about container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnode['port'], self.port2server, self.pids) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') start_server(cnode['port'], self.port2server, self.pids) self.assert_(not direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) processes = [] for node in xrange(1, 5): processes.append( Popen([ 'swift-object-updater', self.configs['object'] % node, 'once' ])) for process in processes: process.wait() objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] self.assert_(obj in objs)
def test_missing_container(self): # In this test, we need to put container at handoff devices, so we # need container devices more than replica count if len(self.container_ring.devs) <= self.container_ring.replica_count: raise SkipTest("Need devices more that replica count") container = "container-%s" % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container) # Kill all primary container servers for cnode in cnodes: kill_server((cnode["ip"], cnode["port"]), self.ipport2server) # Create container, and all of its replicas are placed at handoff # device try: client.put_container(self.url, self.token, container) except ClientException as err: # if the cluster doesn't have enough devices, swift may return # error (ex. When we only have 4 devices in 3-replica cluster). self.assertEqual(err.http_status, 503) # Assert handoff device has a container replica another_cnode = self.container_ring.get_more_nodes(cpart).next() direct_client.direct_get_container(another_cnode, cpart, self.account, container) # Restart all primary container servers for cnode in cnodes: start_server((cnode["ip"], cnode["port"]), self.ipport2server) # Create container/obj obj = "object-%s" % uuid4() client.put_object(self.url, self.token, container, obj, "") # Run the object-updater Manager(["object-updater"]).once() # Run the container-replicator, and now, container replicas # at handoff device get moved to primary servers Manager(["container-replicator"]).once() # Assert container replicas in primary servers, just moved by # replicator don't know about the object for cnode in cnodes: self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1]) # since the container is empty - we can delete it! client.delete_container(self.url, self.token, container) # Re-run the object-updaters and now container replicas in primary # container servers should get updated Manager(["object-updater"]).once() # Assert all primary container servers know about container/obj for cnode in cnodes: objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]] self.assertIn(obj, objs)
def test_update_during_POST_only(self): # verify correct update values when PUT update is missed but then a # POST update succeeds *before* the PUT async pending update is sent cpart, cnodes = self.container_ring.get_nodes(self.account, "c1") client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server) content = u"stuff" client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype") meta = client.head_object(self.url, self.token, "c1", "o1") # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server) self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1]) # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata(self.account, "c1", "o1", {"X-Object-Meta-Fruit": "Tomato"}) self.assertEqual( "Tomato", int_client.get_object_metadata(self.account, "c1", "o1")["x-object-meta-fruit"] ) # sanity # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual("test/ctype", listing[0]["content_type"]) listing_etags.add(listing[0]["hash"]) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, "c1") self.assertEqual(1, len(listing)) self.assertEqual("o1", listing[0]["name"]) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual(meta["etag"], listing[0]["hash"]) self.assertEqual("test/ctype", listing[0]["content_type"]) # Run the object-updaters to send the async pending from the PUT Manager(["object-updater"]).once() # check container listing metadata is still correct for cnode in cnodes: listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual("test/ctype", listing[0]["content_type"]) listing_etags.add(listing[0]["hash"]) self.assertEqual(1, len(listing_etags))
def test_async_updates_after_PUT_and_POST(self): # verify correct update values when PUT update and POST updates are # missed but then async updates are sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # PUT and POST to object while one container server is stopped so that # we force async updates to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata( self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual( 'Tomato', int_client.get_object_metadata(self.account, 'c1', 'o1') ['x-object-meta-fruit']) # sanity # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to send the async pendings Manager(['object-updater']).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type'])
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type'])
def do_test(req_params): headers = HeaderKeyDict({'key': 'value'}) body = (b'[{"hash": "8f4e3", "last_modified": "317260", ' b'"bytes": 209}]') with mocked_http_conn(200, headers, body) as conn: resp_headers, resp = direct_client.direct_get_container( self.node, self.part, self.account, self.container, **req_params) try: self.assertEqual(conn.method, 'GET') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(headers, resp_headers) self.assertEqual(json.loads(body), resp) self.assertIn('format=json', conn.query_string) for k, v in req_params.items(): if v is None: self.assertNotIn('&%s' % k, conn.query_string) else: self.assertIn('&%s=%s' % (k, v), conn.query_string) except AssertionError as err: self.fail('Failed with params %s: %s' % (req_params, err))
def do_test(req_params): headers = HeaderKeyDict({'key': 'value'}) body = ('[{"hash": "8f4e3", "last_modified": "317260", ' '"bytes": 209}]') with mocked_http_conn(200, headers, body) as conn: resp_headers, resp = direct_client.direct_get_container( self.node, self.part, self.account, self.container, **req_params) try: self.assertEqual(conn.method, 'GET') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(headers, resp_headers) self.assertEqual(json.loads(body), resp) self.assertIn('format=json', conn.query_string) for k, v in req_params.items(): if v is None: self.assertNotIn('&%s' % k, conn.query_string) else: self.assertIn('&%s=%s' % (k, v), conn.query_string) except AssertionError as err: self.fail('Failed with params %s: %s' % (req_params, err))
def test_one_node_fails(self): # Create container1 # Kill container1 servers excepting two of the primaries # Delete container1 # Restart other container1 primary server # Create container1/object1 (allowed because at least server thinks the # container exists) # Get to a final state # Assert all container1 servers indicate container1 is alive and # well with object1 # Assert account level also indicates container1 is alive and # well with object1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container1, 'object1', '123') get_to_final_state() for cnode in cnodes: self.assertEquals([ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container1)[1] ], ['object1']) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '3')
def test_direct_get_container(self): headers = HeaderKeyDict({'key': 'value'}) body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]' with mocked_http_conn(200, headers, body) as conn: resp_headers, resp = direct_client.direct_get_container( self.node, self.part, self.account, self.container, marker='marker', prefix='prefix', delimiter='delimiter', limit=1000, end_marker='endmarker', reverse='on') self.assertEqual(conn.req_headers['user-agent'], 'direct-client %s' % os.getpid()) self.assertEqual(headers, resp_headers) self.assertEqual(json.loads(body), resp) self.assertTrue('marker=marker' in conn.query_string) self.assertTrue('delimiter=delimiter' in conn.query_string) self.assertTrue('limit=1000' in conn.query_string) self.assertTrue('prefix=prefix' in conn.query_string) self.assertTrue('format=json' in conn.query_string) self.assertTrue('end_marker=endmarker' in conn.query_string) self.assertTrue('reverse=on' in conn.query_string)
def test_one_node_fails(self): # Create container1 # Kill container1 servers excepting two of the primaries # Delete container1 # Restart other container1 primary server # Create container1/object1 (allowed because at least server thinks the # container exists) # Get to a final state # Assert all container1 servers indicate container1 is alive and # well with object1 # Assert account level also indicates container1 is alive and # well with object1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container1, 'object1', '123') get_to_final_state() for cnode in cnodes: self.assertEquals( [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container1)[1]], ['object1']) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '3')
def test_update_during_POST_only(self): # verify correct update values when PUT update is missed but then a # POST update succeeds *before* the PUT async pending update is sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata(self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual('Tomato', int_client.get_object_metadata( self.account, 'c1', 'o1')['x-object-meta-fruit']) # sanity # check the re-started container server has update with override values obj = direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes']) # Run the object-updaters to send the async pending from the PUT Manager(['object-updater']).once() # check container listing metadata is still correct obj = direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def test_two_nodes_fail(self): # Create container1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) # Kill container1 servers excepting one of the primaries cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server, self.pids) kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server, self.pids) kill_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server, self.pids) # Delete container1 directly to the one primary still up direct_client.direct_delete_container(cnodes[2], cpart, self.account, container1) # Restart other container1 servers start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server, self.pids) start_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server, self.pids) start_server(cnp_ipport, self.ipport2server, self.pids) # Get to a final state self.get_to_final_state() # Assert all container1 servers indicate container1 is gone (happens # because the one node that knew about the delete replicated to the # others.) for cnode in cnodes: try: direct_client.direct_get_container(cnode, cpart, self.account, container1) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert account level also indicates container1 is gone headers, containers = client.get_account(self.url, self.token) self.assertEqual(headers['x-account-container-count'], '0') self.assertEqual(headers['x-account-object-count'], '0') self.assertEqual(headers['x-account-bytes-used'], '0')
def test_update_during_POST_only(self): # verify correct update values when PUT update is missed but then a # POST update succeeds *before* the PUT async pending update is sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata( self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual( 'Tomato', int_client.get_object_metadata(self.account, 'c1', 'o1') ['x-object-meta-fruit']) # sanity # check the re-started container server has update with override values obj = direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes']) # Run the object-updaters to send the async pending from the PUT Manager(['object-updater']).once() # check container listing metadata is still correct obj = direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type'])
def test_direct_get_container_no_content_does_not_decode_body(self): headers = {} body = '' with mocked_http_conn(204, headers, body) as conn: resp_headers, resp = direct_client.direct_get_container( self.node, self.part, self.account, self.container) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(headers, resp_headers) self.assertEqual([], resp)
def test_first_two_nodes_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) for x in xrange(2): kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) for x in xrange(2): self.pids[self.port2server[cnodes[x]['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnodes[x]['port'] - 6001) / 10)]).pid sleep(2) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got # the update that the object was deleted yet), the whole must 503 # (until every is synced up, then the delete would work). exc = None try: client.delete_container(self.url, self.token, container) except client.ClientException, err: exc = err
def test_direct_get_container(self): headers = HeaderKeyDict({'key': 'value'}) body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]' with mocked_http_conn(200, headers, body) as conn: resp_headers, resp = direct_client.direct_get_container( self.node, self.part, self.account, self.container) self.assertEqual(conn.req_headers['user-agent'], 'direct-client %s' % os.getpid()) self.assertEqual(headers, resp_headers) self.assertEqual(json.loads(body), resp)
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, "c1") client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server) content = u"stuff" client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype") meta = client.head_object(self.url, self.token, "c1", "o1") # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server) self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1]) # Run the object-updaters to be sure updates are done Manager(["object-updater"]).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual("test/ctype", listing[0]["content_type"]) listing_etags.add(listing[0]["hash"]) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, "c1") self.assertEqual(1, len(listing)) self.assertEqual("o1", listing[0]["name"]) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual(meta["etag"], listing[0]["hash"]) self.assertEqual("test/ctype", listing[0]["content_type"])
def test_first_two_nodes_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) for x in xrange(2): kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) for x in xrange(2): self.pids[self.port2server[cnodes[x]['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnodes[x]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in direct_client.direct_get_container(cnodes[0], cpart, self.account, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got # the update that the object was deleted yet), the whole must 503 # (until every is synced up, then the delete would work). exc = None try: client.delete_container(self.url, self.token, container) except client.ClientException, err: exc = err
def get_info_1(container_ring, obj_path, logger): path_comps = split_path(obj_path, 1, 3, True) account_name = path_comps[0] container_name = path_comps[1] obj_name = path_comps[2] container_part, container_nodes = \ container_ring.get_nodes(account_name, container_name) if not container_nodes: raise ContainerError() # Perhaps we should do something about the way we select the container # nodes. For now we just shuffle. It spreads the load, but it does not # improve upon the the case when some nodes are down, so auditor slows # to a crawl (if this plugin is enabled). random.shuffle(container_nodes) dark_flag = 0 for node in container_nodes: try: headers, objs = direct_get_container(node, container_part, account_name, container_name, prefix=obj_name, limit=1) except (ClientException, Timeout): # Something is wrong with that server, treat as an error. continue if not objs or objs[0]['name'] != obj_name: dark_flag += 1 continue return objs[0] # We do not ask for a quorum of container servers to know the object. # Even if 1 server knows the object, we return with the info above. # So, we only end here when all servers either have no record of the # object or error out. In such case, even one non-error server means # that the object is dark. if dark_flag: return None raise ContainerError()
def test_direct_get_container(self): headers = HeaderKeyDict({'key': 'value'}) body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]' with mocked_http_conn(200, headers, body) as conn: resp_headers, resp = direct_client.direct_get_container( self.node, self.part, self.account, self.container, marker='marker', prefix='prefix', delimiter='delimiter', limit=1000) self.assertEqual(conn.req_headers['user-agent'], 'direct-client %s' % os.getpid()) self.assertEqual(headers, resp_headers) self.assertEqual(json.loads(body), resp) self.assertTrue('marker=marker' in conn.query_string) self.assertTrue('delimiter=delimiter' in conn.query_string) self.assertTrue('limit=1000' in conn.query_string) self.assertTrue('prefix=prefix' in conn.query_string) self.assertTrue('format=json' in conn.query_string)
def test_one_node_fails(self): # Create container1 container1 = "container-%s" % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) # Kill container1 servers excepting two of the primaries kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]["port"], self.port2server, self.pids) # Delete container1 client.delete_container(self.url, self.token, container1) # Restart other container1 primary server start_server(cnodes[0]["port"], self.port2server, self.pids) # Create container1/object1 (allowed because at least server thinks the # container exists) client.put_object(self.url, self.token, container1, "object1", "123") # Get to a final state self.get_to_final_state() # Assert all container1 servers indicate container1 is alive and # well with object1 for cnode in cnodes: self.assertEquals( [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container1)[1]], ["object1"], ) # Assert account level also indicates container1 is alive and # well with object1 headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers["x-account-container-count"], "1") self.assertEquals(headers["x-account-object-count"], "1") self.assertEquals(headers["x-account-bytes-used"], "3")
def get_info_1(container_ring, obj_path, logger): path_comps = split_path(obj_path, 1, 3, True) account_name = path_comps[0] container_name = path_comps[1] obj_name = path_comps[2] container_part, container_nodes = \ container_ring.get_nodes(account_name, container_name) if not container_nodes: raise ContainerError() # Perhaps we should do something about the way we select the container # nodes. For now we just shuffle. It spreads the load, but it does not # improve upon the the case when some nodes are down, so auditor slows # to a crawl (if this plugin is enabled). random.shuffle(container_nodes) err_flag = 0 for node in container_nodes: try: headers, objs = direct_get_container( node, container_part, account_name, container_name, prefix=obj_name, limit=1) except (ClientException, Timeout): # Something is wrong with that server, treat as an error. err_flag += 1 continue if objs and objs[0]['name'] == obj_name: return objs[0] # We only report the object as dark if all known servers agree that it is. if err_flag: raise ContainerError() return None
def test_main(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) apart, anodes = self.account_ring.get_nodes(self.account) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill all primaries to ensure GET handoff works for node in onodes[1:]: kill(self.pids[self.port2server[node['port']]], SIGTERM) odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) for node in onodes[1:]: self.pids[self.port2server[node['port']]] = Popen([ 'swift-object-server', '/etc/swift/object-server/%d.conf' % ((node['port'] - 6000) / 10)]).pid sleep(2) # We've indirectly verified the handoff node has the object, but let's # directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object(another_onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) self.pids[self.port2server[onode['port']]] = Popen([ 'swift-object-server', '/etc/swift/object-server/%d.conf' % ((onode['port'] - 6000) / 10)]).pid sleep(2) exc = False try: direct_client.direct_get_object(onode, opart, self.account, container, obj) except Exception: exc = True if not exc: raise Exception('Previously downed object server had test object') # Run the extra server last so it'll remove its extra partition ps = [] for n in onodes: ps.append(Popen(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((n['port'] - 6000) / 10), 'once'])) for p in ps: p.wait() call(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((another_onode['port'] - 6000) / 10), 'once']) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = False try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except Exception: exc = True if not exc: raise Exception('Handoff object server still had test object') # Because POST has changed to a COPY by default, POSTs will succeed on all up # nodes now if at least one up node has the object. # kill(self.pids[self.port2server[onode['port']]], SIGTERM) # client.post_object(self.url, self.token, container, obj, # headers={'x-object-meta-probe': 'value'}) # oheaders = client.head_object(self.url, self.token, container, obj) # if oheaders.get('x-object-meta-probe') != 'value': # raise Exception('Metadata incorrect, was %s' % repr(oheaders)) # exc = False # try: # direct_client.direct_get_object(another_onode, opart, self.account, # container, obj) # except Exception: # exc = True # if not exc: # raise Exception('Handoff server claimed it had the object when ' # 'it should not have it') # self.pids[self.port2server[onode['port']]] = Popen([ # 'swift-object-server', # '/etc/swift/object-server/%d.conf' % # ((onode['port'] - 6000) / 10)]).pid # sleep(2) # oheaders = direct_client.direct_get_object(onode, opart, self.account, # container, obj)[0] # if oheaders.get('x-object-meta-probe') == 'value': # raise Exception('Previously downed object server had the new ' # 'metadata when it should not have it') # # Run the extra server last so it'll remove its extra partition # ps = [] # for n in onodes: # ps.append(Popen(['swift-object-replicator', # '/etc/swift/object-server/%d.conf' % # ((n['port'] - 6000) / 10), 'once'])) # for p in ps: # p.wait() # call(['swift-object-replicator', # '/etc/swift/object-server/%d.conf' % # ((another_onode['port'] - 6000) / 10), 'once']) # oheaders = direct_client.direct_get_object(onode, opart, self.account, # container, obj)[0] # if oheaders.get('x-object-meta-probe') != 'value': # raise Exception( # 'Previously downed object server did not have the new metadata') kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.delete_object(self.url, self.token, container, obj) exc = False try: client.head_object(self.url, self.token, container, obj) except Exception: exc = True if not exc: raise Exception('Regular object HEAD was still successful') objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) self.pids[self.port2server[onode['port']]] = Popen([ 'swift-object-server', '/etc/swift/object-server/%d.conf' % ((onode['port'] - 6000) / 10)]).pid sleep(2) direct_client.direct_get_object(onode, opart, self.account, container, obj) # Run the extra server last so it'll remove its extra partition ps = [] for n in onodes: ps.append(Popen(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((n['port'] - 6000) / 10), 'once'])) for p in ps: p.wait() call(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((another_onode['port'] - 6000) / 10), 'once']) exc = False try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except Exception: exc = True if not exc: raise Exception('Handoff object server still had the object')
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) # We've indirectly verified the handoff node has the container/object, # but let's directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) # Bring the first container/obj primary server back up start_server(onode['port'], self.port2server, self.pids) # Assert that it doesn't have container/obj yet try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) kill_server(onode['port'], self.port2server, self.pids) # Delete container/obj try: client.delete_object(self.url, self.token, container, obj) except client.ClientException as err: if self.object_ring.replica_count > 2: raise # Object DELETE returning 503 for (404, 204) # remove this with fix for # https://bugs.launchpad.net/swift/+bug/1318375 self.assertEqual(503, err.http_status) # Assert we can't head container/obj try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert container/obj is not in the container listing, both indirectly # and directly objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) # Restart the first container/obj primary server again start_server(onode['port'], self.port2server, self.pids) # Assert it still has container/obj direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
def test_main(self): # Create container # Kill one container/obj primary server # Delete the "objects" directory on the primary server # Create container/obj (goes to two primary servers and one handoff) # Kill other two container/obj primary servers # Indirectly through proxy assert we can get container/obj # Restart those other two container/obj primary servers # Directly to handoff server assert we can get container/obj # Assert container listing (via proxy and directly) has container/obj # Bring the first container/obj primary server back up # Assert that it doesn't have container/obj yet # Run object replication for first container/obj primary server # Run object replication for handoff node # Assert the first container/obj primary server now has container/obj # Assert the handoff server no longer has container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes(self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) obj_dir = '%s/objects' % self._get_objects_dir(onode) shutil.rmtree(obj_dir, True) self.assertFalse(os.path.exists(obj_dir)) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill all primaries to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) # We've indirectly verified the handoff node has the object, but # let's directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object(another_onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ] if obj not in objs: raise Exception('Container listing did not know about object') timeout = time.time() + 5 found_objs_on_cnode = [] while time.time() < timeout: for cnode in [ c for c in cnodes if cnodes not in found_objs_on_cnode ]: objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] if obj in objs: found_objs_on_cnode.append(cnode) if len(found_objs_on_cnode) >= len(cnodes): break time.sleep(0.3) if len(found_objs_on_cnode) < len(cnodes): missing = [ '%s:%s' % (cnode['ip'], cnode['port']) for cnode in cnodes if cnode not in found_objs_on_cnode ] raise Exception('Container servers %r did not know about object' % missing) start_server(onode['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) exc = None try: direct_client.direct_get_object(onode, opart, self.account, container, obj) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) self.assertFalse(os.path.exists(obj_dir)) try: port_num = onode['replication_port'] except KeyError: port_num = onode['port'] try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] call([ 'swift-object-replicator', self.configs['object-replicator'] % ((port_num - 6000) / 10), 'once' ]) call([ 'swift-object-replicator', self.configs['object-replicator'] % ((another_port_num - 6000) / 10), 'once' ]) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = None try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404)
def test_first_node_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) self.pids[self.port2server[cnodes[0]['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnodes[0]['port'] - 6001) / 10)]).pid sleep(2) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # Unfortunately, the following might pass or fail, depending on the # position of the account server associated with the first container # server we had killed. If the associated happens to be the first # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # And, of course, our new object2 exists. self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ])
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes(self.account, container, obj) onode = onodes[0] kill_server((onode['ip'], onode['port']), self.ipport2server, self.pids) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server((node['ip'], node['port']), self.ipport2server, self.pids) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server((node['ip'], node['port']), self.ipport2server, self.pids) # We've indirectly verified the handoff node has the container/object, # but let's directly verify it. another_onode = next(self.object_ring.get_more_nodes(opart)) odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert container listing (via proxy and directly) has container/obj objs = [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) # Bring the first container/obj primary server back up start_server((onode['ip'], onode['port']), self.ipport2server, self.pids) # Assert that it doesn't have container/obj yet try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) kill_server((onode['ip'], onode['port']), self.ipport2server, self.pids) # Delete container/obj try: client.delete_object(self.url, self.token, container, obj) except client.ClientException as err: if self.object_ring.replica_count > 2: raise # Object DELETE returning 503 for (404, 204) # remove this with fix for # https://bugs.launchpad.net/swift/+bug/1318375 self.assertEqual(503, err.http_status) # Assert we can't head container/obj try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert container/obj is not in the container listing, both indirectly # and directly objs = [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) # Restart the first container/obj primary server again start_server((onode['ip'], onode['port']), self.ipport2server, self.pids) # Assert it still has container/obj direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
class TestContainerFailures(unittest.TestCase): def setUp(self): self.pids, self.port2server, self.account_ring, self.container_ring, \ self.object_ring, self.url, self.token, self.account = \ reset_environment() def tearDown(self): kill_pids(self.pids) def test_first_node_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) self.pids[self.port2server[cnodes[0]['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnodes[0]['port'] - 6001) / 10)]).pid sleep(2) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # Unfortunately, the following might pass or fail, depending on the # position of the account server associated with the first container # server we had killed. If the associated happens to be the first # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # And, of course, our new object2 exists. self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) def test_second_node_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) self.pids[self.port2server[cnodes[1]['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnodes[1]['port'] - 6001) / 10)]).pid sleep(2) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) # Unfortunately, the following might pass or fail, depending on the # position of the account server associated with the first container # server we had killed. If the associated happens to be the first # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # And, of course, our new object2 exists. self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) def test_first_two_nodes_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) for x in xrange(2): kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) for x in xrange(2): self.pids[self.port2server[cnodes[x]['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnodes[x]['port'] - 6001) / 10)]).pid sleep(2) self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got # the update that the object was deleted yet), the whole must 503 # (until every is synced up, then the delete would work). exc = None try: client.delete_container(self.url, self.token, container) except client.ClientException, err: exc = err self.assert_(exc) self.assert_(exc.http_status, 503) # Unfortunately, the following might pass or fail, depending on the # position of the account server associated with the first container # server we had killed. If the associated happens to be the first # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # And, of course, our new object2 exists. self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ])
def test_main(self): # Create container # Kill one container/obj primary server # Create container/obj (goes to two primary servers and one handoff) # Kill other two container/obj primary servers # Indirectly through proxy assert we can get container/obj # Restart those other two container/obj primary servers # Directly to handoff server assert we can get container/obj # Assert container listing (via proxy and directly) has container/obj # Bring the first container/obj primary server back up # Assert that it doesn't have container/obj yet # Run object replication, ensuring we run the handoff node last so it # should remove its extra handoff partition # Assert the first container/obj primary server now has container/obj # Assert the handoff server no longer has container/obj # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) # Delete container/obj # Assert we can't head container/obj # Assert container/obj is not in the container listing, both indirectly # and directly # Restart the first container/obj primary server again # Assert it still has container/obj # Run object replication, ensuring we run the handoff node last so it # should remove its extra handoff partition # Assert primary node no longer has container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes(self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill all primaries to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) # We've indirectly verified the handoff node has the object, but let's # directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object(another_onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) start_server(onode['port'], self.port2server, self.pids) exc = None try: direct_client.direct_get_object(onode, opart, self.account, container, obj) except direct_client.ClientException, err: exc = err
self.assert_(exc) self.assert_(exc.http_status, 503) # Unfortunately, the following might pass or fail, depending on the # position of the account server associated with the first container # server we had killed. If the associated happens to be the first # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [ o['name'] for o in direct_client.direct_get_container( cnodes[0], cpart, self.account, container)[1] ]) # And, of course, our new object2 exists. self.assert_(object2 in [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_( container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized.
def reap_container(self, account, account_partition, account_nodes, container): """ Deletes the data and the container itself for the given container. This will call :func:`reap_object` up to sqrt(self.concurrency) times concurrently for the objects in the container. If there is any exception while deleting a single object, the process will continue for any other objects in the container and the failed objects will be tried again the next time this function is called with the same parameters. If there is any exception while listing the objects for deletion, the process will stop (but will obviously be tried again the next time this function is called with the same parameters). This is a possibility since the listing comes from querying just the primary remote container server. Once all objects have been attempted to be deleted, the container itself will be attempted to be deleted by sending a delete request to all container nodes. The format of the delete request is such that each container server will update a corresponding account server, removing the container from the account's listing. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param account: The name of the account for the container. :param account_partition: The partition for the account on the account ring. :param account_nodes: The primary node dicts for the account. :param container: The name of the container to delete. * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the account node dicts. """ account_nodes = list(account_nodes) part, nodes = self.get_container_ring().get_nodes(account, container) node = nodes[-1] pool = GreenPool(size=self.object_concurrency) marker = '' while True: objects = None try: objects = direct_get_container( node, part, account, container, marker=marker, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout)[1] self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 except ClientException, err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes.get(err.http_status / 100, 0) + 1 if not objects: break try: for obj in objects: if isinstance(obj['name'], unicode): obj['name'] = obj['name'].encode('utf8') pool.spawn(self.reap_object, account, container, part, nodes, obj['name']) pool.waitall() except Exception: self.logger.exception( _('Exception with objects for container ' '%(container)s for account %(account)s'), { 'container': container, 'account': account }) marker = objects[-1]['name']
def test_first_node_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) self.pids[self.port2server[cnodes[0]['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnodes[0]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in direct_client.direct_get_container(cnodes[0], cpart, self.account, container)[1]]) # Unfortunately, the following might pass or fail, depending on the # position of the account server associated with the first container # server we had killed. If the associated happens to be the first # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [o['name'] for o in direct_client.direct_get_container(cnodes[0], cpart, self.account, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]])
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server((onode['ip'], onode['port']), self.ipport2server) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, b'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Stash the on disk data from a primary for future comparison with the # handoff - this may not equal 'VERIFY' if for example the proxy has # crypto enabled direct_get_data = direct_client.direct_get_object( onodes[1], opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server((node['ip'], node['port']), self.ipport2server) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server((node['ip'], node['port']), self.ipport2server) # We've indirectly verified the handoff node has the container/object, # but let's directly verify it. another_onode = next(self.object_ring.get_more_nodes(opart)) odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] self.assertEqual(direct_get_data, odata) # drop a tempfile in the handoff's datadir, like it might have # had if there was an rsync failure while it was previously a # primary handoff_device_path = self.device_dir(another_onode) data_filename = None for root, dirs, files in os.walk(handoff_device_path): for filename in files: if filename.endswith('.data'): data_filename = filename temp_filename = '.%s.6MbL6r' % data_filename temp_filepath = os.path.join(root, temp_filename) if not data_filename: self.fail('Did not find any data files on %r' % handoff_device_path) open(temp_filepath, 'w') # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) # Bring the first container/obj primary server back up start_server((onode['ip'], onode['port']), self.ipport2server) # Assert that it doesn't have container/obj yet try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) // 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) // 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] self.assertEqual(direct_get_data, odata) # and that it does *not* have a temporary rsync dropping! found_data_filename = False primary_device_path = self.device_dir(onode) for root, dirs, files in os.walk(primary_device_path): for filename in files: if filename.endswith('.6MbL6r'): self.fail('Found unexpected file %s' % os.path.join(root, filename)) if filename == data_filename: found_data_filename = True self.assertTrue(found_data_filename, 'Did not find data file %r on %r' % ( data_filename, primary_device_path)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) kill_server((onode['ip'], onode['port']), self.ipport2server) # Delete container/obj try: client.delete_object(self.url, self.token, container, obj) except client.ClientException as err: if self.object_ring.replica_count > 2: raise # Object DELETE returning 503 for (404, 204) # remove this with fix for # https://bugs.launchpad.net/swift/+bug/1318375 self.assertEqual(503, err.http_status) # Assert we can't head container/obj try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert container/obj is not in the container listing, both indirectly # and directly objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) # Restart the first container/obj primary server again start_server((onode['ip'], onode['port']), self.ipport2server) # Assert it still has container/obj direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) // 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) // 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
def test_main(self): # Create container # Kill one container/obj primary server # Delete the "objects" directory on the primary server # Create container/obj (goes to two primary servers and one handoff) # Kill other two container/obj primary servers # Indirectly through proxy assert we can get container/obj # Restart those other two container/obj primary servers # Directly to handoff server assert we can get container/obj # Assert container listing (via proxy and directly) has container/obj # Bring the first container/obj primary server back up # Assert that it doesn't have container/obj yet # Run object replication for first container/obj primary server # Run object replication for handoff node # Assert the first container/obj primary server now has container/obj # Assert the handoff server no longer has container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) obj_dir = '%s/objects' % self._get_objects_dir(onode) shutil.rmtree(obj_dir, True) self.assertFalse(os.path.exists(obj_dir)) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill all primaries to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) # We've indirectly verified the handoff node has the object, but # let's directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') timeout = time.time() + 5 found_objs_on_cnode = [] while time.time() < timeout: for cnode in [c for c in cnodes if cnodes not in found_objs_on_cnode]: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: found_objs_on_cnode.append(cnode) if len(found_objs_on_cnode) >= len(cnodes): break time.sleep(0.3) if len(found_objs_on_cnode) < len(cnodes): missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in cnodes if cnode not in found_objs_on_cnode] raise Exception('Container servers %r did not know about object' % missing) start_server(onode['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) exc = None try: direct_client.direct_get_object(onode, opart, self.account, container, obj) except direct_client.ClientException as err: exc = err self.assertEquals(exc.http_status, 404) self.assertFalse(os.path.exists(obj_dir)) try: port_num = onode['replication_port'] except KeyError: port_num = onode['port'] try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] call(['swift-object-replicator', self.configs['object-replicator'] % ((port_num - 6000) / 10), 'once']) call(['swift-object-replicator', self.configs['object-replicator'] % ((another_port_num - 6000) / 10), 'once']) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = None try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except direct_client.ClientException as err: exc = err self.assertEquals(exc.http_status, 404)
def test_main(self): # Create container # Kill one container/obj primary server # Delete the "objects" directory on the primary server # Create container/obj (goes to two primary servers and one handoff) # Kill other two container/obj primary servers # Indirectly through proxy assert we can get container/obj # Restart those other two container/obj primary servers # Directly to handoff server assert we can get container/obj # Assert container listing (via proxy and directly) has container/obj # Bring the first container/obj primary server back up # Assert that it doesn't have container/obj yet # Run object replication for first container/obj primary server # Run object replication for handoff node # Assert the first container/obj primary server now has container/obj # Assert the handoff server no longer has container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) obj_dir = '%s/objects' % self._get_objects_dir(onode) shutil.rmtree(obj_dir, True) self.assertFalse(os.path.exists(obj_dir)) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill all primaries to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) # We've indirectly verified the handoff node has the object, but # let's directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) start_server(onode['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) exc = None try: direct_client.direct_get_object(onode, opart, self.account, container, obj) except direct_client.ClientException, err: exc = err
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] # Kill one container/obj primary server kill_server(onode['port'], self.port2server, self.pids) # Delete the default data directory for objects on the primary server obj_dir = '%s/%s' % (self._get_objects_dir(onode), get_data_dir(self.policy)) shutil.rmtree(obj_dir, True) self.assertFalse(os.path.exists(obj_dir)) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) # We've indirectly verified the handoff node has the object, but # let's directly verify it. # Directly to handoff server assert we can get container/obj another_onode = next(self.object_ring.get_more_nodes(opart)) odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') timeout = time.time() + 5 found_objs_on_cnode = [] while time.time() < timeout: for cnode in [c for c in cnodes if cnodes not in found_objs_on_cnode]: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: found_objs_on_cnode.append(cnode) if len(found_objs_on_cnode) >= len(cnodes): break time.sleep(0.3) if len(found_objs_on_cnode) < len(cnodes): missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in cnodes if cnode not in found_objs_on_cnode] raise Exception('Container servers %r did not know about object' % missing) # Bring the first container/obj primary server back up start_server(onode['port'], self.port2server, self.pids) # Assert that it doesn't have container/obj yet self.assertFalse(os.path.exists(obj_dir)) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) self.assertFalse(os.path.exists(obj_dir)) else: self.fail("Expected ClientException but didn't get it") try: port_num = onode['replication_port'] except KeyError: port_num = onode['port'] try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] # Run object replication for first container/obj primary server num = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=num) # Run object replication for handoff node another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
self.assert_(exc) self.assert_(exc.http_status, 503) # Unfortunately, the following might pass or fail, depending on the # position of the account server associated with the first container # server we had killed. If the associated happens to be the first # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [o['name'] for o in direct_client.direct_get_container(cnodes[0], cpart, self.account, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]])
def reap_container(self, account, account_partition, account_nodes, container): """ Deletes the data and the container itself for the given container. This will call :func:`reap_object` up to sqrt(self.concurrency) times concurrently for the objects in the container. If there is any exception while deleting a single object, the process will continue for any other objects in the container and the failed objects will be tried again the next time this function is called with the same parameters. If there is any exception while listing the objects for deletion, the process will stop (but will obviously be tried again the next time this function is called with the same parameters). This is a possibility since the listing comes from querying just the primary remote container server. Once all objects have been attempted to be deleted, the container itself will be attempted to be deleted by sending a delete request to all container nodes. The format of the delete request is such that each container server will update a corresponding account server, removing the container from the account's listing. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param account: The name of the account for the container. :param account_partition: The partition for the account on the account ring. :param account_nodes: The primary node dicts for the account. :param container: The name of the container to delete. * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the account node dicts. """ account_nodes = list(account_nodes) part, nodes = self.get_container_ring().get_nodes(account, container) node = nodes[-1] pool = GreenPool(size=self.object_concurrency) marker = '' while True: objects = None try: headers, objects = direct_get_container( node, part, account, container, marker=marker, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout) self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 self.logger.increment('return_codes.2') except ClientException as err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.logger.increment( 'return_codes.%d' % (err.http_status / 100,)) if not objects: break try: policy_index = headers.get('X-Backend-Storage-Policy-Index', 0) for obj in objects: if isinstance(obj['name'], unicode): obj['name'] = obj['name'].encode('utf8') pool.spawn(self.reap_object, account, container, part, nodes, obj['name'], policy_index) pool.waitall() except (Exception, Timeout): self.logger.exception(_('Exception with objects for container ' '%(container)s for account %(account)s' ), {'container': container, 'account': account}) marker = objects[-1]['name'] if marker == '': break successes = 0 failures = 0 for node in nodes: anode = account_nodes.pop() try: direct_delete_container( node, part, account, container, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout, headers={'X-Account-Host': '%(ip)s:%(port)s' % anode, 'X-Account-Partition': str(account_partition), 'X-Account-Device': anode['device'], 'X-Account-Override-Deleted': 'yes'}) successes += 1 self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 self.logger.increment('return_codes.2') except ClientException as err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) failures += 1 self.logger.increment('containers_failures') self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.logger.increment( 'return_codes.%d' % (err.http_status / 100,)) if successes > failures: self.stats_containers_deleted += 1 self.logger.increment('containers_deleted') elif not successes: self.stats_containers_remaining += 1 self.logger.increment('containers_remaining') else: self.stats_containers_possibly_remaining += 1 self.logger.increment('containers_possibly_remaining')
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server((onode['ip'], onode['port']), self.ipport2server) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server((node['ip'], node['port']), self.ipport2server) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server((node['ip'], node['port']), self.ipport2server) # We've indirectly verified the handoff node has the container/object, # but let's directly verify it. another_onode = next(self.object_ring.get_more_nodes(opart)) odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # drop a tempfile in the handoff's datadir, like it might have # had if there was an rsync failure while it was previously a # primary handoff_device_path = self.device_dir('object', another_onode) data_filename = None for root, dirs, files in os.walk(handoff_device_path): for filename in files: if filename.endswith('.data'): data_filename = filename temp_filename = '.%s.6MbL6r' % data_filename temp_filepath = os.path.join(root, temp_filename) if not data_filename: self.fail('Did not find any data files on %r' % handoff_device_path) open(temp_filepath, 'w') # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) # Bring the first container/obj primary server back up start_server((onode['ip'], onode['port']), self.ipport2server) # Assert that it doesn't have container/obj yet try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # and that it does *not* have a temporary rsync dropping! found_data_filename = False primary_device_path = self.device_dir('object', onode) for root, dirs, files in os.walk(primary_device_path): for filename in files: if filename.endswith('.6MbL6r'): self.fail('Found unexpected file %s' % os.path.join(root, filename)) if filename == data_filename: found_data_filename = True self.assertTrue(found_data_filename, 'Did not find data file %r on %r' % ( data_filename, primary_device_path)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) kill_server((onode['ip'], onode['port']), self.ipport2server) # Delete container/obj try: client.delete_object(self.url, self.token, container, obj) except client.ClientException as err: if self.object_ring.replica_count > 2: raise # Object DELETE returning 503 for (404, 204) # remove this with fix for # https://bugs.launchpad.net/swift/+bug/1318375 self.assertEqual(503, err.http_status) # Assert we can't head container/obj try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert container/obj is not in the container listing, both indirectly # and directly objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) # Restart the first container/obj primary server again start_server((onode['ip'], onode['port']), self.ipport2server) # Assert it still has container/obj direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")